def get_instrument_quote_close_dict_by_period(db_session, instrument_id, start_date, end_date, is_primary=False):
quote_close_dto_list = QuoteCloseService.get_instrument_quote_close_list_by_period(db_session, instrument_id,
start_date, end_date,
is_primary)
# 转化为字典类型
quote_close_dto_dict = {}
for quote_close in quote_close_dto_list:
quote_close_dto_dict[quote_close.tradeDate] = quote_close
# 校验缺失的收盘价数据
missing_dates = []
holidays = TradingDayService.get_holiday_list(db_session, start_date, end_date)
trade_date = start_date
while trade_date <= end_date:
if DateTimeUtils.is_trading_day(trade_date, holidays):
if trade_date not in quote_close_dto_dict:
# 将缺失的日期添加到诊断中
missing_dates.append(trade_date)
# 是否需要将缺失的数据按None补齐
quote_close_dto_dict[trade_date] = None
trade_date += timedelta(days=1)
# 添加诊断信息
diagnostics = []
if len(missing_dates) != 0:
message = '标的物%s在[%s, %s]时段内中缺失%d条收盘数据' % (
instrument_id, DateTimeUtils.date2str(start_date), DateTimeUtils.date2str(end_date), len(missing_dates))
diagnostics.append(DiagnosticDTO(DiagnosticType.WARNING, message, missing_dates))
logging.debug(message)
return quote_close_dto_dict, diagnostics
def check_date_params(db_session, start_date, end_date, window):
if start_date > date.today() or end_date > date.today():
raise CustomException('所选日期不能超过今天')
if TradingDayService.get_effective_days_num(db_session, start_date,
end_date) < window:
raise CustomException('所选日期范围[%s,%s]交易日个数小于窗口大小，无法进行计算' %
(DateTimeUtils.date2str(start_date),
DateTimeUtils.date2str(end_date)))
def get_future_contract_order(db_session, contract_type, exist_dates,
start_date, end_date):
holidays = TradingCalendarRepo.get_holiday_list(db_session)
special_dates = []
if start_date > end_date:
logging.error(
"日期指定非法:%s 大于 %s" % DateTimeUtils.date2str(start_date),
DateTimeUtils.date2str(end_date))
return None
else:
all_dates = [
DateTimeUtils.str2date(date.strftime("%Y-%m-%d"))
for date in rrule(DAILY, dtstart=start_date, until=end_date)
]
if exist_dates:
missing_dates = list(set(all_dates) - set(exist_dates))
else:
missing_dates = all_dates
listed_date, delisted_date = DateTimeUtils.offset_trade_date(
start_date, end_date, holidays, special_dates)
close_dict = CommodityFutureContractService.get_corresponding_close(
db_session, contract_type, listed_date, delisted_date)
instruments_list = db_session.query(Instrument).filter(
Instrument.contractType == contract_type)
fc_list = []
if close_dict:
for trade_date in missing_dates:
if DateTimeUtils.is_trading_day(trade_date, holidays,
special_dates):
logging.info("处理交易日%s" %
DateTimeUtils.date2str(trade_date))
previous_trade_date, _ = DateTimeUtils.offset_trade_date(
trade_date, trade_date, holidays, special_dates)
pri, sec = FutureContractInfoAlgo.calc_one_future_contract_info(
instruments_list, close_dict, contract_type,
previous_trade_date)
if pri:
fc_info = FutureContractInfo(
contractType=contract_type,
tradeDate=trade_date,
primaryContractId=pri,
secondaryContractId=sec)
logging.info(
'合约的主力和次主力为：%s %s %s %s' %
(fc_info.contractType,
DateTimeUtils.date2str(fc_info.tradeDate),
fc_info.primaryContractId,
fc_info.secondaryContractId))
fc_list.append(fc_info)
else:
logging.info("跳过非交易日%s" %
DateTimeUtils.date2str(trade_date))
else:
fc_list = None
return fc_list
def delete_data_by_dates(db_session, start_date, end_date):
"""
根据日期和标的删除数据
:param db_session:
:param start_date:
:param end_date:
:return:
"""
logger.info('准备删除: %s 至 %s时间段内的标数据' % (start_date, end_date))
db_session.query(ROTCPosition). \
filter(ROTCPosition.REPORTDATE >= DateTimeUtils.date2str(start_date),
ROTCPosition.REPORTDATE <= DateTimeUtils.date2str(end_date)). \
delete(synchronize_session=False)
db_session.commit()
logger.info('成功删除: %s 至 %s时间段内的标的数据' % (start_date, end_date))
def get_primary_instrument_id(db_session, instrument_id, trade_date,
is_primary):
if InstrumentService.is_commodity_variety_type(db_session,
instrument_id):
# 如果传入的是大宗合约类型, 则直接获取当天的主力合约代码
variety_type = instrument_id
else:
if is_primary is False:
return instrument_id
# 如果传入的是具体的大宗标的代码, 则直接获取当天的主力合约代码
variety_type = InstrumentService.get_commodity_variety_type(
db_session, instrument_id)
# 如果获取到的合约种类为空, 则说明该标的不是大宗商品, 直接返回标的代码
if variety_type is None:
instrument = InstrumentRepo.get_instrument(db_session,
instrument_id)
if instrument is None:
raise CustomException('不支持标的%s' % instrument_id)
return instrument_id
else:
future_contract = CommodityFutureContractRepo.get_commodity_future_contract(
db_session, variety_type, trade_date)
if future_contract is None or future_contract.primaryContractId is None:
raise CustomException(
"合约类型%s在%s下无主力合约数据" %
(variety_type, DateTimeUtils.date2str(trade_date)))
return future_contract.primaryContractId
async def get_instrument_id_list(self,
trade_date=DateTimeUtils.date2str(
date.today()),
filtering=True):
with self.make_session() as db_session:
instrument_id_list = InstrumentService.\
get_instrument_id_list(db_session, DateTimeUtils.str2date(trade_date), filtering)
return instrument_id_list
def calc_instrument_atm_vol_list(db_session, underlyer, start_date,
end_date, is_primary):
atm_quote_list = OtcAtmQuoteService.get_instrument_atm_quote_list_by_period(
db_session, underlyer, start_date, end_date, is_primary)
# TODO：去掉重复数据,相同valuation date, underlyer, source, expire date的数据需要去重
# 根据valuation date进行分类
atm_quote_dict = {}
for atm_quote in atm_quote_list:
if atm_quote.valuationDate not in atm_quote_dict:
atm_quote_dict[atm_quote.valuationDate] = []
atm_quote_dict[atm_quote.valuationDate].append(atm_quote)
# 取一个自然月后到期日的前后3天的ask_vol和bid_vol的平均值作为当天的atm_vol
atm_vol_dict = {}
for valuation_date in atm_quote_dict:
quote_list = atm_quote_dict[valuation_date]
atm_vol = OtcAtmQuoteService.calc_one_atm_vol(
valuation_date, quote_list)
if atm_vol is not None:
atm_vol_dict[valuation_date] = atm_vol
# 检测哪些日期缺失
missing_dates = []
holidays = TradingDayService.get_holiday_list(db_session, start_date,
end_date)
trade_date = start_date
while trade_date <= end_date:
if DateTimeUtils.is_trading_day(trade_date, holidays):
if trade_date not in atm_vol_dict:
# 将缺失的日期添加到诊断中
missing_dates.append(trade_date)
# TODO： 是否需要将缺失的数据按0补足
trade_date += timedelta(days=1)
# 添加诊断信息
diagnostics = []
if len(missing_dates) != 0:
message = '标的物%s在[%s, %s]时段内中缺失%d条ATM Vol数据' % (
underlyer, DateTimeUtils.date2str(start_date),
DateTimeUtils.date2str(end_date), len(missing_dates))
diagnostics.append(
DiagnosticDTO(DiagnosticType.WARNING, message, missing_dates))
logging.error(message)
return atm_vol_dict.values(), diagnostics
def get_record_ids_by_date(db_session, start_date, end_date):
"""
根据传入的时间,获取数据
:param db_session:
:param start_date:
:param end_date:
:return:
"""
logger.info('开始获取数据平台r_otc_position表 %s 到 %s 的数据' %
(start_date, end_date))
record_ids = db_session.query(ROTCPosition.RECORDID).filter(
ROTCPosition.REPORTDATE >= DateTimeUtils.date2str(start_date),
ROTCPosition.REPORTDATE <= DateTimeUtils.date2str(end_date)).all()
if len(record_ids) == 0 or record_ids is None:
logger.info('数据平台r_otc_position表 %s 到 %s 之间没有数据' %
(start_date, end_date))
return []
record_ids = [record_id.RECORDID for record_id in record_ids]
logger.info('数据平台r_otc_position表 %s 到 %s 数据长度为: %d' %
(start_date, end_date, len(record_ids)))
return record_ids
def calc_one_atm_vol(valuation_date, atm_quote_list):
effective_vol_list = []
effective_date_set = OtcAtmQuoteService.get_effective_expire_date_set(
valuation_date)
for quote in atm_quote_list:
expire_date = quote.expireDate
if expire_date not in effective_date_set:
logging.info(
'ATM quote不在有效的到期范围内，当前日期： %s，估值日期： %s, Quote UUID: %s' %
(DateTimeUtils.date2str(expire_date),
DateTimeUtils.date2str(valuation_date), quote.uuid))
continue
if quote.askVol is not None:
effective_vol_list.append(quote.askVol)
if quote.bidVol is not None:
effective_vol_list.append(quote.bidVol)
if len(effective_vol_list) == 0:
return None
atm_vol = RealizedVolDTO()
atm_vol.tradeDate = valuation_date
atm_vol.vol = np.mean(effective_vol_list)
return atm_vol
def get_otc_position_by_date(start_date, end_date):
"""
根据时间段获取数据
:param start_date:
:param end_date:
:return:
"""
logger.info('开始获取%s 至 %s时间段内r_otc_postion表的数据' %
(start_date, end_date))
oracle_session = Oracle_Session()
r_otc_positions = oracle_session.query(ROTCPosition).filter(
ROTCPosition.REPORTDATE >= DateTimeUtils.date2str(start_date),
ROTCPosition.REPORTDATE <= DateTimeUtils.date2str(end_date)).all()
oracle_session.close()
if len(r_otc_positions) == 0 or r_otc_positions is None:
logger.info('在%s 至 %s时间段内r_otc_postion表没有数据' %
(start_date, end_date))
return []
logger.info('在%s 至 %s时间段内的获取了 %d条r_otc_postion表的数据' %
(start_date, end_date, len(r_otc_positions)))
return r_otc_positions
def default(self, o):
# See "Date Time String Format" in the ECMA-262 specification.
if isinstance(o, datetime):
r = o.isoformat()
if o.microsecond:
r = r[:23] + r[26:]
if r.endswith('+00:00'):
r = r[:-6] + 'Z'
return r
elif isinstance(o, date):
return DateTimeUtils.date2str(o)
elif isinstance(o, uuid.UUID):
return str(o)
elif isinstance(o, BaseDTO):
return o.__dict__
else:
return super().default(o)
def clean_one_day_implied_vol(db_session, report_date):
"""
清洗一天的数据
:param db_session:
:param report_date:
:return:
"""
report_date = DateTimeUtils.date2str(report_date)
trans_code_list = ROTCPositionRepo.get_trans_code_list_by_date(
db_session, report_date)
logging.info('正在获取持仓表数据长度为: %d' % (len(trans_code_list)))
position_data_list = ROTCPositionRepo.get_otc_position_by_trans_codes(
db_session, report_date, trans_code_list)
trans_code_list = [
position_data.TRANSCODE for position_data in position_data_list
]
logging.info('正在获取成交表数据,长度为: %d' % (len(trans_code_list)))
trade_data_dict = ROTCTradedataRepo.get_trade_data_by_trans_codes(
db_session, report_date, trans_code_list)
clean_position_dict = {}
for index, position_data in enumerate(position_data_list):
trans_code = position_data.TRANSCODE
trade_data = trade_data_dict.get(trans_code)
if trade_data is not None:
try:
logging.debug('正在组合数据, 当前进度为: %d of %d' %
(index, len(position_data_list)))
clean_position = ROTCPositionService.organize_trade_data(
position_data, trade_data)
implied_vol = clean_position.get('implied_vol')
if implied_vol is not None:
clean_position_dict[trans_code] = clean_position
else:
# TODO: 如果没有implied_vol，则直接根据价格反推隐含波动率，按以下规则进行
# 除了价格，利率 / 分红，还需要检查行权价，到期日和期权类型：
# 行权价可能需要注意是否为百分比，如果为百分比，需要继续检查是否有期初价格。
# 反推隐含波动率需要知道每份期权的价格，因此价格需要检查是否有对应的数量。没有数量，则需要名义本金，参与率，期限（如果年化），期初价格。
clean_position_dict[trans_code] = None
except Exception as e:
logging.error('交易数据转化错误，trans_code: %s, 异常：%s' %
(trans_code, str(e)))
return clean_position_dict
def get_a_index_eod_prices_by_date(trade_date):
"""
根据时间获取数据
:param trade_date:
:return:
"""
logger.info('开始从a_index_eod_prices表获取日期为: %s的数据' % trade_date)
oracle_session = Oracle_Session()
a_index_eod_prices = oracle_session.query(AIndexEODPrices). \
filter(AIndexEODPrices.TRADE_DT == DateTimeUtils.date2str(trade_date, '%Y%m%d')).all()
oracle_session.close()
if len(a_index_eod_prices) == 0 or a_index_eod_prices is None:
logger.info('从a_index_eod_prices表没有获取到日期为: %s的数据' % trade_date)
return []
logger.info('时间为: %s,在a_index_eod_prices表查询到了%d条数据' %
(trade_date, len(a_index_eod_prices)))
return a_index_eod_prices
def update_eod_date(start_date_str=EODDateConfig.eod_start_date,
end_date_str=DateTimeUtils.date2str(date.today()),
eod_cutoff_time=EODDateConfig.eod_cutoff_time):
end_date = DateTimeUtils.str2date(end_date_str)
start_date = DateTimeUtils.str2date(start_date_str)
# 获取节假日信息
db_session = create_db_session()
# TODO: 需要从配置文件读取
if not TradingDayService.is_trading_day(db_session, end_date):
# 如果当天不是交易日则直接计算上一个交易日的值
end_date = TradingDayService.go_trading_day(db_session, end_date, 1, -1)
else:
# 如果是交易日，小于cutoff time还是返回上一个交易日
if datetime.now().time() < DateTimeUtils.str2time(eod_cutoff_time):
end_date = TradingDayService.go_trading_day(db_session, end_date, 1, -1)
else:
end_date = end_date
# 设置变量到airflow variable
AirflowVariableUtils.set_eod_start_date(start_date)
AirflowVariableUtils.set_eod_end_date(end_date)
def get_otc_vol_surface_params(model_info, now_price, observed_date):
"""
构建bct接口请求参数
:param vol_surface:
:param observed_date: 观察日
:param percent_ls:
:return:
"""
model_info['useVolCalendar'] = True
model_info['useCalendarForTenor'] = True
model_info['calendars'] = ["DEFAULT_CALENDAR"]
model_info['volCalendar'] = 'DEFAULT_VOL_CALENDAR'
model_info['valuationDate'] = DateTimeUtils.date2str(observed_date)
model_info['save'] = True
# TODO 定义的结构中没有instance
model_info['instance'] = model_info['underlyer']['instance']
# now price
model_info['underlyer']['quote'] = now_price
return model_info
def get_gold_spot_eod_prices_by_date(trade_date):
"""
根据时间段获取数据
:param trade_date:
:return:
"""
logger.info('开始从gold_spot_eod_prices表获取日期为: %s的数据' % trade_date)
oracle_session = Oracle_Session()
gold_spot_eod_prices = oracle_session.query(GoldSpotEODPrices). \
filter(GoldSpotEODPrices.TRADE_DT == DateTimeUtils.date2str(trade_date, '%Y%m%d')).all()
oracle_session.close()
if len(gold_spot_eod_prices) == 0 or gold_spot_eod_prices is None:
logger.info('从gold_spot_eod_prices表没有获取到日期为: %s的数据' % trade_date)
return []
logger.info('时间为: %s,在gold_spot_eod_prices表查询到了%d条数据' %
(trade_date, len(gold_spot_eod_prices)))
return gold_spot_eod_prices
def get_commodity_futures_eod_prices_by_date(trade_date):
"""
根据时间获取数据
:param trade_date:
:return:
"""
logger.info('开始从commodity_futures_eod_prices表获取日期为: %s的数据' %
trade_date)
oracle_session = Oracle_Session()
commodity_futures_eod_prices = oracle_session.query(CommodityFuturesEODPrices). \
filter(CommodityFuturesEODPrices.TRADE_DT == DateTimeUtils.date2str(trade_date, '%Y%m%d')).all()
oracle_session.close()
if len(commodity_futures_eod_prices
) == 0 or commodity_futures_eod_prices is None:
logger.info('从commodity_futures_eod_prices表没有获取到日期为: %s的数据' %
trade_date)
return []
logger.info('时间为: %s,在commodity_futures_eod_prices表查询到了%d条数据' %
(trade_date, len(commodity_futures_eod_prices)))
return commodity_futures_eod_prices
def get_heat_map(db_session, trade_date):
redis_session = RedisUtils.get_redis_session()
# 获取当天所主力合约
primary_contracts = CommodityFutureContractRepo.get_all_primary_contracts_by_date(
db_session, trade_date)
primary_contract_ids = [
primary_contract.primaryContractId
for primary_contract in primary_contracts
]
# 获取instruments
instruments = InstrumentRepo.get_instruments_by_ids(
db_session, primary_contract_ids)
# 默认开始日期向前半年
start_date = trade_date - timedelta(days=183)
end_date = trade_date
heat_map_list, diagnostic_list = [], []
for index, instrument in enumerate(instruments):
instrument_id = instrument.instrumentId
short_name = instrument.shortName
contract_type = instrument.contractType
key = (contract_type + DateTimeUtils.date2str(trade_date))
heat_map_bytes = redis_session.get(key)
if heat_map_bytes:
heat_map_schema = HeatMapSchema(many=False, exclude=[])
heat_map_dto = heat_map_schema.loads(heat_map_bytes).data
heat_map_list.append(heat_map_dto)
else:
heat_map_dto, diagnostics = OtcAtmQuoteService.calc_one_instrument_heat_map(
db_session, short_name, contract_type, instrument_id,
start_date, end_date)
heat_map_schema = HeatMapSchema(many=False, exclude=[])
heat_map_str = heat_map_schema.dumps(heat_map_dto).data
redis_session.set(key, heat_map_str)
heat_map_list.append(heat_map_dto)
diagnostic_list.append(diagnostics)
return heat_map_list, diagnostic_list
def set_eod_start_date(date):
return Variable.set(Constants.EOD_START_DATE,
DateTimeUtils.date2str(date))
def update_all_instrument_vol_surface_to_terminal(observed_date):
db_session = create_db_session()
holidays = TradingCalendarRepo.get_holiday_list(db_session)
special_dates = []
if not DateTimeUtils.is_trading_day(observed_date, holidays,
special_dates):
observed_date = DateTimeUtils.get_trading_day(observed_date, holidays,
special_dates, 1, -1)
# 登陆
logging.info('update all instrument vol surface to bct观察日为: %s' %
DateTimeUtils.date2str(observed_date))
header = server_utils.login_terminal()
# 从终端加载需要的标的物
terminal_instrument_ids = [
instrument['instrumentId']
for instrument in server_utils.get_instrument_list(header)
]
logging.info(
'从终端加载到的标的物为：%s, 长度为: %d' %
(','.join(terminal_instrument_ids), len(terminal_instrument_ids)))
# 获取数据库中标的物
vol_surface_list = VolSurfaceRepo.get_vol_surface_by_observed_date(
db_session, observed_date)
if not vol_surface_list:
logging.info('observed_date为: %s没有可以导入到终端的vol surface' % observed_date)
# 将instruments去重
unique_vol_surface_list = unique_instruments(vol_surface_list)
logging.info('在trading_date为: %s时,vol surface 表获取到的标的数量为: %d' %
(observed_date, len(unique_vol_surface_list)))
# 写入vol surface 到 bct
failed_instrument_ids = []
# hacky here, 由于bct接收的tenor是自然日，需要将vol surface中工作日转为自然日
for index, unique_vol_surface in enumerate(unique_vol_surface_list):
# try:
instrument_id = unique_vol_surface.instrumentId
a = unique_vol_surface.modelInfo
model_info = unique_vol_surface.modelInfo
if model_info is None:
continue
# 获取到当前的vol surface的实时行情spot价格
market_data = get_real_time_market_data_by_instrument_id(instrument_id)
if market_data is not None and market_data.get(
'lastPrice') is not None:
now_price = market_data.get('lastPrice')
else:
logging.error('标的物%s当前行情为空' % instrument_id)
continue
if unique_vol_surface.instrumentId not in terminal_instrument_ids:
logging.info('当前标的: %s的vol surface不在终端需要的列表里' %
unique_vol_surface.instrumentId)
continue
logging.info('当前准备写入到bct的标的为: %s, 进度为: %d of %d' %
(unique_vol_surface.instrumentId, index + 1,
len(unique_vol_surface_list)))
result = update_one_instrument_to_terminal(unique_vol_surface, header,
now_price, observed_date)
if not result:
failed_instrument_ids.append(instrument_id)
time.sleep(0.3)
# except Exception as e:
# logging.error('更新标的物%s波动率曲面失败， 异常:%s' % (instrument_id, str(e)))
logging.error(
'处理失败的标的物为：%d %s' %
(len(failed_instrument_ids), ','.join(failed_instrument_ids)))
db_session.close()
def set_eod_end_date(date):
return Variable.set(Constants.EOD_END_DATE,
DateTimeUtils.date2str(date))