def __calc(self):
def run_requests(requests_: list, provider_, create_event_loop: bool):
if create_event_loop:
asyncio.set_event_loop(asyncio.new_event_loop())
results = queue.Queue()
done = False
try:
with session:
provider_.run(requests_,
results,
1000,
progress_bar,
timeout=self.__timeout)
except Exception as e:
provider_.enqueue(results,
((k, e) for k in self.__pending.keys()))
while not done:
done, chunk_results = provider_.drain_queue(results)
for (risk_key_, priceable_), result in chunk_results:
if (risk_key_, priceable_) in self.__pending:
future = self.__pending.pop((risk_key_, priceable_))
if self.__use_cache:
PricingCache.put(risk_key_, priceable_, result)
future.set_result(result)
while self.__pending:
(risk_key_, _), future = self.__pending.popitem()
future.set_result(ErrorValue(risk_key_, 'No result returned'))
# Group requests optimally
requests_by_provider = {}
for (key, instrument) in self.__pending.keys():
dates_markets, measures = requests_by_provider.setdefault(key.provider, {})\
.setdefault((key.params, key.scenario), {})\
.setdefault(instrument, (set(), set()))
dates_markets.add((key.date, key.market))
measures.add(key.risk_measure)
requests_for_provider = {}
if requests_by_provider:
session = GsSession.current
request_visible_to_gs = session.is_internal(
) if self.__visible_to_gs is None else self.__visible_to_gs
for provider, by_params_scenario in requests_by_provider.items():
grouped_requests = {}
for (
params, scenario
), positions_by_dates_markets_measures in by_params_scenario.items(
):
for instrument, (
dates_markets, risk_measures
) in positions_by_dates_markets_measures.items():
grouped_requests.setdefault(
(params, scenario, tuple(sorted(dates_markets)),
tuple(sorted(risk_measures))),
[]).append(instrument)
requests = []
# Restrict to 1,000 instruments and 1 date in a batch, until server side changes are made
for (params, scenario, dates_markets,
risk_measures), instruments in grouped_requests.items():
for insts_chunk in [
tuple(filter(None, i))
for i in zip_longest(*[iter(instruments)] * 1000)
]:
for dates_chunk in [
tuple(filter(None, i))
for i in zip_longest(*[iter(dates_markets)] *
1)
]:
requests.append(
RiskRequest(
tuple(
RiskPosition(
instrument=i,
quantity=i.instrument_quantity)
for i in insts_chunk),
risk_measures,
parameters=self._parameters,
wait_for_results=not self.__is_batch,
scenario=scenario,
pricing_and_market_data_as_of=tuple(
PricingDateAndMarketDataAsOf(
pricing_date=d, market=m)
for d, m in dates_chunk),
request_visible_to_gs=request_visible_to_gs
))
requests_for_provider[provider] = requests
show_status = self.__show_progress and\
(len(requests_for_provider) > 1 or len(next(iter(requests_for_provider.values()))) > 1)
request_pool = ThreadPoolExecutor(len(requests_for_provider))\
if len(requests_for_provider) > 1 or self.__is_async else None
progress_bar = tqdm(total=len(self.__pending),
position=0,
maxinterval=1,
file=sys.stdout) if show_status else None
completion_futures = []
for provider, requests in requests_for_provider.items():
if request_pool:
completion_future = request_pool.submit(
run_requests, requests, provider, True)
if not self.__is_async:
completion_futures.append(completion_future)
else:
run_requests(requests, provider, False)
# Wait on results if not async, so exceptions are surfaced
if request_pool:
request_pool.shutdown(False)
all(f.result() for f in completion_futures)
def _calc(self):
def run_request(request: RiskRequest, session: GsSession):
calc_result = {}
try:
with session:
calc_result = provider.calc(request)
except Exception as e:
for risk_measure in request.measures:
measure_results = {}
for result_position in risk_request.positions:
measure_results[result_position] = str(e)
calc_result[risk_measure] = measure_results
finally:
self._handle_results(calc_result)
from gs_quant.api.risk import RiskApi
def get_batch_results(request: RiskRequest, session: GsSession,
batch_provider: RiskApi, batch_result_id: str):
with session:
results = batch_provider.get_results(request, batch_result_id)
self._handle_results(results)
batch_results = []
pool = ThreadPoolExecutor(len(self.__risk_measures_by_provider_and_position)) if self.__is_async else None
while self.__risk_measures_by_provider_and_position:
provider, risk_measures_by_position = self.__risk_measures_by_provider_and_position.popitem()
positions_by_risk_measures = {}
for position, risk_measures in risk_measures_by_position.items():
positions_by_risk_measures.setdefault(tuple(risk_measures), []).append(position)
for risk_measures, positions in positions_by_risk_measures.items():
risk_request = RiskRequest(
tuple(positions),
tuple(sorted(risk_measures, key=lambda m: m.name or m.measure_type.value)),
parameters=RiskRequestParameters(self.__csa_term),
wait_for_results=not self.__is_batch,
pricing_location=self.market_data_location,
scenario=MarketDataScenario(scenario=Scenario.current) if Scenario.current_is_set else None,
pricing_and_market_data_as_of=self._pricing_market_data_as_of,
request_visible_to_gs=self.__visible_to_gs
)
if self.__is_batch:
batch_results.append((provider, risk_request, provider.calc(risk_request)))
elif pool:
pool.submit(run_request, risk_request, GsSession.current)
else:
run_request(risk_request, GsSession.current)
for provider, risk_request, result_id in batch_results:
if pool:
pool.submit(get_batch_results, risk_request, GsSession.current, provider, result_id)
else:
get_batch_results(risk_request, GsSession.current, provider, result_id)
if pool:
pool.shutdown(wait=not self.__is_async)
def __calc(self):
def run_requests(requests_: list, provider_, create_event_loop: bool):
if create_event_loop:
asyncio.set_event_loop(asyncio.new_event_loop())
results = {}
try:
with session:
results = provider_.run(requests_,
1000,
progress_bar,
timeout=self.__timeout)
except Exception as e:
results = {k: e for k in self.__pending.keys()}
finally:
while self.__pending:
(risk_key_, priceable_), future = self.__pending.popitem()
result = results.get((risk_key_, priceable_))
if result is not None:
if self.__use_cache:
PricingCache.put(risk_key_, priceable_, result)
else:
result = ErrorValue(risk_key_, 'No result returned')
future.set_result(result)
# Group requests optimally
requests_by_provider = {}
for (key, priceable) in self.__pending.keys():
requests_by_provider.setdefault(key.provider, {})\
.setdefault((key.params, key.scenario, key.date, key.market), {})\
.setdefault(priceable, set())\
.add(key.risk_measure)
requests_for_provider = {}
if requests_by_provider:
for provider, by_params_scenario_date_market in requests_by_provider.items(
):
grouped_requests = {}
for (
params, scenario, date, market
), positions_by_measures in by_params_scenario_date_market.items(
):
for priceable, risk_measures in positions_by_measures.items(
):
grouped_requests.setdefault(
(params, scenario, date, market,
tuple(sorted(risk_measures))),
[]).append(priceable)
requests = [
RiskRequest(tuple(
RiskPosition(instrument=p, quantity=p.get_quantity())
for p in priceables),
risk_measures,
parameters=self._parameters,
wait_for_results=not self.__is_batch,
scenario=scenario,
pricing_and_market_data_as_of=(
PricingDateAndMarketDataAsOf(
pricing_date=date, market=market), ),
request_visible_to_gs=self.__visible_to_gs)
for (
params, scenario, date, market,
risk_measures), priceables in grouped_requests.items()
]
requests_for_provider[provider] = requests
session = GsSession.current
show_status = self.__show_progress and\
(len(requests_for_provider) > 1 or len(next(iter(requests_for_provider.values()))) > 1)
request_pool = ThreadPoolExecutor(len(requests_for_provider))\
if len(requests_for_provider) > 1 or self.__is_async else None
progress_bar = tqdm(total=len(self.__pending),
position=0,
maxinterval=1) if show_status else None
completion_futures = []
for provider, requests in requests_for_provider.items():
if request_pool:
completion_future = request_pool.submit(
run_requests, requests, provider, True)
if not self.__is_async:
completion_futures.append(completion_future)
else:
run_requests(requests, provider, False)
# Wait on results if not async, so exceptions are surfaced
if request_pool:
request_pool.shutdown(False)
all(f.result() for f in completion_futures)
def __calc(self):
session = GsSession.current
requests_by_provider = {}
def run_requests(requests_: Iterable[RiskRequest], provider_):
results = {}
try:
with session:
results = provider_.calc_multi(requests_)
if self.__is_batch:
results = provider_.get_results(
dict(zip(results, requests_)),
timeout=self.__batch_results_timeout).values()
except Exception as e:
results = ({k: e for k in self.__pending.keys()}, )
finally:
with self.__lock:
for result in results:
for (risk_key,
result_priceable), value in result.items():
if self.__use_cache:
PricingCache.put(risk_key, result_priceable,
value)
self.__pending.pop(
(risk_key, result_priceable)).set_result(value)
with self.__lock:
# Group requests optimally
for (key, priceable) in self.__pending.keys():
requests_by_provider.setdefault(key.provider, {})\
.setdefault((key.params, key.scenario, key.date, key.market), {})\
.setdefault(priceable, set())\
.add(key.risk_measure)
if requests_by_provider:
num_providers = len(requests_by_provider)
request_pool = ThreadPoolExecutor(
num_providers
) if num_providers > 1 or self.__is_async else None
for provider, by_params_scenario_date_market in requests_by_provider.items(
):
requests_for_provider = {}
for (
params, scenario, date, market
), positions_by_measures in by_params_scenario_date_market.items(
):
for priceable, risk_measures in positions_by_measures.items(
):
requests_for_provider.setdefault(
(params, scenario, date, market,
tuple(sorted(risk_measures))),
[]).append(priceable)
# TODO This will optimise for the fewest requests but we might want to just send one request per date
requests_by_date_market = {}
for (params, scenario, date, market, risk_measures
), priceables in requests_for_provider.items():
requests_by_date_market.setdefault((params, scenario, risk_measures, tuple(priceables)), set())\
.add((date, market))
requests = [
RiskRequest(tuple(
RiskPosition(instrument=p, quantity=p.get_quantity())
for p in priceables),
risk_measures,
parameters=self.__parameters,
wait_for_results=not self.__is_batch,
scenario=scenario,
pricing_and_market_data_as_of=tuple(
PricingDateAndMarketDataAsOf(
pricing_date=d, market=m)
for d, m in sorted(dates_markets)),
request_visible_to_gs=self.__visible_to_gs)
for (params, scenario, risk_measures, priceables
), dates_markets in requests_by_date_market.items()
]
if request_pool:
request_pool.submit(run_requests, requests, provider)
else:
run_requests(requests, provider)
if request_pool:
request_pool.shutdown(wait=not self.__is_async)
def _calc(self):
from gs_quant.risk import ScenarioContext
def run_request(request: RiskRequest, session: GsSession):
calc_result = {}
try:
with session:
calc_result = provider.calc(request)
except Exception as e:
for risk_measure in request.measures:
measure_results = {}
for result_position in risk_request.positions:
measure_results[result_position] = str(e)
calc_result[risk_measure] = measure_results
finally:
self._handle_results(calc_result)
def get_batch_results(request: RiskRequest, session: GsSession,
batch_provider: 'RiskApi', batch_result_id: str):
with session:
results = batch_provider.get_results(request, batch_result_id)
self._handle_results(results)
batch_results = []
pool = ThreadPoolExecutor(
len(self.__risk_measures_by_provider_and_position)
) if self.__is_async else None
while self.__risk_measures_by_provider_and_position:
provider, risk_measures_by_position = self.__risk_measures_by_provider_and_position.popitem(
)
positions_by_risk_measures = {}
for position, risk_measures in risk_measures_by_position.items():
positions_by_risk_measures.setdefault(tuple(risk_measures),
[]).append(position)
for risk_measures, positions in positions_by_risk_measures.items():
risk_request = RiskRequest(
tuple(positions),
risk_measures,
waitForResults=not self.__is_batch,
pricingLocation=self.market_data_location,
scenario=ScenarioContext.current
if ScenarioContext.current.scenario is not None else None,
pricingAndMarketDataAsOf=self._pricing_market_data_as_of)
if self.__is_batch:
batch_results.append(
(provider, risk_request, provider.calc(risk_request)))
elif pool:
pool.submit(run_request, risk_request, GsSession.current)
else:
run_request(risk_request, GsSession.current)
for provider, risk_request, result_id in batch_results:
if pool:
pool.submit(get_batch_results, risk_request, GsSession.current,
provider, result_id)
else:
get_batch_results(risk_request, GsSession.current, provider,
result_id)
if pool:
pool.shutdown(wait=not self.__is_async)
def _calc(self):
from gs_quant.api.risk import RiskApi
def run_request(request: RiskRequest, session: GsSession):
calc_result = {}
try:
with session:
calc_result = provider.calc(request)
except Exception as e:
for risk_measure in request.measures:
measure_results = {}
for result_position in risk_request.positions:
measure_results[result_position] = ErrorValue(
self.pricing_key, str(e))
calc_result[risk_measure] = measure_results
finally:
self._handle_results(calc_result)
def set_missing_results():
# Set an error string for any futures for which results were not returned
result = 'Error: no value returned'
while self.__futures:
_, positions_for_measure = self.__futures.popitem()
while positions_for_measure:
_, future = positions_for_measure.popitem()
future.set_result(result)
def get_batch_results(request: RiskRequest, session: GsSession,
batch_provider: RiskApi, batch_result_id: str):
with session:
results = batch_provider.get_results(request, batch_result_id)
self._handle_results(results)
batch_results = []
pool = ThreadPoolExecutor(
len(self.__risk_measures_by_provider_and_position)
) if self.__is_async else None
while self.__risk_measures_by_provider_and_position:
provider, risk_measures_by_position = self.__risk_measures_by_provider_and_position.popitem(
)
positions_by_risk_measures = {}
for position, risk_measures in risk_measures_by_position.items():
positions_by_risk_measures.setdefault(tuple(risk_measures),
[]).append(position)
for risk_measures, positions in positions_by_risk_measures.items():
risk_request = RiskRequest(
tuple(positions),
tuple(
sorted(risk_measures,
key=lambda m: m.name or m.measure_type.value)),
parameters=self.__parameters,
wait_for_results=not self.__is_batch,
pricing_location=self.__market_data_location,
scenario=self.__scenario,
pricing_and_market_data_as_of=self.
_pricing_market_data_as_of,
request_visible_to_gs=self.__visible_to_gs)
if self.__is_batch:
batch_results.append(
(provider, risk_request, provider.calc(risk_request)))
elif pool:
pool.submit(run_request, risk_request, GsSession.current)
else:
run_request(risk_request, GsSession.current)
for provider, risk_request, result_id in batch_results:
if pool:
pool.submit(get_batch_results, risk_request, GsSession.current,
provider, result_id)
else:
get_batch_results(risk_request, GsSession.current, provider,
result_id)
if pool:
pool.submit(set_missing_results)
pool.shutdown(wait=not self.__is_async)
else:
set_missing_results()
