def __calc(self): def run_requests(requests_: list, provider_, create_event_loop: bool): if create_event_loop: asyncio.set_event_loop(asyncio.new_event_loop()) results = queue.Queue() done = False try: with session: provider_.run(requests_, results, 1000, progress_bar, timeout=self.__timeout) except Exception as e: provider_.enqueue(results, ((k, e) for k in self.__pending.keys())) while not done: done, chunk_results = provider_.drain_queue(results) for (risk_key_, priceable_), result in chunk_results: if (risk_key_, priceable_) in self.__pending: future = self.__pending.pop((risk_key_, priceable_)) if self.__use_cache: PricingCache.put(risk_key_, priceable_, result) future.set_result(result) while self.__pending: (risk_key_, _), future = self.__pending.popitem() future.set_result(ErrorValue(risk_key_, 'No result returned')) # Group requests optimally requests_by_provider = {} for (key, instrument) in self.__pending.keys(): dates_markets, measures = requests_by_provider.setdefault(key.provider, {})\ .setdefault((key.params, key.scenario), {})\ .setdefault(instrument, (set(), set())) dates_markets.add((key.date, key.market)) measures.add(key.risk_measure) requests_for_provider = {} if requests_by_provider: session = GsSession.current request_visible_to_gs = session.is_internal( ) if self.__visible_to_gs is None else self.__visible_to_gs for provider, by_params_scenario in requests_by_provider.items(): grouped_requests = {} for ( params, scenario ), positions_by_dates_markets_measures in by_params_scenario.items( ): for instrument, ( dates_markets, risk_measures ) in positions_by_dates_markets_measures.items(): grouped_requests.setdefault( (params, scenario, tuple(sorted(dates_markets)), tuple(sorted(risk_measures))), []).append(instrument) requests = [] # Restrict to 1,000 instruments and 1 date in a batch, until server side changes are made for (params, scenario, dates_markets, risk_measures), instruments in grouped_requests.items(): for insts_chunk in [ tuple(filter(None, i)) for i in zip_longest(*[iter(instruments)] * 1000) ]: for dates_chunk in [ tuple(filter(None, i)) for i in zip_longest(*[iter(dates_markets)] * 1) ]: requests.append( RiskRequest( tuple( RiskPosition( instrument=i, quantity=i.instrument_quantity) for i in insts_chunk), risk_measures, parameters=self._parameters, wait_for_results=not self.__is_batch, scenario=scenario, pricing_and_market_data_as_of=tuple( PricingDateAndMarketDataAsOf( pricing_date=d, market=m) for d, m in dates_chunk), request_visible_to_gs=request_visible_to_gs )) requests_for_provider[provider] = requests show_status = self.__show_progress and\ (len(requests_for_provider) > 1 or len(next(iter(requests_for_provider.values()))) > 1) request_pool = ThreadPoolExecutor(len(requests_for_provider))\ if len(requests_for_provider) > 1 or self.__is_async else None progress_bar = tqdm(total=len(self.__pending), position=0, maxinterval=1, file=sys.stdout) if show_status else None completion_futures = [] for provider, requests in requests_for_provider.items(): if request_pool: completion_future = request_pool.submit( run_requests, requests, provider, True) if not self.__is_async: completion_futures.append(completion_future) else: run_requests(requests, provider, False) # Wait on results if not async, so exceptions are surfaced if request_pool: request_pool.shutdown(False) all(f.result() for f in completion_futures)
def _calc(self): def run_request(request: RiskRequest, session: GsSession): calc_result = {} try: with session: calc_result = provider.calc(request) except Exception as e: for risk_measure in request.measures: measure_results = {} for result_position in risk_request.positions: measure_results[result_position] = str(e) calc_result[risk_measure] = measure_results finally: self._handle_results(calc_result) from gs_quant.api.risk import RiskApi def get_batch_results(request: RiskRequest, session: GsSession, batch_provider: RiskApi, batch_result_id: str): with session: results = batch_provider.get_results(request, batch_result_id) self._handle_results(results) batch_results = [] pool = ThreadPoolExecutor(len(self.__risk_measures_by_provider_and_position)) if self.__is_async else None while self.__risk_measures_by_provider_and_position: provider, risk_measures_by_position = self.__risk_measures_by_provider_and_position.popitem() positions_by_risk_measures = {} for position, risk_measures in risk_measures_by_position.items(): positions_by_risk_measures.setdefault(tuple(risk_measures), []).append(position) for risk_measures, positions in positions_by_risk_measures.items(): risk_request = RiskRequest( tuple(positions), tuple(sorted(risk_measures, key=lambda m: m.name or m.measure_type.value)), parameters=RiskRequestParameters(self.__csa_term), wait_for_results=not self.__is_batch, pricing_location=self.market_data_location, scenario=MarketDataScenario(scenario=Scenario.current) if Scenario.current_is_set else None, pricing_and_market_data_as_of=self._pricing_market_data_as_of, request_visible_to_gs=self.__visible_to_gs ) if self.__is_batch: batch_results.append((provider, risk_request, provider.calc(risk_request))) elif pool: pool.submit(run_request, risk_request, GsSession.current) else: run_request(risk_request, GsSession.current) for provider, risk_request, result_id in batch_results: if pool: pool.submit(get_batch_results, risk_request, GsSession.current, provider, result_id) else: get_batch_results(risk_request, GsSession.current, provider, result_id) if pool: pool.shutdown(wait=not self.__is_async)
def __calc(self): def run_requests(requests_: list, provider_, create_event_loop: bool): if create_event_loop: asyncio.set_event_loop(asyncio.new_event_loop()) results = {} try: with session: results = provider_.run(requests_, 1000, progress_bar, timeout=self.__timeout) except Exception as e: results = {k: e for k in self.__pending.keys()} finally: while self.__pending: (risk_key_, priceable_), future = self.__pending.popitem() result = results.get((risk_key_, priceable_)) if result is not None: if self.__use_cache: PricingCache.put(risk_key_, priceable_, result) else: result = ErrorValue(risk_key_, 'No result returned') future.set_result(result) # Group requests optimally requests_by_provider = {} for (key, priceable) in self.__pending.keys(): requests_by_provider.setdefault(key.provider, {})\ .setdefault((key.params, key.scenario, key.date, key.market), {})\ .setdefault(priceable, set())\ .add(key.risk_measure) requests_for_provider = {} if requests_by_provider: for provider, by_params_scenario_date_market in requests_by_provider.items( ): grouped_requests = {} for ( params, scenario, date, market ), positions_by_measures in by_params_scenario_date_market.items( ): for priceable, risk_measures in positions_by_measures.items( ): grouped_requests.setdefault( (params, scenario, date, market, tuple(sorted(risk_measures))), []).append(priceable) requests = [ RiskRequest(tuple( RiskPosition(instrument=p, quantity=p.get_quantity()) for p in priceables), risk_measures, parameters=self._parameters, wait_for_results=not self.__is_batch, scenario=scenario, pricing_and_market_data_as_of=( PricingDateAndMarketDataAsOf( pricing_date=date, market=market), ), request_visible_to_gs=self.__visible_to_gs) for ( params, scenario, date, market, risk_measures), priceables in grouped_requests.items() ] requests_for_provider[provider] = requests session = GsSession.current show_status = self.__show_progress and\ (len(requests_for_provider) > 1 or len(next(iter(requests_for_provider.values()))) > 1) request_pool = ThreadPoolExecutor(len(requests_for_provider))\ if len(requests_for_provider) > 1 or self.__is_async else None progress_bar = tqdm(total=len(self.__pending), position=0, maxinterval=1) if show_status else None completion_futures = [] for provider, requests in requests_for_provider.items(): if request_pool: completion_future = request_pool.submit( run_requests, requests, provider, True) if not self.__is_async: completion_futures.append(completion_future) else: run_requests(requests, provider, False) # Wait on results if not async, so exceptions are surfaced if request_pool: request_pool.shutdown(False) all(f.result() for f in completion_futures)
def __calc(self): session = GsSession.current requests_by_provider = {} def run_requests(requests_: Iterable[RiskRequest], provider_): results = {} try: with session: results = provider_.calc_multi(requests_) if self.__is_batch: results = provider_.get_results( dict(zip(results, requests_)), timeout=self.__batch_results_timeout).values() except Exception as e: results = ({k: e for k in self.__pending.keys()}, ) finally: with self.__lock: for result in results: for (risk_key, result_priceable), value in result.items(): if self.__use_cache: PricingCache.put(risk_key, result_priceable, value) self.__pending.pop( (risk_key, result_priceable)).set_result(value) with self.__lock: # Group requests optimally for (key, priceable) in self.__pending.keys(): requests_by_provider.setdefault(key.provider, {})\ .setdefault((key.params, key.scenario, key.date, key.market), {})\ .setdefault(priceable, set())\ .add(key.risk_measure) if requests_by_provider: num_providers = len(requests_by_provider) request_pool = ThreadPoolExecutor( num_providers ) if num_providers > 1 or self.__is_async else None for provider, by_params_scenario_date_market in requests_by_provider.items( ): requests_for_provider = {} for ( params, scenario, date, market ), positions_by_measures in by_params_scenario_date_market.items( ): for priceable, risk_measures in positions_by_measures.items( ): requests_for_provider.setdefault( (params, scenario, date, market, tuple(sorted(risk_measures))), []).append(priceable) # TODO This will optimise for the fewest requests but we might want to just send one request per date requests_by_date_market = {} for (params, scenario, date, market, risk_measures ), priceables in requests_for_provider.items(): requests_by_date_market.setdefault((params, scenario, risk_measures, tuple(priceables)), set())\ .add((date, market)) requests = [ RiskRequest(tuple( RiskPosition(instrument=p, quantity=p.get_quantity()) for p in priceables), risk_measures, parameters=self.__parameters, wait_for_results=not self.__is_batch, scenario=scenario, pricing_and_market_data_as_of=tuple( PricingDateAndMarketDataAsOf( pricing_date=d, market=m) for d, m in sorted(dates_markets)), request_visible_to_gs=self.__visible_to_gs) for (params, scenario, risk_measures, priceables ), dates_markets in requests_by_date_market.items() ] if request_pool: request_pool.submit(run_requests, requests, provider) else: run_requests(requests, provider) if request_pool: request_pool.shutdown(wait=not self.__is_async)
def _calc(self): from gs_quant.risk import ScenarioContext def run_request(request: RiskRequest, session: GsSession): calc_result = {} try: with session: calc_result = provider.calc(request) except Exception as e: for risk_measure in request.measures: measure_results = {} for result_position in risk_request.positions: measure_results[result_position] = str(e) calc_result[risk_measure] = measure_results finally: self._handle_results(calc_result) def get_batch_results(request: RiskRequest, session: GsSession, batch_provider: 'RiskApi', batch_result_id: str): with session: results = batch_provider.get_results(request, batch_result_id) self._handle_results(results) batch_results = [] pool = ThreadPoolExecutor( len(self.__risk_measures_by_provider_and_position) ) if self.__is_async else None while self.__risk_measures_by_provider_and_position: provider, risk_measures_by_position = self.__risk_measures_by_provider_and_position.popitem( ) positions_by_risk_measures = {} for position, risk_measures in risk_measures_by_position.items(): positions_by_risk_measures.setdefault(tuple(risk_measures), []).append(position) for risk_measures, positions in positions_by_risk_measures.items(): risk_request = RiskRequest( tuple(positions), risk_measures, waitForResults=not self.__is_batch, pricingLocation=self.market_data_location, scenario=ScenarioContext.current if ScenarioContext.current.scenario is not None else None, pricingAndMarketDataAsOf=self._pricing_market_data_as_of) if self.__is_batch: batch_results.append( (provider, risk_request, provider.calc(risk_request))) elif pool: pool.submit(run_request, risk_request, GsSession.current) else: run_request(risk_request, GsSession.current) for provider, risk_request, result_id in batch_results: if pool: pool.submit(get_batch_results, risk_request, GsSession.current, provider, result_id) else: get_batch_results(risk_request, GsSession.current, provider, result_id) if pool: pool.shutdown(wait=not self.__is_async)
def _calc(self): from gs_quant.api.risk import RiskApi def run_request(request: RiskRequest, session: GsSession): calc_result = {} try: with session: calc_result = provider.calc(request) except Exception as e: for risk_measure in request.measures: measure_results = {} for result_position in risk_request.positions: measure_results[result_position] = ErrorValue( self.pricing_key, str(e)) calc_result[risk_measure] = measure_results finally: self._handle_results(calc_result) def set_missing_results(): # Set an error string for any futures for which results were not returned result = 'Error: no value returned' while self.__futures: _, positions_for_measure = self.__futures.popitem() while positions_for_measure: _, future = positions_for_measure.popitem() future.set_result(result) def get_batch_results(request: RiskRequest, session: GsSession, batch_provider: RiskApi, batch_result_id: str): with session: results = batch_provider.get_results(request, batch_result_id) self._handle_results(results) batch_results = [] pool = ThreadPoolExecutor( len(self.__risk_measures_by_provider_and_position) ) if self.__is_async else None while self.__risk_measures_by_provider_and_position: provider, risk_measures_by_position = self.__risk_measures_by_provider_and_position.popitem( ) positions_by_risk_measures = {} for position, risk_measures in risk_measures_by_position.items(): positions_by_risk_measures.setdefault(tuple(risk_measures), []).append(position) for risk_measures, positions in positions_by_risk_measures.items(): risk_request = RiskRequest( tuple(positions), tuple( sorted(risk_measures, key=lambda m: m.name or m.measure_type.value)), parameters=self.__parameters, wait_for_results=not self.__is_batch, pricing_location=self.__market_data_location, scenario=self.__scenario, pricing_and_market_data_as_of=self. _pricing_market_data_as_of, request_visible_to_gs=self.__visible_to_gs) if self.__is_batch: batch_results.append( (provider, risk_request, provider.calc(risk_request))) elif pool: pool.submit(run_request, risk_request, GsSession.current) else: run_request(risk_request, GsSession.current) for provider, risk_request, result_id in batch_results: if pool: pool.submit(get_batch_results, risk_request, GsSession.current, provider, result_id) else: get_batch_results(risk_request, GsSession.current, provider, result_id) if pool: pool.submit(set_missing_results) pool.shutdown(wait=not self.__is_async) else: set_missing_results()