def celery_evaluate_call(call_id, evaluation_kwds=None): if not evaluation_kwds: evaluation_kwds = {} quantdsl_app = get_quantdsl_app() if not 'interest_rate' in evaluation_kwds: evaluation_kwds['interest_rate'] = 10 if not 'first_market_name' in evaluation_kwds: evaluation_kwds['first_market_name'] = '#1' call_requirement = quantdsl_app.call_requirement_repo[call_id] assert isinstance(call_requirement, CallRequirement) if not 'all_market_prices' in evaluation_kwds: evaluation_kwds['all_market_prices'] = MarketSimulations(simulation_id=call_requirement.simulation_id) if not 'present_time' in evaluation_kwds: evaluation_kwds['present_time'] = call_requirement.effective_present_time if not 'present_time' in evaluation_kwds: evaluation_kwds['simulation_id'] = call_requirement.simulation_id dependency_values = get_dependency_values(call_id, quantdsl_app.call_dependencies_repo, quantdsl_app.call_result_repo) call_spec = CallSpecification( id=call_id, dsl_expr_str=call_requirement.dsl_source, effective_present_time=call_requirement.effective_present_time, evaluation_kwds=evaluation_kwds, dependency_values=dependency_values, ) evaluate_call(call_spec, result_queue=ResultsQueueAdapter())
def test_get_dependency_values(self): call_dependencies_repo = MagicMock(spec=CallDependenciesRepository, __getitem__=lambda self, x: { 1: CallDependencies(dependencies=[2, 3], entity_id=123, entity_version=0, timestamp=1), }[x]) call_result_repo = MagicMock(spec=CallResultRepository, __getitem__=lambda self, x: { 2: Mock(spec=CallResult, result_value=12), 3: Mock(spec=CallResult, result_value=13), }[x]) values = get_dependency_values(1, call_dependencies_repo, call_result_repo) self.assertEqual(values, {2: 12, 3: 13})
def evaluate_calls(self): try: while True: call_requirement_id = self.execution_queue.get() if call_requirement_id is None: break else: call_requirement = self.calls_dict[call_requirement_id] assert isinstance(call_requirement, StubbedCall) dsl_source, effective_present_time = call_requirement evaluation_kwds = self.get_evaluation_kwds( dsl_source, effective_present_time) dependency_values = get_dependency_values( call_requirement_id, self.dependencies, self.results_repo) call_spec = CallSpecification( id=call_requirement_id, dsl_expr_str=dsl_source, effective_present_time=effective_present_time, evaluation_kwds=evaluation_kwds, dependency_values=dependency_values, ) def target(): async_result = self.evaluation_pool.apply_async( evaluate_call, ( call_spec, self.result_queue, )) try: async_result.get() except Exception as error: self.result_queue.put(None) self.execution_queue.put(None) self.errors.append(error) thread = Thread(target=target) thread.daemon = True thread.start() self.call_count += 1 except: self.result_queue.put(None) raise
def test_get_dependency_values(self): call_dependencies_repo = MagicMock( spec=CallDependenciesRepository, __getitem__=lambda self, x: { 1: CallDependencies(dependencies=[2, 3], entity_id=123, entity_version=0, timestamp=1), }[x]) call_result_repo = MagicMock( spec=CallResultRepository, __getitem__=lambda self, x: { 2: Mock(spec=CallResult, result_value=12), 3: Mock(spec=CallResult, result_value=13), }[x]) values = get_dependency_values(1, call_dependencies_repo, call_result_repo) self.assertEqual(values, {2: 12, 3: 13})
def run(self, **kwargs): super(SingleThreadedDependencyGraphRunner, self).run(**kwargs) self.call_queue = six.moves.queue.Queue() self.result_queue = six.moves.queue.Queue() self.calls_dict = self.dependency_graph.call_requirements.copy() self.dependencies = self.dependency_graph.dependencies.copy() self.dependents = self.dependency_graph.dependents.copy() # Put the leaves on the execution queue. for call_requirement_id in self.dependency_graph.leaf_ids: self.call_queue.put(call_requirement_id) # Loop over the required call queue. while not self.call_queue.empty(): # Get a waiting call requirement from the queue. call_requirement_id = self.call_queue.get() # Get the call attributes. dsl_source, effective_present_time = self.calls_dict[ call_requirement_id] evaluation_kwds = self.get_evaluation_kwds(dsl_source, effective_present_time) dependency_values = get_dependency_values(call_requirement_id, self.dependencies, self.results_repo) # Evaluate the call. call_spec = CallSpecification( id=call_requirement_id, dsl_expr_str=dsl_source, effective_present_time=effective_present_time, evaluation_kwds=evaluation_kwds, dependency_values=dependency_values) evaluate_call(call_spec, self.result_queue) while not self.result_queue.empty(): call_requirement_id, result_value = self.result_queue.get() handle_result(call_requirement_id, result_value, self.results_repo, self.dependents, self.dependencies, self.call_queue) self.call_count += 1
def evaluate_calls(self): try: while True: call_requirement_id = self.execution_queue.get() if call_requirement_id is None: break else: call_requirement = self.calls_dict[call_requirement_id] assert isinstance(call_requirement, StubbedCall) dsl_source, effective_present_time = call_requirement evaluation_kwds = self.get_evaluation_kwds(dsl_source, effective_present_time) dependency_values = get_dependency_values(call_requirement_id, self.dependencies, self.results_repo) call_spec = CallSpecification( id=call_requirement_id, dsl_expr_str=dsl_source, effective_present_time=effective_present_time, evaluation_kwds=evaluation_kwds, dependency_values=dependency_values, ) def target(): async_result = self.evaluation_pool.apply_async(evaluate_call, ( call_spec, self.result_queue, )) try: async_result.get() except Exception as error: self.result_queue.put(None) self.execution_queue.put(None) self.errors.append(error) thread = Thread(target=target) thread.daemon = True thread.start() self.call_count += 1 except: self.result_queue.put(None) raise
def run(self, **kwargs): super(SingleThreadedDependencyGraphRunner, self).run(**kwargs) self.call_queue = six.moves.queue.Queue() self.result_queue = six.moves.queue.Queue() self.calls_dict = self.dependency_graph.call_requirements.copy() self.dependencies = self.dependency_graph.dependencies.copy() self.dependents = self.dependency_graph.dependents.copy() # Put the leaves on the execution queue. for call_requirement_id in self.dependency_graph.leaf_ids: self.call_queue.put(call_requirement_id) # Loop over the required call queue. while not self.call_queue.empty(): # Get a waiting call requirement from the queue. call_requirement_id = self.call_queue.get() # Get the call attributes. dsl_source, effective_present_time = self.calls_dict[call_requirement_id] evaluation_kwds = self.get_evaluation_kwds(dsl_source, effective_present_time) dependency_values = get_dependency_values(call_requirement_id, self.dependencies, self.results_repo) # Evaluate the call. call_spec = CallSpecification( id=call_requirement_id, dsl_expr_str=dsl_source, effective_present_time=effective_present_time, evaluation_kwds=evaluation_kwds, dependency_values=dependency_values ) evaluate_call(call_spec, self.result_queue) while not self.result_queue.empty(): call_requirement_id, result_value = self.result_queue.get() handle_result(call_requirement_id, result_value, self.results_repo, self.dependents, self.dependencies, self.call_queue) self.call_count += 1
def generate_contract_valuation(self, dependency_graph_id, market_simulation): assert isinstance(dependency_graph_id, six.string_types), dependency_graph_id assert isinstance(market_simulation, MarketSimulation) v = self.register_contract_valuation(dependency_graph_id) for call_id in regenerate_execution_order(dependency_graph_id, self.call_link_repo): call = self.call_requirement_repo[call_id] assert isinstance(call, CallRequirement) # Evaluate the call requirement. dependency_values = get_dependency_values(call_id, self.call_dependencies_repo, self.call_result_repo) # - parse the expr stubbed_module = dsl_parse(call.dsl_source) assert isinstance(stubbed_module, Module), "Parsed stubbed expr string is not a module: %s" % stubbed_module # - build a namespace from the dependency values dsl_locals = DslNamespace(dependency_values) # - compile the parsed expr dsl_expr = stubbed_module.body[0].reduce(dsl_locals=dsl_locals, dsl_globals=DslNamespace()) assert isinstance(dsl_expr, DslExpression), dsl_expr # - evaluate the compiled expr first_market_name = market_simulation.market_names[0] if market_simulation.market_names else None evaluation_kwds = { 'simulated_price_repo': self.simulated_price_repo, 'simulation_id': market_simulation.id, 'interest_rate': market_simulation.interest_rate, 'present_time': call.effective_present_time or market_simulation.observation_date, 'first_market_name': first_market_name, } result_value = dsl_expr.evaluate(**evaluation_kwds) # - store the result register_call_result(call_id=call_id, result_value=result_value)
def celery_evaluate_call(call_id, evaluation_kwds=None): if not evaluation_kwds: evaluation_kwds = {} quantdsl_app = get_quantdsl_app() if not 'interest_rate' in evaluation_kwds: evaluation_kwds['interest_rate'] = 10 if not 'first_market_name' in evaluation_kwds: evaluation_kwds['first_market_name'] = '#1' call_requirement = quantdsl_app.call_requirement_repo[call_id] assert isinstance(call_requirement, CallRequirement) if not 'all_market_prices' in evaluation_kwds: evaluation_kwds['all_market_prices'] = MarketSimulations( simulation_id=call_requirement.simulation_id) if not 'present_time' in evaluation_kwds: evaluation_kwds[ 'present_time'] = call_requirement.effective_present_time if not 'present_time' in evaluation_kwds: evaluation_kwds['simulation_id'] = call_requirement.simulation_id dependency_values = get_dependency_values( call_id, quantdsl_app.call_dependencies_repo, quantdsl_app.call_result_repo) call_spec = CallSpecification( id=call_id, dsl_expr_str=call_requirement.dsl_source, effective_present_time=call_requirement.effective_present_time, evaluation_kwds=evaluation_kwds, dependency_values=dependency_values, ) evaluate_call(call_spec, result_queue=ResultsQueueAdapter())