def test_module_block(self):
# Expression with one function def.
dsl_source = """
def sqr(n):
n ** 2
sqr(3)
"""
dsl_module = dsl_parse(dsl_source)
self.assertIsInstance(dsl_module, Module)
self.assertEqual(str(dsl_module), dsl_source.strip())
dsl_expr = dsl_compile(dsl_source)
self.assertEqual(dsl_expr.evaluate(), 9)
dsl_value = dsl_eval(dsl_source)
self.assertEqual(dsl_value, 9)
# Expression with two function defs.
dsl_source = """
def add(a, b):
a + b
def mul(a, b):
a if b == 1 else add(a, mul(a, b - 1))
mul(3, 3)
"""
dsl_module = dsl_parse(dsl_source)
self.assertIsInstance(dsl_module, Module)
self.assertEqual(str(dsl_module), dsl_source.strip())
dsl_expr = dsl_compile(dsl_source)
# self.assertEqual(str(dsl_expr), "")
self.assertEqual(dsl_expr.evaluate(), 9)
dsl_value = dsl_eval(dsl_source)
self.assertEqual(dsl_value, 9)
def test_module_block(self):
# Expression with one function def.
dsl_source = """
def sqr(n):
n ** 2
sqr(3)
"""
dsl_module = dsl_parse(dsl_source)
self.assertIsInstance(dsl_module, Module)
self.assertEqual(str(dsl_module), dsl_source.strip())
dsl_expr = dsl_compile(dsl_source)
self.assertEqual(dsl_expr.evaluate(), 9)
dsl_value = dsl_eval(dsl_source)
self.assertEqual(dsl_value, 9)
# Expression with two function defs.
dsl_source = """
def add(a, b):
a + b
def mul(a, b):
a if b == 1 else add(a, mul(a, b - 1))
mul(3, 3)
"""
dsl_module = dsl_parse(dsl_source)
self.assertIsInstance(dsl_module, Module)
self.assertEqual(str(dsl_module), dsl_source.strip())
dsl_expr = dsl_compile(dsl_source)
# self.assertEqual(str(dsl_expr), "")
self.assertEqual(dsl_expr.evaluate(), 9)
dsl_value = dsl_eval(dsl_source)
self.assertEqual(dsl_value, 9)
def dsl_compile(dsl_source, filename='<unknown>', is_parallel=None, dsl_classes=None, compile_kwds=None, **extraCompileKwds):
"""
Returns a DSL expression, created according to the given DSL source module.
That is, if the source module contains a function def and an expression which
calls that function, then the expression's function call will be evaluated
and the resulting DSL expression will be substituted for the function call
in the module's expression, so that calls to user defined functions are eliminated
and a single DSL expression is obtained.
If the source module contains a function def, but no expression, the module is compiled
into a function def object. Calling .apply() on a function def object will return a DSL
expression object, which can be evaluated by calling its .evaluate() method.
"""
if compile_kwds is None:
compile_kwds = DslNamespace()
assert isinstance(compile_kwds, dict)
compile_kwds.update(extraCompileKwds)
# Parse the source into a DSL module object.
dsl_module = dsl_parse(dsl_source, filename=filename, dsl_classes=dsl_classes)
assert isinstance(dsl_module, Module)
# Compile the module into either a dependency graph
# if 'is_parallel' is True, otherwise a single primitive expression.
return compile_dsl_module(dsl_module, DslNamespace(), compile_kwds, is_dependency_graph=is_parallel)
def identify_simulation_requirements(dependency_graph_id,
call_requirement_repo, call_link_repo,
call_dependencies_repo,
market_dependencies_repo,
observation_date, requirements):
assert isinstance(call_requirement_repo, CallRequirementRepository)
assert isinstance(call_link_repo, CallLinkRepository)
all_perturbation_dependencies = {}
for call_id in regenerate_execution_order(dependency_graph_id,
call_link_repo):
# Get the stubbed expression.
call_requirement = call_requirement_repo[call_id]
assert isinstance(call_requirement, CallRequirement)
if call_requirement._dsl_expr is not None:
dsl_expr = call_requirement._dsl_expr
else:
dsl_module = dsl_parse(call_requirement.dsl_source)
dsl_expr = dsl_module.body[0]
call_requirement._dsl_expr = dsl_expr
assert isinstance(dsl_expr, DslObject), dsl_expr
# Todo: Consolidate 'date' attributes to be a single element (rather than a possibly long sum expression).
# Identify this call's requirements for simulated prices.
simulation_requirements = set()
present_time = call_requirement.effective_present_time or observation_date
dsl_expr.identify_price_simulation_requirements(
simulation_requirements, present_time=present_time)
# Register the simulation requirements for each call (needed during evaluation).
register_simulated_price_requirements(call_id,
list(simulation_requirements))
# Update the simulation requirements (needed for the market simulation).
requirements.update(simulation_requirements)
# Identify this call's perturbation dependencies.
perturbation_dependencies = set()
dsl_expr.identify_perturbation_dependencies(perturbation_dependencies,
present_time=present_time)
# Add the expression's perturbation dependencies to the perturbation dependencies of its call dependencies.
call_dependencies = call_dependencies_repo[call_id]
assert isinstance(call_dependencies,
CallDependencies), call_dependencies
for dependency_id in call_dependencies.dependencies:
dependency_perturbation_dependencies = all_perturbation_dependencies[
dependency_id]
perturbation_dependencies.update(
dependency_perturbation_dependencies)
# Register the perturbation dependencies in the repo (needed when evaluating the call).
register_perturbation_dependencies(call_id,
list(perturbation_dependencies))
# Save the pertubation dependencies for this call, so they are available for the dependent calls.
all_perturbation_dependencies[call_id] = perturbation_dependencies
def dsl_compile(dsl_source,
filename='<unknown>',
dsl_classes=None,
compile_kwds=None,
**extraCompileKwds):
"""
Returns a DSL expression, created according to the given DSL source module.
That is, if the source module contains a function def and an expression which
calls that function, then the expression's function call will be evaluated
and the resulting DSL expression will be substituted for the function call
in the module's expression, so that calls to user defined functions are eliminated
and a single DSL expression is obtained.
If the source module contains a function def, but no expression, the module is compiled
into a function def object. Calling .apply() on a function def object will return a DSL
expression object, which can be evaluated by calling its .evaluate() method.
"""
if compile_kwds is None:
compile_kwds = DslNamespace()
# assert isinstance(compile_kwds, dict)
compile_kwds.update(extraCompileKwds)
# Parse the source into a DSL module object.
dsl_module = dsl_parse(dsl_source,
filename=filename,
dsl_classes=dsl_classes)
# assert isinstance(dsl_module, Module)
# Compile the module into either a dependency graph
# if 'is_parallel' is True, otherwise a single primitive expression.
return compile_dsl_module(dsl_module, DslNamespace(), compile_kwds)
def get_evaluation_kwds(self, dsl_source, effective_present_time):
evaluation_kwds = self.run_kwds.copy()
from quantdsl.services import list_fixing_dates
from quantdsl.domain.services.parser import dsl_parse
stubbed_module = dsl_parse(dsl_source)
assert isinstance(stubbed_module, Module)
# market_names = find_market_names(stubbed_module)
fixing_dates = list_fixing_dates(stubbed_module)
if effective_present_time is not None:
fixing_dates.append(effective_present_time)
# return evaluation_kwds
# Rebuild the data structure (there was a problem, but I can't remember what it was.
# Todo: Try without this block, perhaps the problem doesn't exist anymore.
if 'all_market_prices' in evaluation_kwds:
all_market_prices = evaluation_kwds.pop('all_market_prices')
evaluation_kwds['all_market_prices'] = dict()
for market_name in all_market_prices.keys():
# if market_name not in market_names:
# continue
market_prices = dict()
for date in fixing_dates:
market_prices[date] = all_market_prices[market_name][date]
evaluation_kwds['all_market_prices'][market_name] = market_prices
return evaluation_kwds
def evaluate_call(call_spec, register_call_result):
"""
Evaluates the stubbed expr identified by 'call_requirement_id'.
"""
assert isinstance(call_spec, CallSpecification)
evaluation_kwds = call_spec.evaluation_kwds.copy()
# If this call has an effective present time value, use it as the 'present_time' in the evaluation_kwds.
# This results from e.g. the Wait DSL element. Calls near the root of the expression might not have an
# effective present time value, and the present time will be the observation time of the evaluation.
# Evaluate the stubbed expr str.
# - parse the expr
try:
# Todo: Rework this dependency. Figure out how to use alternative set of DSL classes when multiprocessing.
from quantdsl.domain.services.parser import dsl_parse
stubbed_module = dsl_parse(call_spec.dsl_expr_str)
except DslSyntaxError:
raise
assert isinstance(stubbed_module, Module), "Parsed stubbed expr string is not a module: %s" % stubbed_module
# - build a namespace from the dependency values
dsl_locals = DslNamespace(call_spec.dependency_values)
# - compile the parsed expr
dsl_expr = stubbed_module.body[0].reduce(dsl_locals=dsl_locals, dsl_globals=DslNamespace())
assert isinstance(dsl_expr, DslExpression), dsl_expr
# - evaluate the compiled expr
result_value = dsl_expr.evaluate(**evaluation_kwds)
# - store the result
register_call_result(call_id=call_spec.id, result_value=result_value)
def generate_dependency_graph(contract_specification, call_dependencies_repo, call_dependents_repo, call_leafs_repo,
call_requirement_repo):
assert isinstance(contract_specification, ContractSpecification)
dsl_module = dsl_parse(dsl_source=contract_specification.specification)
assert isinstance(dsl_module, Module)
dsl_globals = DslNamespace()
function_defs, expressions = extract_defs_and_exprs(dsl_module, dsl_globals)
dsl_expr = expressions[0]
assert isinstance(dsl_expr, DslExpression)
dsl_locals = DslNamespace()
leaf_ids = []
all_dependents = defaultdict(list)
# Generate stubbed call from the parsed DSL module object.
for stub in generate_stubbed_calls(contract_specification.id, dsl_module, dsl_expr, dsl_globals, dsl_locals):
# assert isinstance(stub, StubbedCall)
# Register the call requirements.
call_id = stub.call_id
dsl_source = str(stub.dsl_expr)
effective_present_time = stub.effective_present_time
call_requirement = register_call_requirement(call_id, dsl_source, effective_present_time)
# Hold onto the dsl_expr, helps in "single process" modes....
call_requirement._dsl_expr = stub.dsl_expr
# - put the entity directly in the cache, otherwise the entity will be regenerated when it is next accessed
# and the _dsl_expr will be lost.
call_requirement_repo.add_cache(call_id, call_requirement)
# Register the call requirements.
dependencies = stub.requirements
register_call_dependencies(call_id, dependencies)
# Keep track of the leaves and the dependents.
if len(dependencies) == 0:
leaf_ids.append(call_id)
else:
for dependency_call_id in dependencies:
all_dependents[dependency_call_id].append(call_id)
# Register the call dependents.
for call_id, dependents in all_dependents.items():
register_call_dependents(call_id, dependents)
register_call_dependents(contract_specification.id, [])
# Generate and register the call order.
link_id = contract_specification.id
for call_id in generate_execution_order(leaf_ids, call_dependents_repo, call_dependencies_repo):
register_call_link(link_id, call_id)
link_id = call_id
# Register the leaf ids.
register_call_leafs(contract_specification.id, leaf_ids)
def generate_dependency_graph(contract_specification, call_dependencies_repo,
call_dependents_repo):
assert isinstance(contract_specification, ContractSpecification)
dsl_module = dsl_parse(dsl_source=contract_specification.specification)
assert isinstance(dsl_module, Module)
dsl_globals = DslNamespace()
function_defs, expressions = extract_defs_and_exprs(
dsl_module, dsl_globals)
dsl_expr = expressions[0]
assert isinstance(dsl_expr, DslExpression)
dsl_locals = DslNamespace()
leaf_call_ids = []
all_dependents = defaultdict(list)
# Generate stubbed call from the parsed DSL module object.
for stub in generate_stubbed_calls(contract_specification.id, dsl_module,
dsl_expr, dsl_globals, dsl_locals):
assert isinstance(stub, StubbedCall)
call_id = stub.call_id
dsl_source = stub.dsl_source
effective_present_time = stub.effective_present_time
dependencies = stub.dependencies
# Register the call requirements.
register_call_requirement(call_id, dsl_source, effective_present_time)
# Register the call dependencies.
register_call_dependencies(call_id, dependencies)
# Keep track of the leaves and the dependents.
if len(dependencies) == 0:
leaf_call_ids.append(call_id)
else:
for dependency_call_id in dependencies:
all_dependents[dependency_call_id].append(call_id)
# Register the call dependents.
for call_id, dependents in all_dependents.items():
register_call_dependents(call_id, dependents)
register_call_dependents(contract_specification.id, [])
# Generate and register the call order.
link_id = contract_specification.id
for call_id in generate_execution_order(leaf_call_ids,
call_dependents_repo,
call_dependencies_repo):
register_call_link(link_id, call_id)
link_id = call_id
def list_fixing_dates(dependency_graph_id, call_requirement_repo, call_link_repo):
assert isinstance(call_requirement_repo, CallRequirementRepository)
assert isinstance(call_link_repo, CallLinkRepository)
fixing_dates = set()
for call_id in regenerate_execution_order(dependency_graph_id, call_link_repo):
# Get the stubbed expression.
call_requirement = call_requirement_repo[call_id]
assert isinstance(call_requirement, CallRequirement)
dsl_expr = dsl_parse(call_requirement.dsl_source)
# Find all the fixing times involved in this expression.
for fixing_date in find_fixing_dates(dsl_expr=dsl_expr):
fixing_dates.add(fixing_date)
# Return a sorted list of fixing times.
return sorted(list(fixing_dates))
def assertDslExprTypeValue(self, dsl_source, expectedDslType, expectedDslValue, **compile_kwds):
# Assumes dsl_source is just one statement.
dsl_module = dsl_parse(dsl_source)
assert isinstance(dsl_module, Module)
# Check the parsed DSL can be rendered as a string that is equal to the original source.
self.assertEqual(str(dsl_module).strip(), dsl_source.strip())
# Check the statement's expression type.
dsl_expr = dsl_module.body[0]
self.assertIsInstance(dsl_expr, expectedDslType)
# Compile the module into an simple DSL expression object (no variables or calls to function defs).
dsl_expr = compile_dsl_module(dsl_module, compile_kwds)
# Evaluate the compiled expression.
self.assertEqual(dsl_expr.evaluate(), expectedDslValue)
def assertDslExprTypeValue(self, dsl_source, expectedDslType, expectedDslValue, **compile_kwds):
# Assumes dsl_source is just one statement.
dsl_module = dsl_parse(dsl_source)
assert isinstance(dsl_module, Module)
# Check the parsed DSL can be rendered as a string that is equal to the original source.
self.assertEqual(str(dsl_module).strip(), dsl_source.strip())
# Check the statement's expression type.
dsl_expr = dsl_module.body[0]
self.assertIsInstance(dsl_expr, expectedDslType)
# Compile the module into an simple DSL expression object (no variables or calls to function defs).
dsl_expr = compile_dsl_module(dsl_module, compile_kwds)
# Evaluate the compiled expression.
self.assertEqual(dsl_expr.evaluate(), expectedDslValue)
def generate_dependency_graph(contract_specification, call_dependencies_repo, call_dependents_repo):
assert isinstance(contract_specification, ContractSpecification)
dsl_module = dsl_parse(dsl_source=contract_specification.specification)
assert isinstance(dsl_module, Module)
dsl_globals = DslNamespace()
function_defs, expressions = extract_defs_and_exprs(dsl_module, dsl_globals)
dsl_expr = expressions[0]
assert isinstance(dsl_expr, DslExpression)
dsl_locals = DslNamespace()
leaf_call_ids = []
all_dependents = defaultdict(list)
# Generate stubbed call from the parsed DSL module object.
for stub in generate_stubbed_calls(contract_specification.id, dsl_module, dsl_expr, dsl_globals, dsl_locals):
assert isinstance(stub, StubbedCall)
call_id = stub.call_id
dsl_source = stub.dsl_source
effective_present_time = stub.effective_present_time
dependencies = stub.dependencies
# Register the call requirements.
register_call_requirement(call_id, dsl_source, effective_present_time)
# Register the call dependencies.
register_call_dependencies(call_id, dependencies)
# Keep track of the leaves and the dependents.
if len(dependencies) == 0:
leaf_call_ids.append(call_id)
else:
for dependency_call_id in dependencies:
all_dependents[dependency_call_id].append(call_id)
# Register the call dependents.
for call_id, dependents in all_dependents.items():
register_call_dependents(call_id, dependents)
register_call_dependents(contract_specification.id, [])
# Generate and register the call order.
link_id = contract_specification.id
for call_id in generate_execution_order(leaf_call_ids, call_dependents_repo, call_dependencies_repo):
register_call_link(link_id, call_id)
link_id = call_id
def generate_contract_valuation(self, dependency_graph_id, market_simulation):
assert isinstance(dependency_graph_id, six.string_types), dependency_graph_id
assert isinstance(market_simulation, MarketSimulation)
v = self.register_contract_valuation(dependency_graph_id)
for call_id in regenerate_execution_order(dependency_graph_id, self.call_link_repo):
call = self.call_requirement_repo[call_id]
assert isinstance(call, CallRequirement)
# Evaluate the call requirement.
dependency_values = get_dependency_values(call_id, self.call_dependencies_repo, self.call_result_repo)
# - parse the expr
stubbed_module = dsl_parse(call.dsl_source)
assert isinstance(stubbed_module, Module), "Parsed stubbed expr string is not a module: %s" % stubbed_module
# - build a namespace from the dependency values
dsl_locals = DslNamespace(dependency_values)
# - compile the parsed expr
dsl_expr = stubbed_module.body[0].reduce(dsl_locals=dsl_locals, dsl_globals=DslNamespace())
assert isinstance(dsl_expr, DslExpression), dsl_expr
# - evaluate the compiled expr
first_market_name = market_simulation.market_names[0] if market_simulation.market_names else None
evaluation_kwds = {
'simulated_price_repo': self.simulated_price_repo,
'simulation_id': market_simulation.id,
'interest_rate': market_simulation.interest_rate,
'present_time': call.effective_present_time or market_simulation.observation_date,
'first_market_name': first_market_name,
}
result_value = dsl_expr.evaluate(**evaluation_kwds)
# - store the result
register_call_result(call_id=call_id, result_value=result_value)
def assertDslExprTypeValue(self, dsl_source, expectedDslType, expectedDslValue, **compile_kwds):
# Assumes dsl_source is just one statement.
dsl_module = dsl_parse(dsl_source)
assert isinstance(dsl_module, Module)
# Check the parsed DSL can be rendered as a string that is equal to the original source.
self.assertEqual(str(dsl_module), dsl_source.strip())
# Assume this test is dealing with modules that have one statement only.
dsl_expr = dsl_module.body[0]
# Check expression type.
assert isinstance(dsl_expr, DslExpression)
self.assertIsInstance(dsl_expr, expectedDslType)
# Compile the module into a simple DSL expression object (no variables or calls to function defs).
dsl_expr = dsl_expr.substitute_names(DslNamespace(compile_kwds))
# Evaluate the compiled expression.
dsl_value = dsl_expr.evaluate()
self.assertEqual(dsl_value, expectedDslValue)
def assertDslExprTypeValue(self, dsl_source, expectedDslType,
expectedDslValue, **compile_kwds):
# Assumes dsl_source is just one statement.
dsl_module = dsl_parse(dsl_source)
assert isinstance(dsl_module, Module)
# Check the parsed DSL can be rendered as a string that is equal to the original source.
self.assertEqual(str(dsl_module), dsl_source.strip())
# Assume this test is dealing with modules that have one statement only.
dsl_expr = dsl_module.body[0]
# Check expression type.
assert isinstance(dsl_expr, DslExpression)
self.assertIsInstance(dsl_expr, expectedDslType)
# Compile the module into a simple DSL expression object (no variables or calls to function defs).
dsl_expr = dsl_expr.substitute_names(DslNamespace(compile_kwds))
# Evaluate the compiled expression.
dsl_value = dsl_expr.evaluate()
self.assertEqual(dsl_value, expectedDslValue)
def compute_call_result(contract_valuation,
call_requirement,
market_simulation,
perturbation_dependencies,
call_dependencies_repo,
call_result_repo,
simulated_price_repo,
perturbation_dependencies_repo,
simulation_requirements,
compute_pool=None):
"""
Parses, compiles and evaluates a call requirement.
"""
# assert isinstance(contract_valuation, ContractValuation), contract_valuation
assert isinstance(call_requirement, CallRequirement), call_requirement
# assert isinstance(market_simulation, MarketSimulation), market_simulation
# assert isinstance(call_dependencies_repo, CallDependenciesRepository), call_dependencies_repo
# assert isinstance(call_result_repo, CallResultRepository)
# assert isinstance(simulated_price_dict, SimulatedPriceRepository)
# Todo: Put getting the dependency values in a separate thread, and perhaps make each call a separate thread.
# Parse the DSL source into a DSL module object (composite tree of objects that provide the DSL semantics).
if call_requirement._dsl_expr is not None:
dsl_expr = call_requirement._dsl_expr
else:
dsl_module = dsl_parse(call_requirement.dsl_source)
dsl_expr = dsl_module.body[0]
# assert isinstance(dsl_module, Module), "Parsed stubbed expr string is not a module: %s" % dsl_module
present_time = call_requirement.effective_present_time or market_simulation.observation_date
simulated_value_dict = {}
# market_simulation.dependencies
#
# all_fixing_dates = set([present_time] + list_fixing_dates(dsl_expr))
# market_dependencies = perturbation_dependencies_repo[call_requirement.id]
# assert isinstance(market_dependencies, PerturbationDependencies)
# all_delivery_points = market_dependencies.dependencies
# for fixing_date in all_fixing_dates:
# for delivery_point in all_delivery_points:
# market_name = delivery_point[0]
# delivery_date = delivery_point[1]
# simulation_id = contract_valuation.market_simulation_id
# price_id = make_simulated_price_id(simulation_id, market_name, fixing_date, delivery_date)
# simulated_price = simulated_price_dict[price_id]
# # assert isinstance(simulated_price, SimulatedPrice)
# # assert isinstance(simulated_price.value, scipy.ndarray)
# simulated_value_dict[price_id] = simulated_price.value
simulation_id = contract_valuation.market_simulation_id
first_commodity_name = None
for simulation_requirement in simulation_requirements:
(commodity_name, fixing_date, delivery_date) = simulation_requirement
if first_commodity_name is None:
first_commodity_name = commodity_name
price_id = make_simulated_price_id(simulation_id, commodity_name,
fixing_date, delivery_date)
simulated_price = simulated_price_repo[price_id]
simulated_value_dict[price_id] = simulated_price.value
# Get all the call results depended on by this call.
dependency_results = get_dependency_results(
contract_valuation_id=contract_valuation.id,
call_id=call_requirement.id,
dependencies_repo=call_dependencies_repo,
result_repo=call_result_repo,
)
# Compute the call result.
if compute_pool is None:
result_value, perturbed_values = evaluate_dsl_expr(
dsl_expr, first_commodity_name, market_simulation.id,
market_simulation.interest_rate, present_time,
simulated_value_dict, perturbation_dependencies.dependencies,
dependency_results, market_simulation.path_count)
else:
assert isinstance(compute_pool, Pool)
async_result = compute_pool.apply_async(
evaluate_dsl_expr,
args=(dsl_expr, first_commodity_name, market_simulation.id,
market_simulation.interest_rate, present_time,
simulated_value_dict, perturbation_dependencies.dependencies,
dependency_results, market_simulation.path_count),
)
gevent.sleep(0.0001)
result_value, perturbed_values = async_result.get()
# Return the result.
return result_value, perturbed_values
def generate_dependency_graph(contract_specification,
call_dependencies_repo,
call_dependents_repo,
call_requirement_repo,
dsl_classes=None):
assert isinstance(contract_specification, ContractSpecification)
dsl_module = dsl_parse(
dsl_source=contract_specification.source_code,
dsl_classes=dsl_classes,
)
assert isinstance(dsl_module, Module)
dsl_globals = dsl_module.namespace.copy()
function_defs, expressions = extract_defs_and_exprs(
dsl_module, dsl_globals)
dsl_expr = expressions[0]
assert isinstance(dsl_expr, DslExpression)
dsl_locals = DslNamespace()
leaf_ids = []
all_dependents = defaultdict(list)
# Generate stubbed call from the parsed DSL module object.
for stubed_call in generate_stubbed_calls(
contract_specification.id, dsl_expr, dsl_globals, dsl_locals,
contract_specification.observation_date):
# assert isinstance(stub, StubbedCall)
# Estimate the cost of evaluating this expression.
estimated_cost = stubed_call.dsl_expr.cost_expression()
# Register the call requirements.
call_id = stubed_call.call_id
dsl_source = str(stubed_call.dsl_expr)
present_time = stubed_call.present_time
call_requirement = register_call_requirement(
call_id=call_id,
dsl_source=dsl_source,
present_time=present_time,
contract_specification_id=contract_specification.id,
cost=estimated_cost,
)
# Hold onto the dsl_expr, helps in "single process" modes.
# - put the entity directly in the cache, otherwise the entity will be
# regenerated when it is next accessed and the _dsl_expr will be "lost"
call_requirement._dsl_expr = stubed_call.dsl_expr
call_requirement_repo.add_cache(call_id, call_requirement)
# Register the call dependencies (things needed by this call).
dependencies = stubed_call.requirements
register_call_dependencies(call_id, dependencies)
# Keep track of the leaves and the dependents.
if len(dependencies) == 0:
leaf_ids.append(call_id)
else:
for dependency_call_id in dependencies:
all_dependents[dependency_call_id].append(call_id)
# Register the call dependents.
for call_id, dependents in all_dependents.items():
register_call_dependents(call_id, dependents)
register_call_dependents(contract_specification.id, [])
# Generate and register the call order.
link_id = contract_specification.id
for call_id in generate_execution_order(leaf_ids, call_dependents_repo,
call_dependencies_repo):
register_call_link(link_id, call_id)
link_id = call_id
# Register the leaf ids.
register_call_leafs(contract_specification.id, leaf_ids)
def test_empty_string(self):
# Empty string can be parsed as a module, but not compiled into an expression, or evaluated.
self.assertTrue(isinstance(dsl_parse(""), Module))
self.assertRaises(DslSyntaxError, dsl_compile, "")
self.assertRaises(DslSyntaxError, dsl_eval, "")
def generate_dependency_graph(contract_specification, call_dependencies_repo, call_dependents_repo,
call_requirement_repo, dsl_classes=None):
assert isinstance(contract_specification, ContractSpecification)
dsl_module = dsl_parse(
dsl_source=contract_specification.source_code,
dsl_classes=dsl_classes,
)
assert isinstance(dsl_module, Module)
dsl_globals = dsl_module.namespace.copy()
function_defs, expressions = extract_defs_and_exprs(dsl_module, dsl_globals)
dsl_expr = expressions[0]
assert isinstance(dsl_expr, DslExpression)
dsl_locals = DslNamespace()
leaf_ids = []
all_dependents = defaultdict(list)
# Generate stubbed call from the parsed DSL module object.
for stubed_call in generate_stubbed_calls(contract_specification.id, dsl_expr, dsl_globals, dsl_locals,
contract_specification.observation_date):
# assert isinstance(stub, StubbedCall)
# Estimate the cost of evaluating this expression.
estimated_cost = stubed_call.dsl_expr.cost_expression()
# Register the call requirements.
call_id = stubed_call.call_id
dsl_source = str(stubed_call.dsl_expr)
present_time = stubed_call.present_time
call_requirement = register_call_requirement(
call_id=call_id,
dsl_source=dsl_source,
present_time=present_time,
contract_specification_id=contract_specification.id,
cost=estimated_cost,
)
# Hold onto the dsl_expr, helps in "single process" modes.
# - put the entity directly in the cache, otherwise the entity will be
# regenerated when it is next accessed and the _dsl_expr will be "lost"
call_requirement._dsl_expr = stubed_call.dsl_expr
call_requirement_repo.add_cache(call_id, call_requirement)
# Register the call dependencies (things needed by this call).
dependencies = stubed_call.requirements
register_call_dependencies(call_id, dependencies)
# Keep track of the leaves and the dependents.
if len(dependencies) == 0:
leaf_ids.append(call_id)
else:
for dependency_call_id in dependencies:
all_dependents[dependency_call_id].append(call_id)
# Register the call dependents.
for call_id, dependents in all_dependents.items():
register_call_dependents(call_id, dependents)
register_call_dependents(contract_specification.id, [])
# Generate and register the call order.
link_id = contract_specification.id
for call_id in generate_execution_order(leaf_ids, call_dependents_repo, call_dependencies_repo):
register_call_link(link_id, call_id)
link_id = call_id
# Register the leaf ids.
register_call_leafs(contract_specification.id, leaf_ids)
def test_empty_string(self):
self.assertTrue(isinstance(dsl_parse(""), Module))
self.assertRaises(DslSyntaxError, dsl_compile, "")
self.assertRaises(DslSyntaxError, dsl_eval, "")
def identify_simulation_requirements(contract_specification_id, call_requirement_repo, call_link_repo,
call_dependencies_repo, observation_date, requirements, periodisation=None):
assert isinstance(call_requirement_repo, CallRequirementRepository)
assert isinstance(call_link_repo, CallLinkRepository)
all_perturbation_dependencies = {}
call_requirements = []
for call_id in regenerate_execution_order(contract_specification_id, call_link_repo):
# Get the stubbed expression.
call_requirement = call_requirement_repo[call_id]
assert isinstance(call_requirement, CallRequirement)
if call_requirement._dsl_expr is not None:
dsl_expr = call_requirement._dsl_expr
else:
dsl_module = dsl_parse(call_requirement.dsl_source)
dsl_expr = dsl_module.body[0]
call_requirement._dsl_expr = dsl_expr
assert isinstance(dsl_expr, DslObject), dsl_expr
call_requirements.append(call_requirement)
all_market_names = set()
for call_requirement in call_requirements:
dsl_expr = call_requirement._dsl_expr
for market in dsl_expr.find_instances(AbstractMarket):
assert isinstance(market.commodity_name, six.string_types)
all_market_names.add(market.commodity_name)
for call_requirement in call_requirements:
dsl_expr = call_requirement._dsl_expr
call_id = call_requirement.id
# Todo: Consolidate 'date' attributes to be a single element (rather than a possibly long sum expression).
# Identify this call's requirements for simulated prices.
simulation_requirements = set()
present_time = call_requirement.present_time
dsl_expr.identify_price_simulation_requirements(
requirements=simulation_requirements,
present_time=present_time,
observation_date=observation_date,
periodisation=periodisation,
all_market_names=all_market_names,
)
# Register the simulation requirements for each call (needed during evaluation).
register_simulated_price_requirements(call_id, list(simulation_requirements))
# Update the simulation requirements (needed for the market simulation).
requirements.update(simulation_requirements)
# Identify this call's perturbation dependencies.
perturbation_dependencies = set()
dsl_expr.identify_perturbation_dependencies(
dependencies=perturbation_dependencies,
present_time=present_time,
periodisation=periodisation,
)
# Add the expression's perturbation dependencies to the perturbation dependencies of its call dependencies.
call_dependencies = call_dependencies_repo[call_id]
assert isinstance(call_dependencies, CallDependencies), call_dependencies
for dependency_id in call_dependencies.dependencies:
dependency_perturbation_dependencies = all_perturbation_dependencies[dependency_id]
perturbation_dependencies.update(dependency_perturbation_dependencies)
# Register the perturbation dependencies in the repo (needed when evaluating the call).
register_perturbation_dependencies(call_id, list(perturbation_dependencies))
# Save the perturbation dependencies for this call, so they are available for the dependent calls.
all_perturbation_dependencies[call_id] = perturbation_dependencies
def list_market_names(contract_specification):
assert isinstance(contract_specification, ContractSpecification)
dsl_module = dsl_parse(contract_specification.specification)
return list(find_market_names(dsl_expr=dsl_module))
def test_empty_string(self):
self.assertTrue(isinstance(dsl_parse(""), Module))
self.assertRaises(DslSyntaxError, dsl_compile, "")
self.assertRaises(DslSyntaxError, dsl_eval, "")
def identify_simulation_requirements(contract_specification_id,
call_requirement_repo,
call_link_repo,
call_dependencies_repo,
observation_date,
requirements,
periodisation=None):
assert isinstance(call_requirement_repo, CallRequirementRepository)
assert isinstance(call_link_repo, CallLinkRepository)
all_perturbation_dependencies = {}
call_requirements = []
for call_id in regenerate_execution_order(contract_specification_id,
call_link_repo):
# Get the stubbed expression.
call_requirement = call_requirement_repo[call_id]
assert isinstance(call_requirement, CallRequirement)
if call_requirement._dsl_expr is not None:
dsl_expr = call_requirement._dsl_expr
else:
dsl_module = dsl_parse(call_requirement.dsl_source)
dsl_expr = dsl_module.body[0]
call_requirement._dsl_expr = dsl_expr
assert isinstance(dsl_expr, DslObject), dsl_expr
call_requirements.append(call_requirement)
all_market_names = set()
for call_requirement in call_requirements:
dsl_expr = call_requirement._dsl_expr
for market in dsl_expr.find_instances(AbstractMarket):
assert isinstance(market.commodity_name, six.string_types)
all_market_names.add(market.commodity_name)
for call_requirement in call_requirements:
dsl_expr = call_requirement._dsl_expr
call_id = call_requirement.id
# Todo: Consolidate 'date' attributes to be a single element (rather than a possibly long sum expression).
# Identify this call's requirements for simulated prices.
simulation_requirements = set()
present_time = call_requirement.present_time
dsl_expr.identify_price_simulation_requirements(
requirements=simulation_requirements,
present_time=present_time,
observation_date=observation_date,
periodisation=periodisation,
all_market_names=all_market_names,
)
# Register the simulation requirements for each call (needed during evaluation).
register_simulated_price_requirements(call_id,
list(simulation_requirements))
# Update the simulation requirements (needed for the market simulation).
requirements.update(simulation_requirements)
# Identify this call's perturbation dependencies.
perturbation_dependencies = set()
dsl_expr.identify_perturbation_dependencies(
dependencies=perturbation_dependencies,
present_time=present_time,
periodisation=periodisation,
)
# Add the expression's perturbation dependencies to the perturbation dependencies of its call dependencies.
call_dependencies = call_dependencies_repo[call_id]
assert isinstance(call_dependencies,
CallDependencies), call_dependencies
for dependency_id in call_dependencies.dependencies:
dependency_perturbation_dependencies = all_perturbation_dependencies[
dependency_id]
perturbation_dependencies.update(
dependency_perturbation_dependencies)
# Register the perturbation dependencies in the repo (needed when evaluating the call).
register_perturbation_dependencies(call_id,
list(perturbation_dependencies))
# Save the perturbation dependencies for this call, so they are available for the dependent calls.
all_perturbation_dependencies[call_id] = perturbation_dependencies
def test_empty_string(self):
# Empty string can be parsed as a module, but not compiled into an expression, or evaluated.
self.assertTrue(isinstance(dsl_parse(""), Module))
self.assertRaises(DslSyntaxError, dsl_compile, "")
self.assertRaises(DslSyntaxError, dsl_eval, "")