def test_call_functions_without_pending_call_stack(self):
fc = FunctionCall(Name('f'), [Name('x')])
fd = FunctionDef('f', [FunctionArg('a', '')], Name('a'), [])
number1 = Number(1234)
number2 = Number(2345)
ns1 = DslNamespace({
'f': fd,
'x': number1,
})
ns2 = DslNamespace({
'f': fd,
'x': number2,
})
# Call functions with pending call stack.
queue = Mock()
fc1 = fc.substitute_names(ns1)
fc2 = fc.substitute_names(ns2)
self.assertNotEqual(fd.create_hash(fc1), fd.create_hash(fc2))
t1 = datetime.datetime(2011, 1, 1)
expr = fc1.call_functions(pending_call_stack=queue, present_time=t1)
# Check we got a stub.
self.assertIsInstance(expr, Stub)
self.assertEqual(queue.put.call_count, 1)
# Check the call to the stub was queued.
first_call = queue.put.mock_calls[0]
self.assertEqual(first_call[2]['stub_id'], expr.name)
self.assertEqual(first_call[2]['stacked_function_def'], fd)
self.assertEqual(first_call[2]['stacked_locals'], {'a': 1234}) # Maybe this should be Number(1234)?
self.assertEqual(first_call[2]['present_time'], t1)
def dsl_compile(dsl_source, filename='<unknown>', is_parallel=None, dsl_classes=None, compile_kwds=None, **extraCompileKwds):
"""
Returns a DSL expression, created according to the given DSL source module.
That is, if the source module contains a function def and an expression which
calls that function, then the expression's function call will be evaluated
and the resulting DSL expression will be substituted for the function call
in the module's expression, so that calls to user defined functions are eliminated
and a single DSL expression is obtained.
If the source module contains a function def, but no expression, the module is compiled
into a function def object. Calling .apply() on a function def object will return a DSL
expression object, which can be evaluated by calling its .evaluate() method.
"""
if compile_kwds is None:
compile_kwds = DslNamespace()
assert isinstance(compile_kwds, dict)
compile_kwds.update(extraCompileKwds)
# Parse the source into a DSL module object.
dsl_module = dsl_parse(dsl_source, filename=filename, dsl_classes=dsl_classes)
assert isinstance(dsl_module, Module)
# Compile the module into either a dependency graph
# if 'is_parallel' is True, otherwise a single primitive expression.
return compile_dsl_module(dsl_module, DslNamespace(), compile_kwds, is_dependency_graph=is_parallel)
def evaluate_call(call_spec, register_call_result):
"""
Evaluates the stubbed expr identified by 'call_requirement_id'.
"""
assert isinstance(call_spec, CallSpecification)
evaluation_kwds = call_spec.evaluation_kwds.copy()
# If this call has an effective present time value, use it as the 'present_time' in the evaluation_kwds.
# This results from e.g. the Wait DSL element. Calls near the root of the expression might not have an
# effective present time value, and the present time will be the observation time of the evaluation.
# Evaluate the stubbed expr str.
# - parse the expr
try:
# Todo: Rework this dependency. Figure out how to use alternative set of DSL classes when multiprocessing.
from quantdsl.domain.services.parser import dsl_parse
stubbed_module = dsl_parse(call_spec.dsl_expr_str)
except DslSyntaxError:
raise
assert isinstance(stubbed_module, Module), "Parsed stubbed expr string is not a module: %s" % stubbed_module
# - build a namespace from the dependency values
dsl_locals = DslNamespace(call_spec.dependency_values)
# - compile the parsed expr
dsl_expr = stubbed_module.body[0].reduce(dsl_locals=dsl_locals, dsl_globals=DslNamespace())
assert isinstance(dsl_expr, DslExpression), dsl_expr
# - evaluate the compiled expr
result_value = dsl_expr.evaluate(**evaluation_kwds)
# - store the result
register_call_result(call_id=call_spec.id, result_value=result_value)
def dsl_compile(dsl_source,
filename='<unknown>',
dsl_classes=None,
compile_kwds=None,
**extraCompileKwds):
"""
Returns a DSL expression, created according to the given DSL source module.
That is, if the source module contains a function def and an expression which
calls that function, then the expression's function call will be evaluated
and the resulting DSL expression will be substituted for the function call
in the module's expression, so that calls to user defined functions are eliminated
and a single DSL expression is obtained.
If the source module contains a function def, but no expression, the module is compiled
into a function def object. Calling .apply() on a function def object will return a DSL
expression object, which can be evaluated by calling its .evaluate() method.
"""
if compile_kwds is None:
compile_kwds = DslNamespace()
# assert isinstance(compile_kwds, dict)
compile_kwds.update(extraCompileKwds)
# Parse the source into a DSL module object.
dsl_module = dsl_parse(dsl_source,
filename=filename,
dsl_classes=dsl_classes)
# assert isinstance(dsl_module, Module)
# Compile the module into either a dependency graph
# if 'is_parallel' is True, otherwise a single primitive expression.
return compile_dsl_module(dsl_module, DslNamespace(), compile_kwds)
def generate_dependency_graph(contract_specification, call_dependencies_repo, call_dependents_repo, call_leafs_repo,
call_requirement_repo):
assert isinstance(contract_specification, ContractSpecification)
dsl_module = dsl_parse(dsl_source=contract_specification.specification)
assert isinstance(dsl_module, Module)
dsl_globals = DslNamespace()
function_defs, expressions = extract_defs_and_exprs(dsl_module, dsl_globals)
dsl_expr = expressions[0]
assert isinstance(dsl_expr, DslExpression)
dsl_locals = DslNamespace()
leaf_ids = []
all_dependents = defaultdict(list)
# Generate stubbed call from the parsed DSL module object.
for stub in generate_stubbed_calls(contract_specification.id, dsl_module, dsl_expr, dsl_globals, dsl_locals):
# assert isinstance(stub, StubbedCall)
# Register the call requirements.
call_id = stub.call_id
dsl_source = str(stub.dsl_expr)
effective_present_time = stub.effective_present_time
call_requirement = register_call_requirement(call_id, dsl_source, effective_present_time)
# Hold onto the dsl_expr, helps in "single process" modes....
call_requirement._dsl_expr = stub.dsl_expr
# - put the entity directly in the cache, otherwise the entity will be regenerated when it is next accessed
# and the _dsl_expr will be lost.
call_requirement_repo.add_cache(call_id, call_requirement)
# Register the call requirements.
dependencies = stub.requirements
register_call_dependencies(call_id, dependencies)
# Keep track of the leaves and the dependents.
if len(dependencies) == 0:
leaf_ids.append(call_id)
else:
for dependency_call_id in dependencies:
all_dependents[dependency_call_id].append(call_id)
# Register the call dependents.
for call_id, dependents in all_dependents.items():
register_call_dependents(call_id, dependents)
register_call_dependents(contract_specification.id, [])
# Generate and register the call order.
link_id = contract_specification.id
for call_id in generate_execution_order(leaf_ids, call_dependents_repo, call_dependencies_repo):
register_call_link(link_id, call_id)
link_id = call_id
# Register the leaf ids.
register_call_leafs(contract_specification.id, leaf_ids)
def compile_dsl_module(dsl_module, dsl_locals=None, dsl_globals=None):
"""
Returns something that can be evaluated.
"""
# It's a module compilation, so create a new namespace "context".
if dsl_locals is None:
dsl_locals = {}
dsl_locals = DslNamespace(dsl_locals)
if dsl_globals is None:
dsl_globals = {}
dsl_globals = DslNamespace(dsl_globals)
# Can't do much with an empty module.
if len(dsl_module.body) == 0:
raise DslSyntaxError('empty module', node=dsl_module.node)
function_defs, expressions = extract_defs_and_exprs(
dsl_module, dsl_globals)
# Handle different combinations of functions and module level expressions in different ways.
# Todo: Simplify this, but support library files first?
# Can't meaningfully evaluate more than one expression (since assignments are not supported).
if len(expressions) > 1:
raise DslSyntaxError('more than one expression in module',
node=expressions[1].node)
# Can't meaningfully evaluate more than one function def without a module level expression.
elif len(expressions) == 0 and len(function_defs) > 1:
second_def = function_defs[1]
raise DslSyntaxError(
'more than one function def in module without an expression',
'"def %s"' % second_def.name,
node=function_defs[1].node)
# If it's just a module with one function, then return the function def.
elif len(expressions) == 0 and len(function_defs) == 1:
return function_defs[0]
# If there is one expression, reduce it with the function defs that it calls.
else:
assert len(expressions) == 1
dsl_expr = expressions[0]
# assert isinstance(dsl_expr, DslExpression), dsl_expr
# Compile the module for a single threaded recursive operation (faster but not distributed,
# so also limited in space and perhaps time). For smaller computations only.
dsl_obj = dsl_expr.substitute_names(dsl_globals.combine(dsl_locals))
return dsl_obj
def evaluate_dsl_expr(dsl_expr, first_commodity_name, simulation_id,
interest_rate, present_time, simulated_value_dict,
perturbation_dependencies, dependency_results,
path_count):
evaluation_kwds = {
'simulated_value_dict': simulated_value_dict,
'simulation_id': simulation_id,
'interest_rate': interest_rate,
'present_time': present_time,
'first_commodity_name': first_commodity_name,
'path_count': path_count,
}
result_value = None
perturbed_values = {}
for perturbation in [None] + perturbation_dependencies:
evaluation_kwds['active_perturbation'] = perturbation
# Initialise namespace with the dependency values.
dependency_values = {}
for stub_id in dependency_results.keys():
(dependency_result_value,
dependency_perturbed_values) = dependency_results[stub_id]
try:
dependency_value = dependency_perturbed_values[str(
perturbation)]
except KeyError:
dependency_value = dependency_result_value
dependency_values[stub_id] = dependency_value
dsl_locals = DslNamespace(dependency_values)
# Compile the parsed expr using the namespace to give something that can be evaluated.
dsl_expr_reduced = dsl_expr.reduce(dsl_locals=dsl_locals,
dsl_globals=DslNamespace())
# assert isinstance(dsl_expr, DslExpression), dsl_expr
expr_value = dsl_expr_reduced.evaluate(**evaluation_kwds)
if perturbation is None:
assert result_value is None
result_value = expr_value
else:
perturbed_values[perturbation] = expr_value
return result_value, perturbed_values
def generate_dependency_graph(contract_specification, call_dependencies_repo,
call_dependents_repo):
assert isinstance(contract_specification, ContractSpecification)
dsl_module = dsl_parse(dsl_source=contract_specification.specification)
assert isinstance(dsl_module, Module)
dsl_globals = DslNamespace()
function_defs, expressions = extract_defs_and_exprs(
dsl_module, dsl_globals)
dsl_expr = expressions[0]
assert isinstance(dsl_expr, DslExpression)
dsl_locals = DslNamespace()
leaf_call_ids = []
all_dependents = defaultdict(list)
# Generate stubbed call from the parsed DSL module object.
for stub in generate_stubbed_calls(contract_specification.id, dsl_module,
dsl_expr, dsl_globals, dsl_locals):
assert isinstance(stub, StubbedCall)
call_id = stub.call_id
dsl_source = stub.dsl_source
effective_present_time = stub.effective_present_time
dependencies = stub.dependencies
# Register the call requirements.
register_call_requirement(call_id, dsl_source, effective_present_time)
# Register the call dependencies.
register_call_dependencies(call_id, dependencies)
# Keep track of the leaves and the dependents.
if len(dependencies) == 0:
leaf_call_ids.append(call_id)
else:
for dependency_call_id in dependencies:
all_dependents[dependency_call_id].append(call_id)
# Register the call dependents.
for call_id, dependents in all_dependents.items():
register_call_dependents(call_id, dependents)
register_call_dependents(contract_specification.id, [])
# Generate and register the call order.
link_id = contract_specification.id
for call_id in generate_execution_order(leaf_call_ids,
call_dependents_repo,
call_dependencies_repo):
register_call_link(link_id, call_id)
link_id = call_id
def visitModule(self, node):
"""
Visitor method for ast.Module nodes.
Returns a DSL Module, with a list of DSL expressions as the body.
"""
# assert isinstance(node, ast.Module)
body = []
# Namespace for function defs in module.
module_namespace = DslNamespace()
for n in node.body:
dsl_object = self.visitAstNode(n)
# Put function defs in module namespace.
if isinstance(dsl_object, FunctionDef):
module_namespace[dsl_object.name] = dsl_object
# Share module namespace with this function.
if dsl_object.module_namespace is None:
dsl_object.module_namespace = module_namespace
# Include imported things.
if isinstance(dsl_object, list):
for _dsl_object in dsl_object:
if isinstance(_dsl_object, FunctionDef):
module_namespace[_dsl_object.name] = _dsl_object
else:
body.append(dsl_object)
return self.dsl_classes['Module'](body, module_namespace, node=node)
def evaluate_dsl_expr(dsl_expr, simulation_id, interest_rate, present_time,
simulated_value_dict, perturbation_dependencies,
dependency_results, path_count, perturbation_factor,
periodisation, estimated_cost_of_expr, observation_date,
is_double_sided_deltas, involved_market_names):
evaluation_kwds = {
'simulated_value_dict': simulated_value_dict,
'simulation_id': simulation_id,
'interest_rate': interest_rate,
'perturbation_factor': perturbation_factor,
'observation_date': observation_date,
'present_time': present_time,
'involved_market_names': involved_market_names,
'path_count': path_count,
'periodisation': periodisation,
}
result_value = None
perturbed_values = {}
# Decide perturbation names.
perturbation_names = perturbation_dependencies
if is_double_sided_deltas:
perturbation_names += ['-' + p for p in perturbation_dependencies]
for perturbation_name in [''] + perturbation_names:
evaluation_kwds['active_perturbation'] = perturbation_name
# Initialise namespace with the dependency values.
dependency_values = {}
for stub_id in dependency_results.keys():
dependency_result = dependency_results[stub_id]
try:
dependency_value = dependency_result.perturbed_values[
perturbation_name]
except KeyError:
dependency_value = dependency_result.result_value
dependency_values[stub_id] = dependency_value
# Prepare the namespace with the values the expression depends on.
dsl_locals = DslNamespace(dependency_values)
# Substitute Name elements, to give something that can be evaluated.
dsl_expr_resolved = dsl_expr.substitute_names(dsl_locals)
# Evaluate the expression.
expr_value = dsl_expr_resolved.evaluate(**evaluation_kwds)
if not perturbation_name:
assert result_value is None
result_value = expr_value
else:
perturbed_values[perturbation_name] = expr_value
# Publish result value computed event.
publish(ResultValueComputed(estimated_cost_of_expr))
return result_value, perturbed_values
def compile_dsl_module(dsl_module, dsl_locals=None, dsl_globals=None):
"""
Returns something that can be evaluated.
"""
# It's a module compilation, so create a new namespace "context".
if dsl_locals is None:
dsl_locals = {}
dsl_locals = DslNamespace(dsl_locals)
if dsl_globals is None:
dsl_globals = {}
dsl_globals = DslNamespace(dsl_globals)
# Can't do much with an empty module.
if len(dsl_module.body) == 0:
raise DslSyntaxError('empty module', node=dsl_module.node)
function_defs, expressions = extract_defs_and_exprs(dsl_module, dsl_globals)
# Handle different combinations of functions and module level expressions in different ways.
# Todo: Simplify this, but support library files first?
# Can't meaningfully evaluate more than one expression (since assignments are not supported).
if len(expressions) > 1:
raise DslSyntaxError('more than one expression in module', node=expressions[1].node)
# Can't meaningfully evaluate more than one function def without a module level expression.
elif len(expressions) == 0 and len(function_defs) > 1:
second_def = function_defs[1]
raise DslSyntaxError('more than one function def in module without an expression',
'"def %s"' % second_def.name, node=function_defs[1].node)
# If it's just a module with one function, then return the function def.
elif len(expressions) == 0 and len(function_defs) == 1:
return function_defs[0]
# If there is one expression, reduce it with the function defs that it calls.
else:
assert len(expressions) == 1
dsl_expr = expressions[0]
# assert isinstance(dsl_expr, DslExpression), dsl_expr
# Compile the module for a single threaded recursive operation (faster but not distributed,
# so also limited in space and perhaps time). For smaller computations only.
dsl_obj = dsl_expr.substitute_names(dsl_globals.combine(dsl_locals))
return dsl_obj
def test_substitute_names(self):
fc = FunctionCall(Name('f'), [Name('x')])
fd = FunctionDef('f', [], Name('a'), [])
ns = DslNamespace({
'f': fd,
'x': Number(1),
})
fc1 = fc.substitute_names(ns)
self.assertEqual(fc1.functionDef, fd)
self.assertEqual(fc1.callArgExprs[0], Number(1))
def test_call_functions_with_pending_call_stack(self):
fc = FunctionCall(Name('f'), [Name('x')])
fd = FunctionDef('f', [FunctionArg('a', '')], Name('a'), [])
namespace = DslNamespace({fd.name: fd})
fd.module_namespace = namespace
number = Number(1234)
ns = DslNamespace({
'f': fd,
'x': number,
})
# Substitute names.
fc1 = fc.substitute_names(ns)
self.assertEqual(fc1.functionDef, fd)
self.assertEqual(fc1.callArgExprs[0], number)
# Call functions.
expr = fc1.call_functions()
self.assertEqual(expr, number)
def test_substitute(self):
# Maybe if Name can take a string, perhaps also other things can?
# Maybe the parser should return Python string, numbers etc?
# Maybe String and Number etc don't add anything?
obj = Name('a')
self.assertEqual(obj.name, 'a')
obj = Name(String('a'))
self.assertEqual(obj.name, 'a')
ns = DslNamespace()
with self.assertRaises(DslNameError):
obj.substitute_names(ns)
ns = DslNamespace({'a': 1})
self.assertEqual(obj.substitute_names(ns), Number(1))
ns = DslNamespace({'a': datetime.timedelta(1)})
self.assertEqual(obj.substitute_names(ns), TimeDelta(datetime.timedelta(1)))
function_def = FunctionDef('f', [], Number(1), [])
ns = DslNamespace({'a': function_def})
self.assertEqual(obj.substitute_names(ns), function_def)
def calc_value(self, dsl_source, observation_date):
# Todo: Rename 'allRvs' to 'simulatedPrices'?
evaluation_kwds = DslNamespace({
'observation_date': observation_date,
'interest_rate': '2.5',
'market_calibration': {
'#1-LAST-PRICE': 10,
'#1-ACTUAL-HISTORICAL-VOLATILITY': 50,
'#2-LAST-PRICE': 10,
'#2-ACTUAL-HISTORICAL-VOLATILITY': 50,
'#1-#2-CORRELATION': 0.0,
'NBP-LAST-PRICE': 10,
'NBP-ACTUAL-HISTORICAL-VOLATILITY': 50,
'TTF-LAST-PRICE': 11,
'TTF-ACTUAL-HISTORICAL-VOLATILITY': 40,
'BRENT-LAST-PRICE': 90,
'BRENT-ACTUAL-HISTORICAL-VOLATILITY': 60,
'NBP-TTF-CORRELATION': 0.4,
'BRENT-TTF-CORRELATION': 0.5,
'BRENT-NBP-CORRELATION': 0.3,
},
'path_count': 200000,
# 'simulated_price_repo': {
# 'sim1#12011-01-01': Mock(spec=SimulatedPrice, value=numpy.array([10])),
# 'sim1#12011-01-03': Mock(spec=SimulatedPrice, value=numpy.array([10])),
# 'sim1#12011-06-01': Mock(spec=SimulatedPrice, value=numpy.array([10])),
# 'sim1#12012-01-01': Mock(spec=SimulatedPrice, value=numpy.array([10])),
# 'sim1#12012-01-02': Mock(spec=SimulatedPrice, value=numpy.array([10])),
# 'sim1#12012-01-03': Mock(spec=SimulatedPrice, value=numpy.array([10])),
# 'sim1#12012-06-01': Mock(spec=SimulatedPrice, value=numpy.array([10])),
# 'sim1#12013-01-01': Mock(spec=SimulatedPrice, value=numpy.array([10])),
#
# 'sim1#22011-01-01': Mock(spec=SimulatedPrice, value=numpy.array([10])),
# 'sim1#22012-01-01': Mock(spec=SimulatedPrice, value=numpy.array([10])),
#
# 'sim1TTF2012-01-01': Mock(spec=SimulatedPrice, value=numpy.array([10])),
# 'sim1NBP2012-01-01': Mock(spec=SimulatedPrice, value=numpy.array([10])),
# 'sim1TTF2013-01-01': Mock(spec=SimulatedPrice, value=numpy.array([10])),
# 'sim1NBP2013-01-01': Mock(spec=SimulatedPrice, value=numpy.array([10])),
# },
'simulation_id': 'sim1',
'first_market_name': '#1',
})
return dsl_eval(dsl_source, evaluation_kwds=evaluation_kwds)
def generate_contract_valuation(self, dependency_graph_id, market_simulation):
assert isinstance(dependency_graph_id, six.string_types), dependency_graph_id
assert isinstance(market_simulation, MarketSimulation)
v = self.register_contract_valuation(dependency_graph_id)
for call_id in regenerate_execution_order(dependency_graph_id, self.call_link_repo):
call = self.call_requirement_repo[call_id]
assert isinstance(call, CallRequirement)
# Evaluate the call requirement.
dependency_values = get_dependency_values(call_id, self.call_dependencies_repo, self.call_result_repo)
# - parse the expr
stubbed_module = dsl_parse(call.dsl_source)
assert isinstance(stubbed_module, Module), "Parsed stubbed expr string is not a module: %s" % stubbed_module
# - build a namespace from the dependency values
dsl_locals = DslNamespace(dependency_values)
# - compile the parsed expr
dsl_expr = stubbed_module.body[0].reduce(dsl_locals=dsl_locals, dsl_globals=DslNamespace())
assert isinstance(dsl_expr, DslExpression), dsl_expr
# - evaluate the compiled expr
first_market_name = market_simulation.market_names[0] if market_simulation.market_names else None
evaluation_kwds = {
'simulated_price_repo': self.simulated_price_repo,
'simulation_id': market_simulation.id,
'interest_rate': market_simulation.interest_rate,
'present_time': call.effective_present_time or market_simulation.observation_date,
'first_market_name': first_market_name,
}
result_value = dsl_expr.evaluate(**evaluation_kwds)
# - store the result
register_call_result(call_id=call_id, result_value=result_value)
def assertDslExprTypeValue(self, dsl_source, expectedDslType,
expectedDslValue, **compile_kwds):
# Assumes dsl_source is just one statement.
dsl_module = dsl_parse(dsl_source)
assert isinstance(dsl_module, Module)
# Check the parsed DSL can be rendered as a string that is equal to the original source.
self.assertEqual(str(dsl_module), dsl_source.strip())
# Assume this test is dealing with modules that have one statement only.
dsl_expr = dsl_module.body[0]
# Check expression type.
assert isinstance(dsl_expr, DslExpression)
self.assertIsInstance(dsl_expr, expectedDslType)
# Compile the module into a simple DSL expression object (no variables or calls to function defs).
dsl_expr = dsl_expr.substitute_names(DslNamespace(compile_kwds))
# Evaluate the compiled expression.
dsl_value = dsl_expr.evaluate()
self.assertEqual(dsl_value, expectedDslValue)
def dsl_eval(dsl_source, filename='<unknown>', is_parallel=None, dsl_classes=None, compile_kwds=None,
evaluation_kwds=None, price_process_name=None, is_multiprocessing=False, pool_size=0, is_verbose=False,
is_show_source=False, **extra_evaluation_kwds):
"""
Returns the result of evaluating a compiled module (an expression, or a user defined function).
An expression (with optional function defs) will evaluate to a simple value.
A function def will evaluate to a DSL expression, will may then be evaluated (more than one
function def without an expression is an error).
"""
if price_process_name is None:
price_process_name = DEFAULT_PRICE_PROCESS_NAME
if evaluation_kwds is None:
evaluation_kwds = DslNamespace()
assert isinstance(evaluation_kwds, dict)
evaluation_kwds.update(extra_evaluation_kwds)
if is_show_source:
print_("Reading DSL source:")
print_()
print_('"""')
print_(dsl_source.strip())
print_('"""')
print_()
if is_verbose:
print_("Compiling DSL source, please wait...")
print_()
compile_start_time = datetime.datetime.now()
# Compile the source into a primitive DSL expression, with optional dependency graph.
dsl_expr = dsl_compile(dsl_source, filename=filename, is_parallel=is_parallel, dsl_classes=dsl_classes,
compile_kwds=compile_kwds)
# Measure the compile_dsl_module time.
compile_time_delta = datetime.datetime.now() - compile_start_time
# Check the result of the compilation.
# Todo: This feels unnecessary?
if is_parallel:
assert isinstance(dsl_expr, DependencyGraph), type(dsl_expr)
else:
assert isinstance(dsl_expr, DslExpression), type(dsl_expr)
if is_verbose:
if isinstance(dsl_expr, DependencyGraph):
print_("Compiled DSL source into %d partial expressions (root ID: %s)." % (
len(dsl_expr.stubbed_calls), dsl_expr.root_stub_id))
print_()
print_("Duration of compilation: %s" % compile_time_delta)
print_()
if isinstance(dsl_expr, DependencyGraph):
if is_show_source:
print_("Expression stack:")
for stubbed_exprData in dsl_expr.stubbed_calls:
print_(" " + str(stubbed_exprData[0]) + ": " + str(stubbed_exprData[1]))
print_()
# If the expression has any stochastic elements, the evaluation kwds must have an 'observation_date' (datetime).
if dsl_expr.has_instances(dsl_type=StochasticObject):
observation_date = evaluation_kwds['observation_date']
assert isinstance(observation_date, datetime.date)
if is_verbose:
print_("Observation time: %s" % observation_date)
print_()
# Avoid any confusion with the internal 'present_time' variable.
if 'present_time' in evaluation_kwds:
msg = ("Don't set present_time here, set observation_date instead. "
"Hint: Adjust effective present time with Fixing or Wait elements.")
raise DslError(msg)
# Initialise present_time as observation_date.
evaluation_kwds['present_time'] = observation_date
# If the expression has any Market elements, a market simulation is required
if dsl_expr.has_instances(dsl_type=Market):
# If a market simulation is required, evaluation kwds must have 'path_count' (integer).
if 'path_count' not in evaluation_kwds:
evaluation_kwds['path_count'] = DEFAULT_PATH_COUNT
path_count = evaluation_kwds['path_count']
assert isinstance(path_count, int)
# If a market simulation is required, evaluation_kwds must have 'market_calibration' (integer).
market_calibration = evaluation_kwds['market_calibration']
assert isinstance(market_calibration, dict)
# If a market simulation is required, generate the simulated prices using the price process.
if not 'all_market_prices' in evaluation_kwds:
if is_verbose:
print_("Price process: %s" % price_process_name)
print_()
price_process = get_price_process(price_process_name)
if is_verbose:
print_("Path count: %d" % path_count)
print_()
if is_verbose:
print_("Finding all Market names and Fixing dates...")
print_()
# Extract market names from the expression.
# Todo: Avoid doing this on the dependency graph, when all the Market elements must be in the original.
market_names = find_market_names(dsl_expr)
# Extract fixing dates from the expression.
# Todo: Perhaps collect the fixing dates?
fixing_dates = list_fixing_dates(dsl_expr)
if is_verbose:
print_("Simulating future prices for Market%s '%s' from observation time %s through fixing dates: %s." % (
'' if len(market_names) == 1 else 's',
", ".join(market_names),
"'%04d-%02d-%02d'" % (observation_date.year, observation_date.month, observation_date.day),
# Todo: Only print first and last few, if there are loads.
", ".join(["'%04d-%02d-%02d'" % (d.year, d.month, d.day) for d in fixing_dates[:8]]) + \
(", [...]" if len(fixing_dates) > 9 else '') + \
((", '%04d-%02d-%02d'" % (fixing_dates[-1].year, fixing_dates[-1].month, fixing_dates[-1].day)) if len(fixing_dates) > 8 else '')
))
print_()
# Simulate the future prices.
all_market_prices = price_process.simulate_future_prices(market_names, fixing_dates, observation_date, path_count, market_calibration)
# Add future price simulation to evaluation_kwds.
evaluation_kwds['all_market_prices'] = all_market_prices
# Initialise the evaluation timer variable (needed by showProgress thread).
evalStartTime = None
if is_parallel:
if is_verbose:
len_stubbed_exprs = len(dsl_expr.stubbed_calls)
lenLeafIds = len(dsl_expr.leaf_ids)
msg = "Evaluating %d expressions (%d %s) with " % (len_stubbed_exprs, lenLeafIds, 'leaf' if lenLeafIds == 1 else 'leaves')
if is_multiprocessing and pool_size:
msg += "a multiprocessing pool of %s workers" % pool_size
else:
msg += "a single thread"
msg += ", please wait..."
print_(msg)
print_()
# Define showProgress() thread.
def showProgress(stop):
progress = 0
movingRates = []
while progress < 100 and not stop.is_set():
time.sleep(0.3)
if evalStartTime is None:
continue
# Avoid race condition.
if not hasattr(dsl_expr, 'runner') or not hasattr(dsl_expr.runner, 'resultIds'):
continue
if stop.is_set():
break
try:
lenResults = len(dsl_expr.runner.resultIds)
except IOError:
break
resultsTime = datetime.datetime.now()
movingRates.append((lenResults, resultsTime))
if len(movingRates) >= 15:
movingRates.pop(0)
if len(movingRates) > 1:
firstLenResults, firstTimeResults = movingRates[0]
lastLenResults, lastTimeResults = movingRates[-1]
lenDelta = lastLenResults - firstLenResults
resultsTimeDelta = lastTimeResults - firstTimeResults
timeDeltaSeconds = resultsTimeDelta.seconds + resultsTimeDelta.microseconds * 0.000001
rateStr = "%.2f expr/s" % (lenDelta / timeDeltaSeconds)
else:
rateStr = ''
progress = 100.0 * lenResults / len_stubbed_exprs
sys.stdout.write("\rProgress: %01.2f%% (%s/%s) %s " % (progress, lenResults, len_stubbed_exprs, rateStr))
sys.stdout.flush()
sys.stdout.write("\r")
sys.stdout.flush()
stop = threading.Event()
thread = threading.Thread(target=showProgress, args=(stop,))
# Start showProgress() thread.
thread.start()
# Start timing the evaluation.
evalStartTime = datetime.datetime.now()
try:
# Evaluate the primitive DSL expression.
if is_parallel:
if is_multiprocessing:
dependency_graph_runner_class = MultiProcessingDependencyGraphRunner
else:
dependency_graph_runner_class = SingleThreadedDependencyGraphRunner
value = dsl_expr.evaluate(dependency_graph_runner_class=dependency_graph_runner_class, pool_size=pool_size, **evaluation_kwds)
else:
value = dsl_expr.evaluate(**evaluation_kwds)
except:
if is_parallel:
if is_verbose:
if thread.isAlive():
# print "Thread is alive..."
stop.set()
# print "Waiting to join with thread..."
thread.join(timeout=1)
# print "Joined with thread..."
raise
# Stop timing the evaluation.
evalTimeDelta = datetime.datetime.now() - evalStartTime
if isinstance(dsl_expr, DependencyGraph):
if is_verbose:
# Join with showProgress thread.
thread.join(timeout=3)
if is_verbose:
timeDeltaSeconds = evalTimeDelta.seconds + evalTimeDelta.microseconds * 0.000001
if is_parallel:
len_stubbed_exprs = len(dsl_expr.stubbed_calls)
rateStr = "(%.2f expr/s)" % (len_stubbed_exprs / timeDeltaSeconds)
else:
rateStr = ''
print_("Duration of evaluation: %s %s" % (evalTimeDelta, rateStr))
print_()
# Prepare the result.
import scipy
if isinstance(value, scipy.ndarray):
mean = value.mean()
stderr = value.std() / math.sqrt(path_count)
return {
'mean': mean,
'stderr': stderr
}
else:
return value
def dsl_eval(dsl_source,
filename='<unknown>',
is_parallel=None,
dsl_classes=None,
compile_kwds=None,
evaluation_kwds=None,
price_process_name=None,
is_multiprocessing=False,
pool_size=0,
is_verbose=False,
is_show_source=False,
**extra_evaluation_kwds):
"""
Returns the result of evaluating a compiled module (an expression, or a user defined function).
An expression (with optional function defs) will evaluate to a simple value.
A function def will evaluate to a DSL expression, will may then be evaluated (more than one
function def without an expression is an error).
"""
if price_process_name is None:
price_process_name = DEFAULT_PRICE_PROCESS_NAME
if evaluation_kwds is None:
evaluation_kwds = DslNamespace()
assert isinstance(evaluation_kwds, dict)
evaluation_kwds.update(extra_evaluation_kwds)
if is_show_source:
print_("Reading DSL source:")
print_()
print_('"""')
print_(dsl_source.strip())
print_('"""')
print_()
if is_verbose:
print_("Compiling DSL source, please wait...")
print_()
compile_start_time = datetime.datetime.now()
# Compile the source into a primitive DSL expression, with optional dependency graph.
dsl_expr = dsl_compile(dsl_source,
filename=filename,
is_parallel=is_parallel,
dsl_classes=dsl_classes,
compile_kwds=compile_kwds)
# Measure the compile_dsl_module time.
compile_time_delta = datetime.datetime.now() - compile_start_time
# Check the result of the compilation.
# Todo: This feels unnecessary?
if is_parallel:
assert isinstance(dsl_expr, DependencyGraph), type(dsl_expr)
else:
assert isinstance(dsl_expr, DslExpression), type(dsl_expr)
if is_verbose:
if isinstance(dsl_expr, DependencyGraph):
print_(
"Compiled DSL source into %d partial expressions (root ID: %s)."
% (len(dsl_expr.stubbed_calls), dsl_expr.root_stub_id))
print_()
print_("Duration of compilation: %s" % compile_time_delta)
print_()
if isinstance(dsl_expr, DependencyGraph):
if is_show_source:
print_("Expression stack:")
for stubbed_exprData in dsl_expr.stubbed_calls:
print_(" " + str(stubbed_exprData[0]) + ": " +
str(stubbed_exprData[1]))
print_()
# If the expression has any stochastic elements, the evaluation kwds must have an 'observation_date' (datetime).
if dsl_expr.has_instances(dsl_type=StochasticObject):
observation_date = evaluation_kwds['observation_date']
assert isinstance(observation_date, datetime.date)
if is_verbose:
print_("Observation time: %s" % observation_date)
print_()
# Avoid any confusion with the internal 'present_time' variable.
if 'present_time' in evaluation_kwds:
msg = (
"Don't set present_time here, set observation_date instead. "
"Hint: Adjust effective present time with Fixing or Wait elements."
)
raise DslError(msg)
# Initialise present_time as observation_date.
evaluation_kwds['present_time'] = observation_date
# If the expression has any Market elements, a market simulation is required
if dsl_expr.has_instances(dsl_type=Market):
# If a market simulation is required, evaluation kwds must have 'path_count' (integer).
if 'path_count' not in evaluation_kwds:
evaluation_kwds['path_count'] = DEFAULT_PATH_COUNT
path_count = evaluation_kwds['path_count']
assert isinstance(path_count, int)
# If a market simulation is required, evaluation_kwds must have 'market_calibration' (integer).
market_calibration = evaluation_kwds['market_calibration']
assert isinstance(market_calibration, dict)
# If a market simulation is required, generate the simulated prices using the price process.
if not 'all_market_prices' in evaluation_kwds:
if is_verbose:
print_("Price process: %s" % price_process_name)
print_()
price_process = get_price_process(price_process_name)
if is_verbose:
print_("Path count: %d" % path_count)
print_()
if is_verbose:
print_("Finding all Market names and Fixing dates...")
print_()
# Extract market names from the expression.
# Todo: Avoid doing this on the dependency graph, when all the Market elements must be in the original.
market_names = find_market_names(dsl_expr)
# Extract fixing dates from the expression.
# Todo: Perhaps collect the fixing dates?
fixing_dates = list_fixing_dates(dsl_expr)
if is_verbose:
print_("Simulating future prices for Market%s '%s' from observation time %s through fixing dates: %s." % (
'' if len(market_names) == 1 else 's',
", ".join(market_names),
"'%04d-%02d-%02d'" % (observation_date.year, observation_date.month, observation_date.day),
# Todo: Only print first and last few, if there are loads.
", ".join(["'%04d-%02d-%02d'" % (d.year, d.month, d.day) for d in fixing_dates[:8]]) + \
(", [...]" if len(fixing_dates) > 9 else '') + \
((", '%04d-%02d-%02d'" % (fixing_dates[-1].year, fixing_dates[-1].month, fixing_dates[-1].day)) if len(fixing_dates) > 8 else '')
))
print_()
# Simulate the future prices.
all_market_prices = price_process.simulate_future_prices(
market_names, fixing_dates, observation_date, path_count,
market_calibration)
# Add future price simulation to evaluation_kwds.
evaluation_kwds['all_market_prices'] = all_market_prices
# Initialise the evaluation timer variable (needed by showProgress thread).
evalStartTime = None
if is_parallel:
if is_verbose:
len_stubbed_exprs = len(dsl_expr.stubbed_calls)
lenLeafIds = len(dsl_expr.leaf_ids)
msg = "Evaluating %d expressions (%d %s) with " % (
len_stubbed_exprs, lenLeafIds,
'leaf' if lenLeafIds == 1 else 'leaves')
if is_multiprocessing and pool_size:
msg += "a multiprocessing pool of %s workers" % pool_size
else:
msg += "a single thread"
msg += ", please wait..."
print_(msg)
print_()
# Define showProgress() thread.
def showProgress(stop):
progress = 0
movingRates = []
while progress < 100 and not stop.is_set():
time.sleep(0.3)
if evalStartTime is None:
continue
# Avoid race condition.
if not hasattr(dsl_expr, 'runner') or not hasattr(
dsl_expr.runner, 'resultIds'):
continue
if stop.is_set():
break
try:
lenResults = len(dsl_expr.runner.resultIds)
except IOError:
break
resultsTime = datetime.datetime.now()
movingRates.append((lenResults, resultsTime))
if len(movingRates) >= 15:
movingRates.pop(0)
if len(movingRates) > 1:
firstLenResults, firstTimeResults = movingRates[0]
lastLenResults, lastTimeResults = movingRates[-1]
lenDelta = lastLenResults - firstLenResults
resultsTimeDelta = lastTimeResults - firstTimeResults
timeDeltaSeconds = resultsTimeDelta.seconds + resultsTimeDelta.microseconds * 0.000001
rateStr = "%.2f expr/s" % (lenDelta / timeDeltaSeconds)
else:
rateStr = ''
progress = 100.0 * lenResults / len_stubbed_exprs
sys.stdout.write(
"\rProgress: %01.2f%% (%s/%s) %s " %
(progress, lenResults, len_stubbed_exprs, rateStr))
sys.stdout.flush()
sys.stdout.write("\r")
sys.stdout.flush()
stop = threading.Event()
thread = threading.Thread(target=showProgress, args=(stop, ))
# Start showProgress() thread.
thread.start()
# Start timing the evaluation.
evalStartTime = datetime.datetime.now()
try:
# Evaluate the primitive DSL expression.
if is_parallel:
if is_multiprocessing:
dependency_graph_runner_class = MultiProcessingDependencyGraphRunner
else:
dependency_graph_runner_class = SingleThreadedDependencyGraphRunner
value = dsl_expr.evaluate(
dependency_graph_runner_class=dependency_graph_runner_class,
pool_size=pool_size,
**evaluation_kwds)
else:
value = dsl_expr.evaluate(**evaluation_kwds)
except:
if is_parallel:
if is_verbose:
if thread.isAlive():
# print "Thread is alive..."
stop.set()
# print "Waiting to join with thread..."
thread.join(timeout=1)
# print "Joined with thread..."
raise
# Stop timing the evaluation.
evalTimeDelta = datetime.datetime.now() - evalStartTime
if isinstance(dsl_expr, DependencyGraph):
if is_verbose:
# Join with showProgress thread.
thread.join(timeout=3)
if is_verbose:
timeDeltaSeconds = evalTimeDelta.seconds + evalTimeDelta.microseconds * 0.000001
if is_parallel:
len_stubbed_exprs = len(dsl_expr.stubbed_calls)
rateStr = "(%.2f expr/s)" % (len_stubbed_exprs / timeDeltaSeconds)
else:
rateStr = ''
print_("Duration of evaluation: %s %s" % (evalTimeDelta, rateStr))
print_()
# Prepare the result.
import scipy
if isinstance(value, scipy.ndarray):
mean = value.mean()
stderr = value.std() / math.sqrt(path_count)
return {'mean': mean, 'stderr': stderr}
else:
return value
def generate_dependency_graph(contract_specification,
call_dependencies_repo,
call_dependents_repo,
call_requirement_repo,
dsl_classes=None):
assert isinstance(contract_specification, ContractSpecification)
dsl_module = dsl_parse(
dsl_source=contract_specification.source_code,
dsl_classes=dsl_classes,
)
assert isinstance(dsl_module, Module)
dsl_globals = dsl_module.namespace.copy()
function_defs, expressions = extract_defs_and_exprs(
dsl_module, dsl_globals)
dsl_expr = expressions[0]
assert isinstance(dsl_expr, DslExpression)
dsl_locals = DslNamespace()
leaf_ids = []
all_dependents = defaultdict(list)
# Generate stubbed call from the parsed DSL module object.
for stubed_call in generate_stubbed_calls(
contract_specification.id, dsl_expr, dsl_globals, dsl_locals,
contract_specification.observation_date):
# assert isinstance(stub, StubbedCall)
# Estimate the cost of evaluating this expression.
estimated_cost = stubed_call.dsl_expr.cost_expression()
# Register the call requirements.
call_id = stubed_call.call_id
dsl_source = str(stubed_call.dsl_expr)
present_time = stubed_call.present_time
call_requirement = register_call_requirement(
call_id=call_id,
dsl_source=dsl_source,
present_time=present_time,
contract_specification_id=contract_specification.id,
cost=estimated_cost,
)
# Hold onto the dsl_expr, helps in "single process" modes.
# - put the entity directly in the cache, otherwise the entity will be
# regenerated when it is next accessed and the _dsl_expr will be "lost"
call_requirement._dsl_expr = stubed_call.dsl_expr
call_requirement_repo.add_cache(call_id, call_requirement)
# Register the call dependencies (things needed by this call).
dependencies = stubed_call.requirements
register_call_dependencies(call_id, dependencies)
# Keep track of the leaves and the dependents.
if len(dependencies) == 0:
leaf_ids.append(call_id)
else:
for dependency_call_id in dependencies:
all_dependents[dependency_call_id].append(call_id)
# Register the call dependents.
for call_id, dependents in all_dependents.items():
register_call_dependents(call_id, dependents)
register_call_dependents(contract_specification.id, [])
# Generate and register the call order.
link_id = contract_specification.id
for call_id in generate_execution_order(leaf_ids, call_dependents_repo,
call_dependencies_repo):
register_call_link(link_id, call_id)
link_id = call_id
# Register the leaf ids.
register_call_leafs(contract_specification.id, leaf_ids)