def check_tracing_params(accessor, param_key):
tracer = Tracer()
tracer.record_calculation_start('A', '2015-01')
tracingParams = TracingParameterNodeAtInstant(parameters('2015-01-01'),
tracer)
param = accessor(tracingParams)
assert_near(tracer.trace['A<2015-01>']['parameters'][param_key], param)
def test_variable_stats():
tracer = Tracer()
tracer.record_calculation_start("A", 2017)
tracer.record_calculation_start("B", 2017)
tracer.record_calculation_start("B", 2017)
tracer.record_calculation_start("B", 2016)
assert tracer.usage_stats['B']['nb_requests'] == 3
assert tracer.usage_stats['A']['nb_requests'] == 1
assert tracer.usage_stats['C']['nb_requests'] == 0
def __init__(
self,
tax_benefit_system,
simulation_json=None,
debug=False,
period=None,
trace=False,
opt_out_cache=False,
memory_config=None,
):
"""
Create an empty simulation
To fill the simulation with input data, you can use the :any:`SimulationBuilder` or proceed manually.
"""
self.tax_benefit_system = tax_benefit_system
assert tax_benefit_system is not None
if period:
assert isinstance(period, periods.Period)
self.period = period
# To keep track of the values (formulas and periods) being calculated to detect circular definitions.
# See use in formulas.py.
# The data structure of requested_periods_by_variable_name is: {variable_name: [period1, period2]}
self.requested_periods_by_variable_name = {}
self.max_nb_cycles = None
self.debug = debug
self.trace = trace or self.debug
if self.trace:
self.tracer = Tracer()
else:
self.tracer = None
self.opt_out_cache = opt_out_cache
self.memory_config = memory_config
self._data_storage_dir = None
self.instantiate_entities()
if simulation_json is not None:
warnings.warn(
' '.join([
"The 'simulation_json' argument of the Simulation is deprecated since version 25.0, and will be removed in the future.",
"The proper way to init a simulation from a JSON-like dict is to use SimulationBuilder.build_from_entities. See <https://openfisca.org/doc/openfisca-python-api/simulation_builder.html#openfisca_core.simulation_builder.SimulationBuilder.build_from_dict>"
]), Warning)
from openfisca_core.simulation_builder import SimulationBuilder
SimulationBuilder().build_from_entities(tax_benefit_system,
simulation_json,
simulation=self)
def clone(self, debug = False, trace = False):
"""
Copy the simulation just enough to be able to run the copy without modifying the original simulation
"""
new = empty_clone(self)
new_dict = new.__dict__
for key, value in self.__dict__.items():
if key not in ('debug', 'trace', 'tracer'):
new_dict[key] = value
new.persons = self.persons.clone(new)
setattr(new, new.persons.key, new.persons)
new.entities = {new.persons.key: new.persons}
for entity_class in self.tax_benefit_system.group_entities:
entity = self.entities[entity_class.key].clone(new)
new.entities[entity.key] = entity
setattr(new, entity_class.key, entity) # create shortcut simulation.household (for instance)
if debug:
new_dict['debug'] = True
if trace:
new_dict['trace'] = True
if debug or trace:
if self.debug or self.trace:
new_dict['tracer'] = self.tracer.clone()
else:
new_dict['tracer'] = Tracer()
return new
def test_log_format():
tracer = Tracer()
tracer.record_calculation_start("A", 2017)
tracer.record_calculation_start("B", 2017)
tracer.record_calculation_end("B", 2017, 1)
tracer.record_calculation_end("A", 2017, 2)
lines = tracer.computation_log()
assert_equals(lines[0], ' A<2017> >> 2')
assert_equals(lines[1], ' B<2017> >> 1')
def __init__(
self,
tax_benefit_system,
simulation_json = None,
debug = False,
period = None,
trace = False,
opt_out_cache = False,
memory_config = None,
):
"""
If a ``simulation_json`` is given, initialises a simulation from a JSON dictionary.
Note: This way of initialising a simulation, still under experimentation, aims at replacing the initialisation from `scenario.make_json_or_python_to_attributes`.
If no ``simulation_json`` is given, initialises an empty simulation.
"""
self.tax_benefit_system = tax_benefit_system
assert tax_benefit_system is not None
if period:
assert isinstance(period, periods.Period)
self.period = period
# To keep track of the values (formulas and periods) being calculated to detect circular definitions.
# See use in formulas.py.
# The data structure of requested_periods_by_variable_name is: {variable_name: [period1, period2]}
self.requested_periods_by_variable_name = {}
self.max_nb_cycles = None
self.debug = debug
self.trace = trace or self.debug
if self.trace:
self.tracer = Tracer()
else:
self.tracer = None
self.opt_out_cache = opt_out_cache
self.memory_config = memory_config
self._data_storage_dir = None
self.instantiate_entities(simulation_json)
def test_trace_enums():
tracer = Tracer()
tracer.record_calculation_start("A", 2017)
tracer.record_calculation_end("A", 2017, HousingOccupancyStatus.encode(np.array(['tenant'])))
lines = tracer.computation_log()
assert lines[0] == " A<2017> >> ['tenant']"
def test_variable_stats():
tracer = Tracer()
tracer.record_calculation_start("A", 2017)
tracer.record_calculation_start("B", 2017)
tracer.record_calculation_start("B", 2017)
tracer.record_calculation_start("B", 2016)
assert_equals(tracer.usage_stats['B']['nb_requests'], 3)
assert_equals(tracer.usage_stats['A']['nb_requests'], 1)
assert_equals(tracer.usage_stats['C']['nb_requests'], 0)
def test_log_format():
tracer = Tracer()
tracer.record_calculation_start("A", 2017)
tracer.record_calculation_start("B", 2017)
tracer.record_calculation_end("B", 2017, 1)
tracer.record_calculation_end("A", 2017, 2)
lines = tracer.computation_log()
assert lines[0] == ' A<2017> >> 2'
assert lines[1] == ' B<2017> >> 1'
def test_consistency():
tracer = Tracer()
tracer.record_calculation_start("rsa", 2017)
tracer.record_calculation_end("unkwonn", 2017, 100)
class Simulation(object):
"""
Represents a simulation, and handles the calculation logic
"""
debug = False
period = None
steps_count = 1
tax_benefit_system = None
trace = False
# ----- Simulation construction ----- #
def __init__(
self,
tax_benefit_system,
simulation_json = None,
debug = False,
period = None,
trace = False,
opt_out_cache = False,
memory_config = None,
):
"""
Create an empty simulation
To fill the simulation with input data, you can use the :any:`SimulationBuilder` or proceed manually.
"""
self.tax_benefit_system = tax_benefit_system
assert tax_benefit_system is not None
if period:
assert isinstance(period, periods.Period)
self.period = period
# To keep track of the values (formulas and periods) being calculated to detect circular definitions.
# See use in formulas.py.
# The data structure of requested_periods_by_variable_name is: {variable_name: [period1, period2]}
self.requested_periods_by_variable_name = {}
self.max_nb_cycles = None
self.debug = debug
self.trace = trace or self.debug
if self.trace:
self.tracer = Tracer()
else:
self.tracer = None
self.opt_out_cache = opt_out_cache
self.memory_config = memory_config
self._data_storage_dir = None
self.instantiate_entities()
if simulation_json is not None:
warnings.warn(' '.join([
"The 'simulation_json' argument of the Simulation is deprecated since version 25.0, and will be removed in the future.",
"The proper way to init a simulation from a JSON-like dict is to use SimulationBuilder.build_from_entities. See <https://openfisca.org/doc/openfisca-python-api/simulation_builder.html#openfisca_core.simulation_builder.SimulationBuilder.build_from_dict>"
]),
Warning
)
from openfisca_core.simulation_builder import SimulationBuilder
SimulationBuilder().build_from_entities(tax_benefit_system, simulation_json, simulation = self)
def instantiate_entities(self):
self.persons = self.tax_benefit_system.person_entity(self)
self.entities = {self.persons.key: self.persons}
setattr(self, self.persons.key, self.persons) # create shortcut simulation.person (for instance)
for entity_class in self.tax_benefit_system.group_entities:
entities = entity_class(self)
self.entities[entity_class.key] = entities
setattr(self, entity_class.key, entities) # create shortcut simulation.household (for instance)
@property
def data_storage_dir(self):
"""
Temporary folder used to store intermediate calculation data in case the memory is saturated
"""
if self._data_storage_dir is None:
self._data_storage_dir = tempfile.mkdtemp(prefix = "openfisca_")
log.warn((
"Intermediate results will be stored on disk in {} in case of memory overflow. "
"You should remove this directory once you're done with your simulation."
).format(self._data_storage_dir).encode('utf-8'))
return self._data_storage_dir
# ----- Calculation methods ----- #
def calculate(self, variable_name, period, **parameters):
"""
Calculate the variable ``variable_name`` for the period ``period``, using the variable formula if it exists.
:returns: A numpy array containing the result of the calculation
"""
entity = self.get_variable_entity(variable_name)
holder = entity.get_holder(variable_name)
variable = self.tax_benefit_system.get_variable(variable_name)
if period is not None and not isinstance(period, periods.Period):
period = periods.period(period)
if self.trace:
self.tracer.record_calculation_start(variable.name, period, **parameters)
self._check_period_consistency(period, variable)
extra_params = parameters.get('extra_params', ())
# First look for a value already cached
cached_array = holder.get_array(period, extra_params)
if cached_array is not None:
if self.trace:
self.tracer.record_calculation_end(variable.name, period, cached_array, **parameters)
return cached_array
max_nb_cycles = parameters.get('max_nb_cycles')
if max_nb_cycles is not None:
self.max_nb_cycles = max_nb_cycles
# First, try to run a formula
array = self._run_formula(variable, entity, period, extra_params, max_nb_cycles)
# If no result, try a base function
if array is None and variable.base_function:
array = variable.base_function(holder, period, *extra_params)
# If no result, use the default value
if array is None:
array = holder.default_array()
self._clean_cycle_detection_data(variable.name)
if max_nb_cycles is not None:
self.max_nb_cycles = None
holder.put_in_cache(array, period, extra_params)
if self.trace:
self.tracer.record_calculation_end(variable.name, period, array, **parameters)
return array
def calculate_add(self, variable_name, period, **parameters):
variable = self.tax_benefit_system.get_variable(variable_name)
if period is not None and not isinstance(period, periods.Period):
period = periods.period(period)
# Check that the requested period matches definition_period
if periods.unit_weight(variable.definition_period) > periods.unit_weight(period.unit):
raise ValueError("Unable to compute variable '{0}' for period {1}: '{0}' can only be computed for {2}-long periods. You can use the DIVIDE option to get an estimate of {0} by dividing the yearly value by 12, or change the requested period to 'period.this_year'.".format(
variable.name,
period,
variable.definition_period
).encode('utf-8'))
if variable.definition_period not in [periods.DAY, periods.MONTH, periods.YEAR]:
raise ValueError("Unable to sum constant variable '{}' over period {}: only variables defined daily, monthly, or yearly can be summed over time.".format(
variable.name,
period).encode('utf-8'))
return sum(
self.calculate(variable_name, sub_period, **parameters)
for sub_period in period.get_subperiods(variable.definition_period)
)
def calculate_divide(self, variable_name, period, **parameters):
variable = self.tax_benefit_system.get_variable(variable_name)
if period is not None and not isinstance(period, periods.Period):
period = periods.period(period)
# Check that the requested period matches definition_period
if variable.definition_period != periods.YEAR:
raise ValueError("Unable to divide the value of '{}' over time on period {}: only variables defined yearly can be divided over time.".format(
variable_name,
period).encode('utf-8'))
if period.size != 1:
raise ValueError("DIVIDE option can only be used for a one-year or a one-month requested period")
if period.unit == periods.MONTH:
computation_period = period.this_year
return self.calculate(variable_name, period = computation_period, **parameters) / 12.
elif period.unit == periods.YEAR:
return self.calculate(variable_name, period, **parameters)
raise ValueError("Unable to divide the value of '{}' to match period {}.".format(
variable_name,
period).encode('utf-8'))
def calculate_output(self, variable_name, period):
"""
Calculate the value of a variable using the ``calculate_output`` attribute of the variable.
"""
variable = self.tax_benefit_system.get_variable(variable_name, check_existence = True)
if variable.calculate_output is None:
return self.calculate(variable_name, period)
return variable.calculate_output(self, variable_name, period)
def trace_parameters_at_instant(self, formula_period):
return TracingParameterNodeAtInstant(
self.tax_benefit_system.get_parameters_at_instant(formula_period),
self.tracer
)
def _run_formula(self, variable, entity, period, extra_params, max_nb_cycles):
"""
Find the ``variable`` formula for the given ``period`` if it exists, and apply it to ``entity``.
"""
formula = variable.get_formula(period)
if formula is None:
return None
if self.trace:
parameters_at = self.trace_parameters_at_instant
else:
parameters_at = self.tax_benefit_system.get_parameters_at_instant
try:
self._check_for_cycle(variable, period)
if formula.__code__.co_argcount == 2:
array = formula(entity, period)
else:
array = formula(entity, period, parameters_at, *extra_params)
except CycleError as error:
self._clean_cycle_detection_data(variable.name)
if max_nb_cycles is None:
if self.trace:
self.tracer.record_calculation_abortion(variable.name, period, extra_params = extra_params)
# Re-raise until reaching the first variable called with max_nb_cycles != None in the stack.
raise error
self.max_nb_cycles = None
return None
self._check_formula_result(array, variable, entity, period)
return self._cast_formula_result(array, variable)
def _check_period_consistency(self, period, variable):
"""
Check that a period matches the variable definition_period
"""
if variable.definition_period == periods.ETERNITY:
return # For variables which values are constant in time, all periods are accepted
if variable.definition_period == periods.MONTH and period.unit != periods.MONTH:
raise ValueError("Unable to compute variable '{0}' for period {1}: '{0}' must be computed for a whole month. You can use the ADD option to sum '{0}' over the requested period, or change the requested period to 'period.first_month'.".format(
variable.name,
period
).encode('utf-8'))
if variable.definition_period == periods.YEAR and period.unit != periods.YEAR:
raise ValueError("Unable to compute variable '{0}' for period {1}: '{0}' must be computed for a whole year. You can use the DIVIDE option to get an estimate of {0} by dividing the yearly value by 12, or change the requested period to 'period.this_year'.".format(
variable.name,
period
).encode('utf-8'))
if period.size != 1:
raise ValueError("Unable to compute variable '{0}' for period {1}: '{0}' must be computed for a whole {2}. You can use the ADD option to sum '{0}' over the requested period.".format(
variable.name,
period,
'month' if variable.definition_period == periods.MONTH else 'year'
).encode('utf-8'))
def _check_formula_result(self, value, variable, entity, period):
assert isinstance(value, np.ndarray), (linesep.join([
"You tried to compute the formula '{0}' for the period '{1}'.".format(variable.name, str(period)),
"The formula '{0}@{1}' should return a Numpy array;".format(variable.name, str(period)),
"instead it returned '{0}' of {1}.".format(value, type(value)),
"Learn more about Numpy arrays and vectorial computing:",
"<https://openfisca.org/doc/coding-the-legislation/25_vectorial_computing.html.>"
]))
assert value.size == entity.count, \
"Function {}@{}<{}>() --> <{}>{} returns an array of size {}, but size {} is expected for {}".format(
variable.name, entity.key, str(period), str(period), stringify_array(value),
value.size, entity.count, entity.key).encode('utf-8')
if self.debug:
try:
# cf https://stackoverflow.com/questions/6736590/fast-check-for-nan-in-numpy
if np.isnan(np.min(value)):
nan_count = np.count_nonzero(np.isnan(value))
raise NaNCreationError("Function {}@{}<{}>() --> <{}>{} returns {} NaN value(s)".format(
variable.name, entity.key, str(period), str(period), stringify_array(value),
nan_count).encode('utf-8'))
except TypeError:
pass
def _cast_formula_result(self, value, variable):
if variable.value_type == Enum and not isinstance(value, EnumArray):
return variable.possible_values.encode(value)
if value.dtype != variable.dtype:
return value.astype(variable.dtype)
return value
# ----- Handle circular dependencies in a calculation ----- #
def _check_for_cycle(self, variable, period):
"""
Return a boolean telling if the current variable has already been called without being allowed by
the parameter max_nb_cycles of the calculate method.
"""
def get_error_message():
return "Circular definition detected on formula {}@{}. Formulas and periods involved: {}.".format(
variable.name,
period,
", ".join(sorted(set(
"{}@{}".format(variable_name, period2)
for variable_name, periods in requested_periods_by_variable_name.items()
for period2 in periods
))).encode('utf-8'),
)
requested_periods_by_variable_name = self.requested_periods_by_variable_name
variable_name = variable.name
if variable_name in requested_periods_by_variable_name:
# Make sure the formula doesn't call itself for the same period it is being called for.
# It would be a pure circular definition.
requested_periods = requested_periods_by_variable_name[variable_name]
assert period not in requested_periods and (variable.definition_period != periods.ETERNITY), get_error_message()
if self.max_nb_cycles is None or len(requested_periods) > self.max_nb_cycles:
message = get_error_message()
if self.max_nb_cycles is None:
message += ' Hint: use "max_nb_cycles = 0" to get a default value, or "= N" to allow N cycles.'
raise CycleError(message)
else:
requested_periods.append(period)
else:
requested_periods_by_variable_name[variable_name] = [period]
def _clean_cycle_detection_data(self, variable_name):
"""
When the value of a formula have been computed, remove the period from
requested_periods_by_variable_name[variable_name] and delete the latter if empty.
"""
requested_periods_by_variable_name = self.requested_periods_by_variable_name
if variable_name in requested_periods_by_variable_name:
requested_periods_by_variable_name[variable_name].pop()
if len(requested_periods_by_variable_name[variable_name]) == 0:
del requested_periods_by_variable_name[variable_name]
# ----- Methods to access stored values ----- #
def get_array(self, variable_name, period):
"""
Return the value of ``variable_name`` for ``period``, if this value is alreay in the cache (if it has been set as an input or previously calculated).
Unlike :any:`calculate`, this method *does not* trigger calculations and *does not* use any formula.
"""
if period is not None and not isinstance(period, periods.Period):
period = periods.period(period)
return self.get_holder(variable_name).get_array(period)
def get_holder(self, variable_name):
"""
Get the :any:`Holder` associated with the variable ``variable_name`` for the simulation
"""
return self.get_variable_entity(variable_name).get_holder(variable_name)
def get_memory_usage(self, variables = None):
"""
Get data about the virtual memory usage of the simulation
"""
result = dict(
total_nb_bytes = 0,
by_variable = {}
)
for entity in self.entities.values():
entity_memory_usage = entity.get_memory_usage(variables = variables)
result['total_nb_bytes'] += entity_memory_usage['total_nb_bytes']
result['by_variable'].update(entity_memory_usage['by_variable'])
return result
# ----- Misc ----- #
def delete_arrays(self, variable, period = None):
"""
Delete a variable's value for a given period
:param variable: the variable to be set
:param period: the period for which the value should be deleted
Example:
>>> from openfisca_country_template import CountryTaxBenefitSystem
>>> simulation = Simulation(CountryTaxBenefitSystem())
>>> simulation.set_input('age', '2018-04', [12, 14])
>>> simulation.set_input('age', '2018-05', [13, 14])
>>> simulation.get_array('age', '2018-05')
array([13, 14], dtype=int32)
>>> simulation.delete_arrays('age', '2018-05')
>>> simulation.get_array('age', '2018-04')
array([12, 14], dtype=int32)
>>> simulation.get_array('age', '2018-05') is None
True
>>> simulation.set_input('age', '2018-05', [13, 14])
>>> simulation.delete_arrays('age')
>>> simulation.get_array('age', '2018-04') is None
True
>>> simulation.get_array('age', '2018-05') is None
True
"""
self.get_holder(variable).delete_arrays(period)
def get_known_periods(self, variable):
"""
Get a list variable's known period, i.e. the periods where a value has been initialized and
:param variable: the variable to be set
Example:
>>> from openfisca_country_template import CountryTaxBenefitSystem
>>> simulation = Simulation(CountryTaxBenefitSystem())
>>> simulation.set_input('age', '2018-04', [12, 14])
>>> simulation.set_input('age', '2018-05', [13, 14])
>>> simulation.get_known_periods('age')
[Period((u'month', Instant((2018, 5, 1)), 1)), Period((u'month', Instant((2018, 4, 1)), 1))]
"""
return self.get_holder(variable).get_known_periods()
def set_input(self, variable, period, value):
"""
Set a variable's value for a given period
:param variable: the variable to be set
:param value: the input value for the variable
:param period: the period for which the value is setted
Example:
>>> from openfisca_country_template import CountryTaxBenefitSystem
>>> simulation = Simulation(CountryTaxBenefitSystem())
>>> simulation.set_input('age', '2018-04', [12, 14])
>>> simulation.get_array('age', '2018-04')
array([12, 14], dtype=int32)
If a ``set_input`` property has been set for the variable, this method may accept inputs for periods not matching the ``definition_period`` of the variable. To read more about this, check the `documentation <https://openfisca.org/doc/coding-the-legislation/35_periods.html#automatically-process-variable-inputs-defined-for-periods-not-matching-the-definitionperiod>`_.
"""
self.get_holder(variable).set_input(period, value)
def get_variable_entity(self, variable_name):
variable = self.tax_benefit_system.get_variable(variable_name, check_existence = True)
return self.get_entity(variable.entity)
def get_entity(self, entity_type = None, plural = None):
if entity_type:
return self.entities[entity_type.key]
if plural:
return next((entity for entity in self.entities.values() if entity.plural == plural), None)
def clone(self, debug = False, trace = False):
"""
Copy the simulation just enough to be able to run the copy without modifying the original simulation
"""
new = empty_clone(self)
new_dict = new.__dict__
for key, value in self.__dict__.items():
if key not in ('debug', 'trace', 'tracer'):
new_dict[key] = value
new.persons = self.persons.clone(new)
setattr(new, new.persons.key, new.persons)
new.entities = {new.persons.key: new.persons}
for entity_class in self.tax_benefit_system.group_entities:
entity = self.entities[entity_class.key].clone(new)
new.entities[entity.key] = entity
setattr(new, entity_class.key, entity) # create shortcut simulation.household (for instance)
if debug:
new_dict['debug'] = True
if trace:
new_dict['trace'] = True
if debug or trace:
if self.debug or self.trace:
new_dict['tracer'] = self.tracer.clone()
else:
new_dict['tracer'] = Tracer()
return new
def check_tracing_params(accessor, param_key):
tracer = Tracer()
tracer.record_calculation_start('A', '2015-01')
tracingParams = TracingParameterNodeAtInstant(parameters('2015-01-01'), tracer)
param = accessor(tracingParams)
assert_near(tracer.trace['A<2015-01>']['parameters'][param_key], param)
def test_consistency():
with pytest.raises(ValueError):
tracer = Tracer()
tracer.record_calculation_start("rsa", 2017)
tracer.record_calculation_end("unkwonn", 2017, 100)
# -*- coding: utf-8 -*-
from nose.tools import raises
from openfisca_core.tracers import Tracer
tracer = Tracer()
@raises(ValueError)
def test_consistency():
tracer.record_calculation_start("rsa", "2016-01")
tracer.record_calculation_end("unkwonn", "2016-01", 100)
def trace(self, trace):
self._trace = trace
if trace:
self.tracer = Tracer()
else:
self.tracer = None
class Simulation(object):
"""
Represents a simulation, and handles the calculation logic
"""
def __init__(
self,
tax_benefit_system,
populations
):
"""
This constructor is reserved for internal use; see :any:`SimulationBuilder`,
which is the preferred way to obtain a Simulation initialized with a consistent
set of Entities.
"""
self.tax_benefit_system = tax_benefit_system
assert tax_benefit_system is not None
self.populations = populations
self.persons = self.populations[tax_benefit_system.person_entity.key]
self.link_to_entities_instances()
self.create_shortcuts()
# To keep track of the values (formulas and periods) being calculated to detect circular definitions.
# The data structure of computation_stack is:
# [('variable_name', 'period1'), ('variable_name', 'period2')]
self.computation_stack = []
self.invalidated_caches = set()
self.debug = False
self.trace = False
self.opt_out_cache = False
# controls the spirals detection; check for performance impact if > 1
self.max_spiral_loops = 1
self.memory_config = None
self._data_storage_dir = None
@property
def trace(self):
return self._trace
@trace.setter
def trace(self, trace):
self._trace = trace
if trace:
self.tracer = Tracer()
else:
self.tracer = None
def link_to_entities_instances(self):
for _key, entity_instance in self.populations.items():
entity_instance.simulation = self
def create_shortcuts(self):
for _key, population in self.populations.items():
# create shortcut simulation.person and simulation.household (for instance)
setattr(self, population.entity.key, population)
@property
def data_storage_dir(self):
"""
Temporary folder used to store intermediate calculation data in case the memory is saturated
"""
if self._data_storage_dir is None:
self._data_storage_dir = tempfile.mkdtemp(prefix = "openfisca_")
log.warn((
"Intermediate results will be stored on disk in {} in case of memory overflow. "
"You should remove this directory once you're done with your simulation."
).format(self._data_storage_dir))
return self._data_storage_dir
# ----- Calculation methods ----- #
def calculate(self, variable_name, period, **parameters):
"""
Calculate the variable ``variable_name`` for the period ``period``, using the variable formula if it exists.
:returns: A numpy array containing the result of the calculation
"""
population = self.get_variable_population(variable_name)
holder = population.get_holder(variable_name)
variable = self.tax_benefit_system.get_variable(variable_name, check_existence = True)
if period is not None and not isinstance(period, periods.Period):
period = periods.period(period)
if self.trace:
self.tracer.record_calculation_start(variable.name, period, **parameters)
self._check_period_consistency(period, variable)
# First look for a value already cached
cached_array = holder.get_array(period)
if cached_array is not None:
if self.trace:
self.tracer.record_calculation_end(variable.name, period, cached_array, **parameters)
return cached_array
array = None
# First, try to run a formula
try:
self._check_for_cycle(variable, period)
array = self._run_formula(variable, population, period)
# If no result, use the default value and cache it
if array is None:
array = holder.default_array()
array = self._cast_formula_result(array, variable)
holder.put_in_cache(array, period)
except SpiralError:
array = holder.default_array()
finally:
if self.trace:
self.tracer.record_calculation_end(variable.name, period, array, **parameters)
self._clean_cycle_detection_data(variable.name)
self.purge_cache_of_invalid_values()
return array
def purge_cache_of_invalid_values(self):
# We wait for the end of calculate(), signalled by an empty stack, before purging the cache
if self.computation_stack:
return
for (_name, _period) in self.invalidated_caches:
holder = self.get_holder(_name)
holder.delete_arrays(_period)
def calculate_add(self, variable_name, period, **parameters):
variable = self.tax_benefit_system.get_variable(variable_name, check_existence = True)
if period is not None and not isinstance(period, periods.Period):
period = periods.period(period)
# Check that the requested period matches definition_period
if periods.unit_weight(variable.definition_period) > periods.unit_weight(period.unit):
raise ValueError("Unable to compute variable '{0}' for period {1}: '{0}' can only be computed for {2}-long periods. You can use the DIVIDE option to get an estimate of {0} by dividing the yearly value by 12, or change the requested period to 'period.this_year'.".format(
variable.name,
period,
variable.definition_period
))
if variable.definition_period not in [periods.DAY, periods.MONTH, periods.YEAR]:
raise ValueError("Unable to sum constant variable '{}' over period {}: only variables defined daily, monthly, or yearly can be summed over time.".format(
variable.name,
period))
return sum(
self.calculate(variable_name, sub_period, **parameters)
for sub_period in period.get_subperiods(variable.definition_period)
)
def calculate_divide(self, variable_name, period, **parameters):
variable = self.tax_benefit_system.get_variable(variable_name, check_existence = True)
if period is not None and not isinstance(period, periods.Period):
period = periods.period(period)
# Check that the requested period matches definition_period
if variable.definition_period != periods.YEAR:
raise ValueError("Unable to divide the value of '{}' over time on period {}: only variables defined yearly can be divided over time.".format(
variable_name,
period))
if period.size != 1:
raise ValueError("DIVIDE option can only be used for a one-year or a one-month requested period")
if period.unit == periods.MONTH:
computation_period = period.this_year
return self.calculate(variable_name, period = computation_period, **parameters) / 12.
elif period.unit == periods.YEAR:
return self.calculate(variable_name, period, **parameters)
raise ValueError("Unable to divide the value of '{}' to match period {}.".format(
variable_name,
period))
def calculate_output(self, variable_name, period):
"""
Calculate the value of a variable using the ``calculate_output`` attribute of the variable.
"""
variable = self.tax_benefit_system.get_variable(variable_name, check_existence = True)
if variable.calculate_output is None:
return self.calculate(variable_name, period)
return variable.calculate_output(self, variable_name, period)
def trace_parameters_at_instant(self, formula_period):
return TracingParameterNodeAtInstant(
self.tax_benefit_system.get_parameters_at_instant(formula_period),
self.tracer
)
def _run_formula(self, variable, population, period):
"""
Find the ``variable`` formula for the given ``period`` if it exists, and apply it to ``population``.
"""
formula = variable.get_formula(period)
if formula is None:
return None
if self.trace:
parameters_at = self.trace_parameters_at_instant
else:
parameters_at = self.tax_benefit_system.get_parameters_at_instant
if formula.__code__.co_argcount == 2:
array = formula(population, period)
else:
array = formula(population, period, parameters_at)
return array
def _check_period_consistency(self, period, variable):
"""
Check that a period matches the variable definition_period
"""
if variable.definition_period == periods.ETERNITY:
return # For variables which values are constant in time, all periods are accepted
if variable.definition_period == periods.MONTH and period.unit != periods.MONTH:
raise ValueError("Unable to compute variable '{0}' for period {1}: '{0}' must be computed for a whole month. You can use the ADD option to sum '{0}' over the requested period, or change the requested period to 'period.first_month'.".format(
variable.name,
period
))
if variable.definition_period == periods.YEAR and period.unit != periods.YEAR:
raise ValueError("Unable to compute variable '{0}' for period {1}: '{0}' must be computed for a whole year. You can use the DIVIDE option to get an estimate of {0} by dividing the yearly value by 12, or change the requested period to 'period.this_year'.".format(
variable.name,
period
))
if period.size != 1:
raise ValueError("Unable to compute variable '{0}' for period {1}: '{0}' must be computed for a whole {2}. You can use the ADD option to sum '{0}' over the requested period.".format(
variable.name,
period,
'month' if variable.definition_period == periods.MONTH else 'year'
))
def _cast_formula_result(self, value, variable):
if variable.value_type == Enum and not isinstance(value, EnumArray):
return variable.possible_values.encode(value)
if not isinstance(value, np.ndarray):
population = self.get_variable_population(variable.name)
value = population.filled_array(value)
if value.dtype != variable.dtype:
return value.astype(variable.dtype)
return value
# ----- Handle circular dependencies in a calculation ----- #
def _check_for_cycle(self, variable, period):
"""
Raise an exception in the case of a circular definition, where evaluating a variable for
a given period loops around to evaluating the same variable/period pair. Also guards, as
a heuristic, against "quasicircles", where the evaluation of a variable at a period involves
the same variable at a different period.
"""
previous_periods = [_period for (_name, _period) in self.computation_stack if _name == variable.name]
self.computation_stack.append((variable.name, str(period)))
if str(period) in previous_periods:
raise CycleError("Circular definition detected on formula {}@{}".format(variable.name, period))
spiral = len(previous_periods) >= self.max_spiral_loops
if spiral:
self.invalidate_spiral_variables(variable)
message = "Quasicircular definition detected on formula {}@{} involving {}".format(variable.name, period, self.computation_stack)
raise SpiralError(message, variable.name)
def invalidate_spiral_variables(self, variable):
# Visit the stack, from the bottom (most recent) up; we know that we'll find
# the variable implicated in the spiral (max_spiral_loops+1) times; we keep the
# intermediate values computed (to avoid impacting performance) but we mark them
# for deletion from the cache once the calculation ends.
count = 0
for frame in reversed(self.computation_stack):
self.invalidated_caches.add(frame)
if frame[0] == variable.name:
count += 1
if count > self.max_spiral_loops:
break
def _clean_cycle_detection_data(self, variable_name):
"""
When the value of a formula have been computed, remove the period from
requested_periods_by_variable_name[variable_name] and delete the latter if empty.
"""
self.computation_stack.pop()
# ----- Methods to access stored values ----- #
def get_array(self, variable_name, period):
"""
Return the value of ``variable_name`` for ``period``, if this value is alreay in the cache (if it has been set as an input or previously calculated).
Unlike :any:`calculate`, this method *does not* trigger calculations and *does not* use any formula.
"""
if period is not None and not isinstance(period, periods.Period):
period = periods.period(period)
return self.get_holder(variable_name).get_array(period)
def get_holder(self, variable_name):
"""
Get the :any:`Holder` associated with the variable ``variable_name`` for the simulation
"""
return self.get_variable_population(variable_name).get_holder(variable_name)
def get_memory_usage(self, variables = None):
"""
Get data about the virtual memory usage of the simulation
"""
result = dict(
total_nb_bytes = 0,
by_variable = {}
)
for entity in self.populations.values():
entity_memory_usage = entity.get_memory_usage(variables = variables)
result['total_nb_bytes'] += entity_memory_usage['total_nb_bytes']
result['by_variable'].update(entity_memory_usage['by_variable'])
return result
# ----- Misc ----- #
def delete_arrays(self, variable, period = None):
"""
Delete a variable's value for a given period
:param variable: the variable to be set
:param period: the period for which the value should be deleted
Example:
>>> from openfisca_country_template import CountryTaxBenefitSystem
>>> simulation = Simulation(CountryTaxBenefitSystem())
>>> simulation.set_input('age', '2018-04', [12, 14])
>>> simulation.set_input('age', '2018-05', [13, 14])
>>> simulation.get_array('age', '2018-05')
array([13, 14], dtype=int32)
>>> simulation.delete_arrays('age', '2018-05')
>>> simulation.get_array('age', '2018-04')
array([12, 14], dtype=int32)
>>> simulation.get_array('age', '2018-05') is None
True
>>> simulation.set_input('age', '2018-05', [13, 14])
>>> simulation.delete_arrays('age')
>>> simulation.get_array('age', '2018-04') is None
True
>>> simulation.get_array('age', '2018-05') is None
True
"""
self.get_holder(variable).delete_arrays(period)
def get_known_periods(self, variable):
"""
Get a list variable's known period, i.e. the periods where a value has been initialized and
:param variable: the variable to be set
Example:
>>> from openfisca_country_template import CountryTaxBenefitSystem
>>> simulation = Simulation(CountryTaxBenefitSystem())
>>> simulation.set_input('age', '2018-04', [12, 14])
>>> simulation.set_input('age', '2018-05', [13, 14])
>>> simulation.get_known_periods('age')
[Period((u'month', Instant((2018, 5, 1)), 1)), Period((u'month', Instant((2018, 4, 1)), 1))]
"""
return self.get_holder(variable).get_known_periods()
def set_input(self, variable_name, period, value):
"""
Set a variable's value for a given period
:param variable: the variable to be set
:param value: the input value for the variable
:param period: the period for which the value is setted
Example:
>>> from openfisca_country_template import CountryTaxBenefitSystem
>>> simulation = Simulation(CountryTaxBenefitSystem())
>>> simulation.set_input('age', '2018-04', [12, 14])
>>> simulation.get_array('age', '2018-04')
array([12, 14], dtype=int32)
If a ``set_input`` property has been set for the variable, this method may accept inputs for periods not matching the ``definition_period`` of the variable. To read more about this, check the `documentation <https://openfisca.org/doc/coding-the-legislation/35_periods.html#automatically-process-variable-inputs-defined-for-periods-not-matching-the-definitionperiod>`_.
"""
variable = self.tax_benefit_system.get_variable(variable_name, check_existence = True)
period = periods.period(period)
if ((variable.end is not None) and (period.start.date > variable.end)):
return
self.get_holder(variable_name).set_input(period, value)
def get_variable_population(self, variable_name):
variable = self.tax_benefit_system.get_variable(variable_name, check_existence = True)
return self.populations[variable.entity.key]
def get_population(self, plural = None):
return next((population for population in self.populations.values() if population.entity.plural == plural), None)
def get_entity(self, plural = None):
population = self.get_population(plural)
return population and population.entity
def describe_entities(self):
return {population.entity.plural: population.ids for population in self.populations.values()}
def clone(self, debug = False, trace = False):
"""
Copy the simulation just enough to be able to run the copy without modifying the original simulation
"""
new = empty_clone(self)
new_dict = new.__dict__
for key, value in self.__dict__.items():
if key not in ('debug', 'trace', 'tracer'):
new_dict[key] = value
new.persons = self.persons.clone(new)
setattr(new, new.persons.entity.key, new.persons)
new.populations = {new.persons.entity.key: new.persons}
for entity in self.tax_benefit_system.group_entities:
population = self.populations[entity.key].clone(new)
new.populations[entity.key] = population
setattr(new, entity.key, population) # create shortcut simulation.household (for instance)
new.debug = debug
new.trace = trace
return new