def validate_slice_json(slice, state=None):
if slice is None:
return None, None
state = conv.add_ancestor_to_state(state, slice)
validated_slice, errors = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
base=validate_values_holder_json,
comment=conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
rate=conv.pipe(
validate_values_holder_json,
conv.not_none,
),
threshold=conv.pipe(
validate_values_holder_json,
conv.not_none,
),
),
constructor=collections.OrderedDict,
drop_none_values='missing',
keep_value_order=True,
),
)(slice, state=state)
conv.remove_ancestor_from_state(state, slice)
return validated_slice, errors
def validate_dated_legislation_json(dated_legislation_json, state = None):
if dated_legislation_json is None:
return None, None
if state is None:
state = conv.default_state
dated_legislation_json, error = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
datesim = conv.pipe(
conv.test_isinstance(basestring),
conv.iso8601_input_to_date,
conv.date_to_iso8601_str,
conv.not_none,
),
),
constructor = collections.OrderedDict,
default = conv.noop,
drop_none_values = 'missing',
keep_value_order = True,
),
)(dated_legislation_json, state = state)
if error is not None:
return dated_legislation_json, error
datesim = dated_legislation_json.pop('datesim')
dated_legislation_json, error = validate_dated_node_json(dated_legislation_json, state = state)
dated_legislation_json['datesim'] = datesim
return dated_legislation_json, error
def input_to_json_data(value, state=None):
return conv.pipe(
conv.make_input_to_json(),
conv.test_isinstance(dict),
conv.struct(
dict(
resultat_officiel=conv.pipe(
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(
conv.test_isinstance(basestring),
conv.not_none,
),
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
code=conv.pipe(
conv.test_isinstance(basestring),
conv.not_none,
),
name=conv.pipe(
conv.test_isinstance(basestring),
conv.not_none,
),
value=conv.pipe(
conv.test_isinstance(float),
conv.not_none,
),
), ),
conv.not_none,
),
),
conv.not_none,
),
scenario=conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
test_case=conv.pipe(
conv.test_isinstance(dict),
# For each entity convert its members from a dict to a list.
conv.uniform_mapping(
conv.noop,
conv.pipe(
conv.test_isinstance(dict),
conv.function(
transform_entity_member_by_id_to_members
),
),
),
), ),
default=conv.noop,
),
tax_benefit_system.Scenario.make_json_to_instance(
tax_benefit_system=tax_benefit_system),
conv.not_none,
),
), ),
)(value, state=state or conv.default_state)
def validate_dated_legislation_json(dated_legislation_json, state=None):
if dated_legislation_json is None:
return None, None
if state is None:
state = conv.default_state
dated_legislation_json, error = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(datesim=conv.pipe(
conv.test_isinstance(basestring),
conv.iso8601_input_to_date,
conv.date_to_iso8601_str,
conv.not_none,
), ),
constructor=collections.OrderedDict,
default=conv.noop,
drop_none_values='missing',
keep_value_order=True,
),
)(dated_legislation_json, state=state)
if error is not None:
return dated_legislation_json, error
datesim = dated_legislation_json.pop('datesim')
dated_legislation_json, error = validate_dated_node_json(
dated_legislation_json, state=state)
dated_legislation_json['datesim'] = datesim
return dated_legislation_json, error
def validate_slice_json(slice, state = None):
if slice is None:
return None, None
state = conv.add_ancestor_to_state(state, slice)
validated_slice, errors = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
base = validate_values_holder_json,
comment = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
rate = conv.pipe(
validate_values_holder_json,
conv.not_none,
),
threshold = conv.pipe(
validate_values_holder_json,
conv.not_none,
),
),
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
),
)(slice, state = state)
conv.remove_ancestor_from_state(state, slice)
return validated_slice, errors
def json_to_dated_python(self):
return conv.pipe(
conv.condition(
conv.test_isinstance((float, int)),
# YAML stores strings containing only digits as numbers.
conv.function(str),
),
conv.test_isinstance(basestring_type),
)
def json_to_dated_python(self):
return conv.pipe(
conv.condition(
conv.test_isinstance((float, int)),
# YAML stores strings containing only digits as numbers.
conv.function(str),
),
conv.test_isinstance(basestring_type),
conv.test(lambda value: len(value) <= self.variable.max_length),
)
def validate_parameter_xml_json(parameter, state = None):
if parameter is None:
return None, None
state = conv.add_ancestor_to_state(state, parameter)
validated_parameter, errors = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
code = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
conv.not_none,
),
description = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
),
format = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in(xml_json_formats),
),
tail = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
taille = conv.pipe(
conv.test_isinstance(basestring),
conv.test_in([
'moinsde20',
'plusde20',
]),
),
text = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
type = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in(json_unit_by_xml_json_type),
),
VALUE = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_value_xml_json,
drop_none_items = True,
),
make_validate_values_xml_json_dates(require_consecutive_dates = True),
conv.empty_to_none,
conv.not_none,
),
),
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
),
)(parameter, state = state)
conv.remove_ancestor_from_state(state, parameter)
return validated_parameter, errors
def validate_node_json(node, state=None):
if node is None:
return None, None
if state is None:
state = conv.default_state
validated_node, errors = conv.pipe(
conv.condition(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_node_json,
drop_none_items=True,
),
conv.pipe(
conv.condition(
conv.test_isinstance(basestring_type),
conv.function(lambda code: dict(code=code)),
conv.test_isinstance(dict),
),
conv.struct(
dict(
children=conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_node_json,
drop_none_items=True,
),
conv.empty_to_none,
),
code=conv.pipe(
conv.test_isinstance(basestring_type),
conv.cleanup_line,
),
),
constructor=collections.OrderedDict,
default=conv.noop,
drop_none_values='missing',
keep_value_order=True,
),
),
),
conv.empty_to_none,
)(node, state=state)
if validated_node is None or errors is not None:
return validated_node, errors
if isinstance(validated_node,
dict) and not validated_node.get('children'):
validated_node, errors = conv.struct(
dict(code=conv.pipe(
conv.test_in(tax_benefit_system.variables),
conv.not_none,
), ),
default=conv.noop,
)(validated_node, state=state)
return validated_node, errors
def json_or_python_to_input_variables(value, state=None):
if value is None:
return value, None
if state is None:
state = conv.default_state
input_variables, errors = conv.pipe(
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(
conv.test_isinstance(basestring_type),
conv.not_none,
),
conv.noop,
),
)(value, state=state)
if errors is not None:
return input_variables, errors
for input_variable_name in input_variables.keys():
if input_variable_name not in variables_name:
# We only import VariableNotFound here to avoid a circular dependency in imports
from .taxbenefitsystems import VariableNotFound
raise VariableNotFound(input_variable_name, tax_benefit_system)
input_variables, errors = conv.pipe(
make_json_or_python_to_array_by_period_by_variable_name(
tax_benefit_system, period),
conv.empty_to_none,
)(value, state=state)
if errors is not None:
return input_variables, errors
if input_variables is None:
return input_variables, None
count_by_entity_key = {}
errors = {}
for variable_name, array_by_period in input_variables.items():
column = tax_benefit_system.get_variable(variable_name)
entity_key = column.entity.key
entity_count = count_by_entity_key.get(entity_key, 0)
for variable_period, variable_array in array_by_period.items():
if entity_count == 0:
count_by_entity_key[entity_key] = entity_count = len(
variable_array)
elif len(variable_array) != entity_count:
errors[column.name] = state._(
"Array has not the same length as other variables of entity {}: {} instead of {}"
).format(column.name, len(variable_array), entity_count)
return input_variables, errors or None
def json_to_dated_python(self):
enum = self.variable.possible_values
possible_names = [item.name for item in list(enum)]
if enum is None:
return conv.pipe(conv.test_isinstance(basestring_type))
return conv.pipe(
conv.test_isinstance(basestring_type),
conv.pipe(
# Verify that item belongs to enumeration.
conv.test_in(possible_names),
# Transform that item into enum object.
conv.function(lambda enum_name: enum[enum_name])))
def validate_dated_value_json(value, state=None):
if value is None:
return None, None
container = state.ancestors[-1]
value_converter = dict(
boolean=conv.condition(
conv.test_isinstance(int),
conv.test_in((0, 1)),
conv.test_isinstance(bool),
),
float=conv.condition(
conv.test_isinstance(int),
conv.anything_to_float,
conv.test_isinstance(float),
),
integer=conv.condition(
conv.test_isinstance(float),
conv.pipe(
conv.test(lambda number: round(number) == number),
conv.function(int),
),
conv.test_isinstance(int),
),
rate=conv.condition(
conv.test_isinstance(int),
conv.anything_to_float,
conv.test_isinstance(float),
),
)[container.get('format') or 'float'] # Only parameters have a "format".
return value_converter(value, state=state or conv.default_state)
def check_entities_and_role(test_case, tax_benefit_system, state):
"""
Check that the test_case describes entities consistent with the tax and benefit system.
Will raise an error if :
- An entity is not recognized
- An entity role is not recognized
- A variable is declared for an entity it is not defined for (e.g. salary for a family)
"""
test_case = deepcopy(
test_case) # Avoid side-effects on other references to test_case
entity_classes = {
entity_class.plural: entity_class
for entity_class in tax_benefit_system.entities
}
for entity_type_name, entities in test_case.items():
if entity_classes.get(entity_type_name) is None:
raise ValueError("Invalid entity name: {}".format(
entity_type_name).encode('utf-8'))
entities, error = conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items=True,
),
conv.function(set_entities_json_id),
)(entities)
if error is not None:
raise ValueError("Invalid list of {}: {}. Error: {}".format(
entity_type_name, entities, error).encode('utf-8'))
if entities is None:
entities = test_case[entity_type_name] = [
] # YAML test runner may set these values to None
entity_class = entity_classes[entity_type_name]
valid_roles = dict(
(role.key, role) if (role.max == 1) else (role.plural, role) for
role in entity_class.roles) if not entity_class.is_person else {}
for entity_json in entities:
check_entity_fields(entity_json, entity_class, valid_roles,
tax_benefit_system)
for entity_class in entity_classes.values():
if test_case.get(entity_class.plural) is None:
test_case[entity_class.plural] = [
] # by convention, all entities must be declared in the test_case
return test_case
def validate_dated_value_json(value, state = None):
if value is None:
return None, None
container = state.ancestors[-1]
value_converter = dict(
boolean = conv.condition(
conv.test_isinstance(int),
conv.test_in((0, 1)),
conv.test_isinstance(bool),
),
float = conv.condition(
conv.test_isinstance(int),
conv.anything_to_float,
conv.test_isinstance(float),
),
integer = conv.condition(
conv.test_isinstance(float),
conv.pipe(
conv.test(lambda number: round(number) == number),
conv.function(int),
),
conv.test_isinstance(int),
),
rate = conv.condition(
conv.test_isinstance(int),
conv.anything_to_float,
conv.test_isinstance(float),
),
)[container.get('format') or 'float'] # Only parameters have a "format".
return value_converter(value, state = state or conv.default_state)
def validate_scale_xml_json(scale, state = None):
if scale is None:
return None, None
state = conv.add_ancestor_to_state(state, scale)
validated_scale, errors = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
code = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
conv.not_none,
),
description = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
),
option = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in((
'contrib',
'main-d-oeuvre',
'noncontrib',
)),
),
TRANCHE = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_slice_xml_json,
drop_none_items = True,
),
validate_slices_xml_json_dates,
conv.empty_to_none,
conv.not_none,
),
tail = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
text = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
type = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in((
'monetary',
)),
),
),
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
),
)(scale, state = state)
conv.remove_ancestor_from_state(state, scale)
return validated_scale, errors
def input_to_json_data(value, state=None):
return conv.pipe(
conv.make_input_to_json(),
conv.test_isinstance(dict),
conv.struct(
dict(
resultat_officiel=conv.pipe(
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(conv.test_isinstance(basestring), conv.not_none),
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
code=conv.pipe(conv.test_isinstance(basestring), conv.not_none),
name=conv.pipe(conv.test_isinstance(basestring), conv.not_none),
value=conv.pipe(conv.test_isinstance(float), conv.not_none),
)
),
conv.not_none,
),
),
conv.not_none,
),
scenario=conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
test_case=conv.pipe(
conv.test_isinstance(dict),
# For each entity convert its members from a dict to a list.
conv.uniform_mapping(
conv.noop,
conv.pipe(
conv.test_isinstance(dict),
conv.function(transform_entity_member_by_id_to_members),
),
),
)
),
default=conv.noop,
),
tax_benefit_system.Scenario.make_json_to_instance(tax_benefit_system=tax_benefit_system),
conv.not_none,
),
)
),
)(value, state=state or conv.default_state)
def check_role(value, key):
role = valid_roles.get(key)
if role.max == 1:
value, error = conv.test_isinstance((basestring_type, int))(value)
else:
value, error = conv.pipe(
conv.make_item_to_singleton(), conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring_type, int)),
drop_none_items=True,
))(value)
if error is not None:
raise ValueError("Invalid description of {}: {}. Error: {}".format(
entity_class.key, entity_json, error).encode('utf-8'))
entity_json[key] = value
def make_json_to_array_by_period(self, period):
return conv.condition(
conv.test_isinstance(dict),
conv.pipe(
# Value is a dict of (period, value) couples.
conv.uniform_mapping(
conv.pipe(
conv.function(periods.period),
conv.not_none,
),
conv.pipe(
conv.make_item_to_singleton(),
conv.uniform_sequence(self.json_to_dated_python, ),
conv.empty_to_none,
conv.function(lambda cells_list: np.array(
cells_list, dtype=self.dtype)),
),
drop_none_values=True,
),
conv.empty_to_none,
),
conv.pipe(
conv.make_item_to_singleton(),
conv.uniform_sequence(self.json_to_dated_python, ),
conv.empty_to_none,
conv.function(
lambda cells_list: np.array(cells_list, dtype=self.dtype)),
conv.function(lambda array: {period: array}),
),
)
def input_to_dated_python(self):
enum = self.variable.possible_values
if enum is None:
return conv.test_isinstance(basestring_type)
return conv.pipe(
# Verify that item index belongs to enumeration.
conv.test_in([item.name for item in list(enum)]))
def json_or_python_to_test(value, state=None):
if value is None:
return value, None
if state is None:
state = conv.default_state
value, error = conv.pipe(
conv.test_isinstance(dict),
validate,
)(value, state=state)
if error is not None:
return value, error
value, error = conv.struct(
dict(output_variables=make_json_or_python_to_input_variables(
tax_benefit_system, value['period']), ),
default=conv.noop,
)(value, state=state)
if error is not None:
return value, error
test_case = value.copy()
absolute_error_margin = test_case.pop('absolute_error_margin')
axes = test_case.pop('axes')
description = test_case.pop('description')
input_variables = test_case.pop('input_variables')
keywords = test_case.pop('keywords')
name = test_case.pop('name')
output_variables = test_case.pop('output_variables')
period = test_case.pop('period')
relative_error_margin = test_case.pop('relative_error_margin')
if test_case is not None and all(
entity_members is None
for entity_members in test_case.values()):
test_case = None
scenario, error = tax_benefit_system.Scenario.make_json_to_instance(
repair=True, tax_benefit_system=tax_benefit_system)(dict(
axes=axes,
input_variables=input_variables,
period=period,
test_case=test_case,
),
state=state)
if error is not None:
return scenario, error
return {
key: value
for key, value in dict(
absolute_error_margin=absolute_error_margin,
description=description,
keywords=keywords,
name=name,
output_variables=output_variables,
relative_error_margin=relative_error_margin,
scenario=scenario,
).items() if value is not None
}, None
def json_to_dated_python(self):
return conv.pipe(
conv.condition(
conv.test_isinstance(datetime.date),
conv.noop,
conv.condition(
conv.test_isinstance(int),
conv.pipe(
conv.test_between(1870, 2099),
conv.function(lambda year: datetime.date(year, 1, 1)),
),
conv.pipe(
conv.test_isinstance(basestring_type),
conv.test(year_or_month_or_day_re.match, error = N_('Invalid date')),
conv.function(lambda birth: '-'.join((birth.split('-') + ['01', '01'])[:3])),
conv.iso8601_input_to_date,
),
),
),
conv.test_between(datetime.date(1870, 1, 1), datetime.date(2099, 12, 31)),
)
def json_to_python(self):
return conv.condition(
conv.test_isinstance(dict),
conv.pipe(
# Value is a dict of (period, value) couples.
conv.uniform_mapping(
conv.pipe(
conv.function(periods.period),
conv.not_none,
),
self.json_to_dated_python,
), ),
self.json_to_dated_python,
)
def validate_dated_slice_json(slice, state=None):
if slice is None:
return None, None
state = conv.add_ancestor_to_state(state, slice)
validated_slice, errors = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
base=conv.pipe(
validate_dated_value_json,
conv.test_greater_or_equal(0),
),
constant_amount=validate_dated_value_json,
comment=conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
rate=conv.pipe(
validate_dated_value_json,
conv.test_between(0, 1),
),
threshold=conv.pipe(
validate_dated_value_json,
conv.test_greater_or_equal(0),
),
date=conv.pipe(
validate_dated_value_json,
conv.test_between(0, 1), #TODO: changer
),
),
constructor=collections.OrderedDict,
drop_none_values='missing',
keep_value_order=True,
),
)(slice, state=state)
conv.remove_ancestor_from_state(state, slice)
return validated_slice, errors
def validate_dated_slice_json(slice, state = None):
if slice is None:
return None, None
state = conv.add_ancestor_to_state(state, slice)
validated_slice, errors = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
base = conv.pipe(
validate_dated_value_json,
conv.test_greater_or_equal(0),
),
constant_amount = validate_dated_value_json,
comment = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
rate = conv.pipe(
validate_dated_value_json,
conv.test_between(0, 1),
),
threshold = conv.pipe(
validate_dated_value_json,
conv.test_greater_or_equal(0),
),
date = conv.pipe(
validate_dated_value_json,
conv.test_between(0, 1), #TODO: changer
),
),
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
),
)(slice, state = state)
conv.remove_ancestor_from_state(state, slice)
return validated_slice, errors
def make_json_or_python_to_axes(tax_benefit_system):
column_by_name = tax_benefit_system.variables
return conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.pipe(
conv.item_or_sequence(
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
count=conv.pipe(
conv.test_isinstance(int),
conv.test_greater_or_equal(1),
conv.not_none,
),
index=conv.pipe(
conv.test_isinstance(int),
conv.test_greater_or_equal(0),
conv.default(0),
),
max=conv.pipe(
conv.test_isinstance((float, int)),
conv.not_none,
),
min=conv.pipe(
conv.test_isinstance((float, int)),
conv.not_none,
),
name=conv.pipe(
conv.test_isinstance(basestring_type),
conv.test_in(column_by_name),
conv.test(
lambda column_name: tax_benefit_system.
get_variable(column_name).dtype in
(np.float32, np.int16, np.int32),
error=N_(
'Invalid type for axe: integer or float expected'
)),
conv.not_none,
),
period=conv.function(periods.period),
), ),
),
drop_none_items=True,
),
conv.make_item_to_singleton(),
),
drop_none_items=True,
),
conv.empty_to_none,
)
conv.not_none,
),
),
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
),
)(value, state = state)
conv.remove_ancestor_from_state(state, value)
return validated_value, errors
validate_values_holder_xml_json = conv.struct(
dict(
VALUE = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_value_xml_json,
drop_none_items = True,
),
make_validate_values_xml_json_dates(require_consecutive_dates = False),
conv.empty_to_none,
conv.not_none,
),
),
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
)
def validate_node_xml_json(node, state = None):
if node is None:
return None, None
state = conv.add_ancestor_to_state(state, node)
validated_node, errors = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
BAREME = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_scale_xml_json,
drop_none_items = True,
),
conv.empty_to_none,
),
CODE = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_parameter_xml_json,
drop_none_items = True,
),
conv.empty_to_none,
),
code = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
conv.not_none,
),
description = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
),
NODE = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_node_xml_json,
drop_none_items = True,
),
conv.empty_to_none,
),
tail = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
text = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
),
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
),
)(node, state = state)
if errors is None:
children_groups_key = ('BAREME', 'CODE', 'NODE', 'VALBYTRANCHES')
if all(
validated_node.get(key) is None
for key in children_groups_key
):
error = state._(u"At least one of the following items must be present: {}").format(state._(u', ').join(
u'"{}"'.format(key)
for key in children_groups_key
))
errors = dict(
(key, error)
for key in children_groups_key
)
else:
errors = {}
children_code = set()
for key in children_groups_key:
for child_index, child in enumerate(validated_node.get(key) or []):
child_code = child['code']
if child_code in children_code:
errors.setdefault(key, {}).setdefault(child_index, {})['code'] = state._(u"Duplicate value")
else:
children_code.add(child_code)
conv.remove_ancestor_from_state(state, node)
return validated_node, errors or None
u'value': conv.pipe(
value_converter,
conv.not_none,
),
},
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
),
)(value, state = state)
conv.remove_ancestor_from_state(state, value)
return validated_value, errors
validate_values_holder_json = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_value_json,
drop_none_items = True,
),
make_validate_values_json_dates(require_consecutive_dates = False),
conv.empty_to_none,
)
# Level-2 Converters
validate_any_legislation_json = conv.pipe(
conv.test_isinstance(dict),
conv.condition(
def make_json_or_python_to_test(tax_benefit_system):
validate = conv.struct(
dict(
itertools.chain(
dict(
absolute_error_margin=conv.pipe(
conv.test_isinstance((float, int)),
conv.test_greater_or_equal(0),
),
axes=make_json_or_python_to_axes(tax_benefit_system),
description=conv.pipe(
conv.test_isinstance(basestring_type),
conv.cleanup_line,
),
input_variables=conv.pipe(
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(
conv.test_isinstance(basestring_type),
conv.not_none,
),
conv.noop,
),
conv.empty_to_none,
),
keywords=conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.pipe(
conv.test_isinstance(basestring_type),
conv.cleanup_line,
),
drop_none_items=True,
),
conv.empty_to_none,
),
name=conv.pipe(
conv.test_isinstance(basestring_type),
conv.cleanup_line,
),
output_variables=conv.test_isinstance(dict),
period=conv.pipe(
conv.function(periods.period),
conv.not_none,
),
relative_error_margin=conv.pipe(
conv.test_isinstance((float, int)),
conv.test_greater_or_equal(0),
),
).items(),
((entity.plural,
conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
)) for entity in tax_benefit_system.entities),
)), )
def json_or_python_to_test(value, state=None):
if value is None:
return value, None
if state is None:
state = conv.default_state
value, error = conv.pipe(
conv.test_isinstance(dict),
validate,
)(value, state=state)
if error is not None:
return value, error
value, error = conv.struct(
dict(output_variables=make_json_or_python_to_input_variables(
tax_benefit_system, value['period']), ),
default=conv.noop,
)(value, state=state)
if error is not None:
return value, error
test_case = value.copy()
absolute_error_margin = test_case.pop('absolute_error_margin')
axes = test_case.pop('axes')
description = test_case.pop('description')
input_variables = test_case.pop('input_variables')
keywords = test_case.pop('keywords')
name = test_case.pop('name')
output_variables = test_case.pop('output_variables')
period = test_case.pop('period')
relative_error_margin = test_case.pop('relative_error_margin')
if test_case is not None and all(
entity_members is None
for entity_members in test_case.values()):
test_case = None
scenario, error = tax_benefit_system.Scenario.make_json_to_instance(
repair=True, tax_benefit_system=tax_benefit_system)(dict(
axes=axes,
input_variables=input_variables,
period=period,
test_case=test_case,
),
state=state)
if error is not None:
return scenario, error
return {
key: value
for key, value in dict(
absolute_error_margin=absolute_error_margin,
description=description,
keywords=keywords,
name=name,
output_variables=output_variables,
relative_error_margin=relative_error_margin,
scenario=scenario,
).items() if value is not None
}, None
return json_or_python_to_test
def json_to_dated_python(self):
return conv.pipe(
conv.test_isinstance((basestring_type, bool, int)),
conv.guess_bool,
)
def json_or_python_to_test_case(value, state=None):
if value is None:
return value, None
if state is None:
state = conv.default_state
column_by_name = self.tax_benefit_system.column_by_name
# First validation and conversion step
test_case, error = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
individus=conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items=True,
),
conv.function(scenarios.set_entities_json_id),
conv.uniform_sequence(
conv.struct(
dict(
itertools.chain(
dict(id=conv.pipe(
conv.test_isinstance(
(basestring, int)),
conv.not_none,
), ).iteritems(),
((column.name,
column.json_to_python) for column
in column_by_name.itervalues()
if column.entity == 'ind' and
column.name not in ('idmen',
'quimen')),
)),
drop_none_values=True,
),
drop_none_items=True,
),
conv.empty_to_none,
conv.not_none,
),
menages=conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items=True,
),
conv.function(scenarios.set_entities_json_id),
conv.uniform_sequence(
conv.struct(
dict(
itertools.chain(
dict(
autres=conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(
(basestring, int)),
drop_none_items=True,
),
conv.default([]),
),
conjoint=conv.pipe(
conv.test_isinstance(
basestring, int),
conv.default(None),
),
enfants=conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(
(basestring, int)),
drop_none_items=True,
),
conv.default([]),
),
id=conv.pipe(
conv.test_isinstance(
(basestring, int)),
conv.not_none,
),
personne_de_reference=conv.
pipe(
conv.test_isinstance(
basestring, int),
conv.default(None),
),
).iteritems(),
((column.name,
column.json_to_python) for column
in column_by_name.itervalues()
if column.entity == 'men'),
)),
drop_none_values=True,
),
drop_none_items=True,
),
conv.default({}),
),
), ),
)(value, state=state)
if error is not None:
return test_case, error
# Second validation step
menages_individus_id = [
individu['id'] for individu in test_case['individus']
]
test_case, error = conv.struct(
dict(menages=conv.uniform_sequence(
conv.struct(
dict(
autres=conv.uniform_sequence(
conv.test_in_pop(menages_individus_id)),
conjoint=conv.test_in_pop(menages_individus_id),
enfants=conv.uniform_sequence(
conv.test_in_pop(menages_individus_id)),
personne_de_reference=conv.test_in_pop(
menages_individus_id),
),
default=conv.noop,
), ), ),
default=conv.noop,
)(test_case, state=state)
remaining_individus_id = set(menages_individus_id)
if remaining_individus_id:
individu_index_by_id = {
individu[u'id']: individu_index
for individu_index, individu in enumerate(
test_case[u'individus'])
}
if error is None:
error = {}
for individu_id in remaining_individus_id:
error.setdefault(
'individus',
{})[individu_index_by_id[individu_id]] = state._(
u"Individual is missing from {}").format(
state._(u' & ').join(word for word in [
u'menages' if individu_id in
menages_individus_id else None,
] if word is not None))
if error is not None:
return test_case, error
return test_case, error
class TaxBenefitSystem(object):
"""
Represents the legislation.
It stores parameters (values defined for everyone) and variables (values defined for some given entity e.g. a person).
:param entities: Entities used by the tax benefit system.
:param string parameters: Directory containing the YAML parameter files.
.. py:attribute:: parameters
:any:`ParameterNode` containing the legislation parameters
"""
_base_tax_benefit_system = None
_parameters_at_instant_cache = None
person_key_plural = None
preprocess_parameters = None
json_to_attributes = staticmethod(
conv.pipe(
conv.test_isinstance(dict),
conv.struct({}),
))
baseline = None # Baseline tax-benefit system. Used only by reforms. Note: Reforms can be chained.
Scenario = AbstractScenario
cache_blacklist = None
decomposition_file_path = None
def __init__(self, entities):
# TODO: Currently: Don't use a weakref, because they are cleared by Paste (at least) at each call.
self.parameters = None
self._parameters_at_instant_cache = {} # weakref.WeakValueDictionary()
self.variables = {}
self.open_api_config = {}
self.entities = entities
if entities is None or len(entities) == 0:
raise Exception(
"A tax and benefit sytem must have at least an entity.")
self.person_entity = [
entity for entity in entities if entity.is_person
][0]
self.group_entities = [
entity for entity in entities if not entity.is_person
]
@property
def base_tax_benefit_system(self):
base_tax_benefit_system = self._base_tax_benefit_system
if base_tax_benefit_system is None:
baseline = self.baseline
if baseline is None:
return self
self._base_tax_benefit_system = base_tax_benefit_system = baseline.base_tax_benefit_system
return base_tax_benefit_system
@classmethod
def json_to_instance(cls, value, state=None):
attributes, error = conv.pipe(
cls.json_to_attributes,
conv.default({}),
)(value, state=state or conv.default_state)
if error is not None:
return attributes, error
return cls(**attributes), None
def new_scenario(self):
scenario = self.Scenario()
scenario.tax_benefit_system = self
return scenario
def prefill_cache(self):
pass
def load_variable(self, variable_class, update=False):
name = to_unicode(variable_class.__name__)
# Check if a Variable with the same name is already registered.
baseline_variable = self.get_variable(name)
if baseline_variable and not update:
raise VariableNameConflict(
'Variable "{}" is already defined. Use `update_variable` to replace it.'
.format(name))
variable = variable_class(baseline_variable=baseline_variable)
self.variables[variable.name] = variable
return variable
def add_variable(self, variable):
"""
Adds an OpenFisca variable to the tax and benefit system.
:param Variable variable: The variable to add. Must be a subclass of Variable.
:raises: :any:`VariableNameConflict` if a variable with the same name have previously been added to the tax and benefit system.
"""
return self.load_variable(variable, update=False)
def replace_variable(self, variable):
"""
Replaces an existing OpenFisca variable in the tax and benefit system by a new one.
The new variable must have the same name than the replaced one.
If no variable with the given name exists in the tax and benefit system, no error will be raised and the new variable will be simply added.
:param Variable variable: New variable to add. Must be a subclass of Variable.
"""
name = to_unicode(variable.__name__)
if self.variables.get(name) is not None:
del self.variables[name]
self.load_variable(variable, update=False)
def update_variable(self, variable):
"""
Updates an existing OpenFisca variable in the tax and benefit system.
All attributes of the updated variable that are not explicitely overridden by the new ``variable`` will stay unchanged.
The new variable must have the same name than the updated one.
If no variable with the given name exists in the tax and benefit system, no error will be raised and the new variable will be simply added.
:param Variable variable: Variable to add. Must be a subclass of Variable.
"""
return self.load_variable(variable, update=True)
def add_variables_from_file(self, file_path):
"""
Adds all OpenFisca variables contained in a given file to the tax and benefit system.
"""
try:
file_name = path.splitext(path.basename(file_path))[0]
# As Python remembers loaded modules by name, in order to prevent collisions, we need to make sure that:
# - Files with the same name, but located in different directories, have a different module names. Hence the file path hash in the module name.
# - The same file, loaded by different tax and benefit systems, has distinct module names. Hence the `id(self)` in the module name.
module_name = '{}_{}_{}'.format(id(self),
hash(path.abspath(file_path)),
file_name)
module_directory = path.dirname(file_path)
try:
module = load_module(
module_name, *find_module(file_name, [module_directory]))
except NameError as e:
logging.error(
str(e) +
": if this code used to work, this error might be due to a major change in OpenFisca-Core. Checkout the changelog to learn more: <https://github.com/openfisca/openfisca-core/blob/master/CHANGELOG.md>"
)
raise
potential_variables = [
getattr(module, item) for item in dir(module)
if not item.startswith('__')
]
for pot_variable in potential_variables:
# We only want to get the module classes defined in this module (not imported)
if isclass(pot_variable) and issubclass(
pot_variable,
Variable) and pot_variable.__module__ == module_name:
self.add_variable(pot_variable)
except Exception:
log.error(
'Unable to load OpenFisca variables from file "{}"'.format(
file_path))
raise
def add_variables_from_directory(self, directory):
"""
Recursively explores a directory, and adds all OpenFisca variables found there to the tax and benefit system.
"""
py_files = glob.glob(path.join(directory, "*.py"))
for py_file in py_files:
self.add_variables_from_file(py_file)
subdirectories = glob.glob(path.join(directory, "*/"))
for subdirectory in subdirectories:
self.add_variables_from_directory(subdirectory)
def add_variables(self, *variables):
"""
Adds a list of OpenFisca Variables to the `TaxBenefitSystem`.
See also :any:`add_variable`
"""
for variable in variables:
self.add_variable(variable)
def load_extension(self, extension):
"""
Loads an extension to the tax and benefit system.
:param string extension: The extension to load. Can be an absolute path pointing to an extension directory, or the name of an OpenFisca extension installed as a pip package.
"""
if path.isdir(extension):
if find_packages(extension):
# Load extension from a package directory
extension_directory = path.join(extension,
find_packages(extension)[0])
else:
# Load extension from a simple directory
extension_directory = extension
else:
# Load extension from installed pip package
try:
package = importlib.import_module(extension)
extension_directory = package.__path__[0]
except ImportError:
message = linesep.join([
traceback.format_exc(),
'Error loading extension: `{}` is neither a directory, nor a package.'
.format(extension),
'Are you sure it is installed in your environment? If so, look at the stack trace above to determine the origin of this error.',
'See more at <https://github.com/openfisca/openfisca-extension-template#installing>.'
])
raise ValueError(message)
self.add_variables_from_directory(extension_directory)
param_dir = path.join(extension_directory, 'parameters')
if path.isdir(param_dir):
extension_parameters = ParameterNode(directory_path=param_dir)
self.parameters.merge(extension_parameters)
def apply_reform(self, reform_path):
"""
Generates a new tax and benefit system applying a reform to the tax and benefit system.
The current tax and benefit system is **not** mutated.
:param string reform_path: The reform to apply. Must respect the format *installed_package.sub_module.reform*
:returns: A reformed tax and benefit system.
Exemple:
>>> self.apply_reform('openfisca_france.reforms.inversion_revenus')
"""
from openfisca_core.reforms import Reform
try:
reform_package, reform_name = reform_path.rsplit('.', 1)
except ValueError:
raise ValueError(
'`{}` does not seem to be a path pointing to a reform. A path looks like `some_country_package.reforms.some_reform.`'
.format(reform_path).encode('utf-8'))
try:
reform_module = importlib.import_module(reform_package)
except ImportError:
message = linesep.join([
traceback.format_exc(), 'Could not import `{}`.'.format(
reform_package).encode('utf-8'),
'Are you sure of this reform module name? If so, look at the stack trace above to determine the origin of this error.'
])
raise ValueError(message)
reform = getattr(reform_module, reform_name, None)
if reform is None:
raise ValueError('{} has no attribute {}'.format(
reform_package, reform_name).encode('utf-8'))
if not issubclass(reform, Reform):
raise ValueError(
'`{}` does not seem to be a valid Openfisca reform.'.format(
reform_path).encode('utf-8'))
return reform(self)
def get_variable(self, variable_name, check_existence=False):
"""
Get a variable from the tax and benefit system.
:param variable_name: Name of the requested variable.
:param check_existence: If True, raise an error if the requested variable does not exist.
"""
variables = self.variables.get(variable_name)
if not variables and check_existence:
raise VariableNotFound(variable_name, self)
return variables
def neutralize_variable(self, variable_name):
"""
Neutralizes an OpenFisca variable existing in the tax and benefit system.
A neutralized variable always returns its default value when computed.
Trying to set inputs for a neutralized variable has no effect except raising a warning.
"""
self.variables[variable_name] = get_neutralized_variable(
self.get_variable(variable_name))
def load_parameters(self, path_to_yaml_dir):
"""
Loads the legislation parameter for a directory containing YAML parameters files.
:param path_to_yaml_dir: Absolute path towards the YAML parameter directory.
Exemples:
>>> self.load_parameters('/path/to/yaml/parameters/dir')
"""
parameters = ParameterNode('', directory_path=path_to_yaml_dir)
if self.preprocess_parameters is not None:
parameters = self.preprocess_parameters(parameters)
self.parameters = parameters
def _get_baseline_parameters_at_instant(self, instant):
baseline = self.baseline
if baseline is None:
return self.get_parameters_at_instant(instant)
return baseline._get_baseline_parameters_at_instant(instant)
def get_parameters_at_instant(self, instant):
"""
Get the parameters of the legislation at a given instant
:param instant: string of the format 'YYYY-MM-DD' or `openfisca_core.periods.Instant` instance.
:returns: The parameters of the legislation at a given instant.
:rtype: :any:`ParameterNodeAtInstant`
"""
if isinstance(instant, periods.Period):
instant = instant.start
elif isinstance(instant, (basestring_type, int)):
instant = periods.instant(instant)
else:
assert isinstance(
instant, periods.Instant
), "Expected an Instant (e.g. Instant((2017, 1, 1)) ). Got: {}.".format(
instant)
parameters_at_instant = self._parameters_at_instant_cache.get(instant)
if parameters_at_instant is None and self.parameters is not None:
parameters_at_instant = self.parameters.get_at_instant(
str(instant))
self._parameters_at_instant_cache[instant] = parameters_at_instant
return parameters_at_instant
def get_package_metadata(self):
"""
Gets metatada relative to the country package the tax and benefit system is built from.
:returns: Country package metadata
:rtype: dict
Exemple:
>>> tax_benefit_system.get_package_metadata()
>>> {
>>> 'location': '/path/to/dir/containing/package',
>>> 'name': 'openfisca-france',
>>> 'repository_url': 'https://github.com/openfisca/openfisca-france',
>>> 'version': '17.2.0'
>>> }
"""
# Handle reforms
if self.baseline:
return self.baseline.get_package_metadata()
fallback_metadata = {
'name': self.__class__.__name__,
'version': '',
'repository_url': '',
'location': '',
}
module = inspect.getmodule(self)
if not module.__package__:
return fallback_metadata
package_name = module.__package__.split('.')[0]
try:
distribution = pkg_resources.get_distribution(package_name)
except pkg_resources.DistributionNotFound:
return fallback_metadata
location = inspect.getsourcefile(module).split(package_name)[0].rstrip(
'/')
home_page_metadatas = [
metadata.split(':', 1)[1].strip(' ')
for metadata in distribution._get_metadata(distribution.PKG_INFO)
if 'Home-page' in metadata
]
repository_url = home_page_metadatas[0] if home_page_metadatas else ''
return {
'name': distribution.key,
'version': distribution.version,
'repository_url': repository_url,
'location': location,
}
def get_variables(self, entity=None):
"""
Gets all variables contained in a tax and benefit system.
:param <Entity subclass> entity: If set, returns only the variable defined for the given entity.
:returns: A dictionnary, indexed by variable names.
:rtype: dict
"""
if not entity:
return self.variables
else:
return {
variable_name: variable
for variable_name, variable in self.variables.items()
if variable.entity == entity
}
def clone(self):
new = empty_clone(self)
new_dict = new.__dict__
for key, value in self.__dict__.items():
if key not in ('parameters', '_parameters_at_instant_cache',
'variables', 'open_api_config'):
new_dict[key] = value
new_dict['parameters'] = self.parameters.clone()
new_dict['_parameters_at_instant_cache'] = {}
new_dict['variables'] = self.variables.copy()
new_dict['open_api_config'] = self.open_api_config.copy()
return new
def entities_plural(self):
return {entity.plural for entity in self.entities}
def entities_by_singular(self):
return {entity.key: entity for entity in self.entities}
def validate_value_json(value, state = None):
if value is None:
return None, None
container = state.ancestors[-1]
value_converter = dict(
boolean = conv.condition(
conv.test_isinstance(int),
conv.test_in((0, 1)),
conv.test_isinstance(bool),
),
float = conv.condition(
conv.test_isinstance(int),
conv.anything_to_float,
conv.test_isinstance(float),
),
integer = conv.condition(
conv.test_isinstance(float),
conv.pipe(
conv.test(lambda number: round(number) == number),
conv.function(int),
),
conv.test_isinstance(int),
),
rate = conv.condition(
conv.test_isinstance(int),
conv.anything_to_float,
conv.test_isinstance(float),
),
)[container.get('format') or 'float'] # Only parameters have a "format".
state = conv.add_ancestor_to_state(state, value)
validated_value, errors = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
{
u'comment': conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
u'from': conv.pipe(
conv.test_isinstance(basestring),
conv.iso8601_input_to_date,
conv.date_to_iso8601_str,
conv.not_none,
),
u'to': conv.pipe(
conv.test_isinstance(basestring),
conv.iso8601_input_to_date,
conv.date_to_iso8601_str,
conv.not_none,
),
u'value': conv.pipe(
value_converter,
conv.not_none,
),
},
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
),
)(value, state = state)
conv.remove_ancestor_from_state(state, value)
return validated_value, errors
def post_process_test_case(self, test_case, period, state):
individu_by_id = {
individu['id']: individu
for individu in test_case['individus']
}
parents_id = set(parent_id
for foyer_fiscal in test_case['foyers_fiscaux']
for parent_id in foyer_fiscal['declarants'])
test_case, error = conv.struct(
dict(
# foyers_fiscaux = conv.pipe(
# conv.uniform_sequence(
# conv.struct(
# dict(
# enfants = conv.uniform_sequence(
# conv.test(
# lambda individu_id:
# individu_by_id[individu_id].get('invalide', False)
# or find_age(individu_by_id[individu_id], period.start.date,
# default = 0) <= 25,
# error = u"Une personne à charge d'un foyer fiscal doit avoir moins de"
# u" 25 ans ou être handicapée",
# ),
# ),
# parents = conv.pipe(
# conv.empty_to_none,
# conv.not_none,
# conv.test(lambda parents: len(parents) <= 2,
# error = N_(u'A "famille" must have at most 2 "parents"'))
# ),
# ),
# default = conv.noop,
# ),
# ),
# conv.empty_to_none,
# conv.not_none,
# ),
foyers_fiscaux=conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(
declarants=conv.pipe(
conv.empty_to_none,
conv.not_none,
conv.test(
lambda declarants: len(declarants) <=
2,
error=N_(
u'A "foyer_fiscal" must have at most 2 "declarants"'
),
),
),
personnes_a_charge=conv.uniform_sequence(
conv.test(
lambda individu_id:
individu_by_id[individu_id].get(
'handicap', False) or find_age(
individu_by_id[individu_id],
period.start.date,
default=0) <= 25,
error=
u"Une personne à charge d'un foyer fiscal doit avoir moins de"
u" 25 ans ou être handicapée",
), ),
),
default=conv.noop,
), ),
conv.empty_to_none,
conv.not_none,
),
menages=conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(personne_de_reference=conv.not_none, ),
default=conv.noop,
), ),
conv.empty_to_none,
conv.not_none,
),
),
default=conv.noop,
)(test_case, state=state)
return test_case, error
# First validation and conversion step
test_case, error = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
foyers_fiscaux=conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items=True,
),
conv.uniform_sequence(
conv.struct(
dict(
itertools.chain(
dict(
declarants=conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(
(basestring, int)),
drop_none_items=True,
),
conv.default([]),
),
id=conv.pipe(
conv.test_isinstance(
(basestring, int)),
conv.not_none,
),
personnes_a_charge=conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(
(basestring, int)),
drop_none_items=True,
),
conv.default([]),
),
).iteritems(), )),
drop_none_values=True,
),
drop_none_items=True,
),
conv.default([]),
),
menages=conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items=True,
),
conv.uniform_sequence(
conv.struct(
dict(
itertools.chain(
dict(
autres=conv.pipe(
# personnes ayant un lien autre avec la personne de référence
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(
(basestring, int)),
drop_none_items=True,
),
conv.default([]),
),
# conjoint de la personne de référence
conjoint=conv.test_isinstance(
(basestring, int)),
enfants=conv.pipe(
# enfants de la personne de référence ou de son conjoint
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(
(basestring, int)),
drop_none_items=True,
),
conv.default([]),
),
id=conv.pipe(
conv.test_isinstance(
(basestring, int)),
conv.not_none,
),
personne_de_reference=conv.
test_isinstance((basestring, int)),
).iteritems(), )),
drop_none_values=True,
),
drop_none_items=True,
),
conv.default([]),
),
), ),
)(test_case, state=state)
return test_case, error
def validate_node_json(node, state = None):
if node is None:
return None, None
state = conv.add_ancestor_to_state(state, node)
validated_node, error = conv.test_isinstance(dict)(node, state = state)
if error is not None:
conv.remove_ancestor_from_state(state, node)
return validated_node, error
validated_node, errors = conv.struct(
{
'@context': conv.pipe(
conv.test_isinstance(basestring),
conv.make_input_to_url(full = True),
conv.test_equals(u'http://openfisca.fr/contexts/legislation.jsonld'),
),
'@type': conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
conv.test_in((u'Node', u'Parameter', u'Scale')),
conv.not_none,
),
'comment': conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
'description': conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
),
},
constructor = collections.OrderedDict,
default = conv.noop,
drop_none_values = 'missing',
keep_value_order = True,
)(validated_node, state = state)
if errors is not None:
conv.remove_ancestor_from_state(state, node)
return validated_node, errors
validated_node.pop('@context', None) # Remove optional @context everywhere. It will be added to root node later.
node_converters = {
'@type': conv.noop,
'comment': conv.noop,
'description': conv.noop,
}
node_type = validated_node['@type']
if node_type == u'Node':
node_converters.update(dict(
children = conv.pipe(
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
conv.not_none,
),
conv.pipe(
validate_node_json,
conv.not_none,
),
),
conv.empty_to_none,
conv.not_none,
),
))
elif node_type == u'Parameter':
node_converters.update(dict(
format = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in([
'boolean',
'float',
'integer',
'rate',
]),
),
unit = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in(units),
),
values = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_value_json,
drop_none_items = True,
),
make_validate_values_json_dates(require_consecutive_dates = True),
conv.empty_to_none,
conv.not_none,
),
))
else:
assert node_type == u'Scale'
node_converters.update(dict(
option = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in((
'contrib',
'main-d-oeuvre',
'noncontrib',
)),
),
slices = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_slice_json,
drop_none_items = True,
),
validate_slices_json_dates,
conv.empty_to_none,
conv.not_none,
),
unit = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in((
'currency',
)),
),
))
validated_node, errors = conv.struct(
node_converters,
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
)(validated_node, state = state)
conv.remove_ancestor_from_state(state, node)
return validated_node, errors
def json_or_python_to_test_case(value, state = None):
if value is None:
return value, None
if state is None:
state = conv.default_state
column_by_name = self.tax_benefit_system.column_by_name
# First validation and conversion step
test_case, error = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
familles = conv.pipe(
conv.condition(
conv.test_isinstance(list),
conv.pipe(
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items = True,
),
conv.function(lambda values: collections.OrderedDict(
(value.pop('id', index), value)
for index, value in enumerate(values)
)),
),
),
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(
conv.test_isinstance((basestring, int)),
conv.not_none,
),
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(itertools.chain(
dict(
enfants = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.default([]),
),
parents = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.default([]),
),
).iteritems(),
(
(column.name, column.json_to_python)
for column in column_by_name.itervalues()
if column.entity == 'fam'
),
)),
drop_none_values = True,
),
),
drop_none_values = True,
),
conv.default({}),
),
individus = conv.pipe(
conv.condition(
conv.test_isinstance(list),
conv.pipe(
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items = True,
),
conv.function(lambda values: collections.OrderedDict(
(value.pop('id', index), value)
for index, value in enumerate(values)
)),
),
),
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(
conv.test_isinstance((basestring, int)),
conv.not_none,
),
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
(column.name, column.json_to_python)
for column in column_by_name.itervalues()
if column.entity == 'ind' and column.name not in (
'idfam', 'idfoy', 'idmen', 'quifam', 'quifoy', 'quimen')
),
drop_none_values = True,
),
),
drop_none_values = True,
),
conv.empty_to_none,
conv.not_none,
),
),
),
)(value, state = state)
if error is not None:
return test_case, error
# Second validation step
familles_individus_id = list(test_case['individus'].iterkeys())
test_case, error = conv.struct(
dict(
familles = conv.uniform_mapping(
conv.noop,
conv.struct(
dict(
enfants = conv.uniform_sequence(conv.test_in_pop(familles_individus_id)),
parents = conv.uniform_sequence(conv.test_in_pop(familles_individus_id)),
),
default = conv.noop,
),
),
),
default = conv.noop,
)(test_case, state = state)
remaining_individus_id = set(familles_individus_id)
if remaining_individus_id:
if error is None:
error = {}
for individu_id in remaining_individus_id:
error.setdefault('individus', {})[individu_id] = state._(u"Individual is missing from {}").format(
state._(u' & ').join(
word
for word in [
u'familles' if individu_id in familles_individus_id else None,
]
if word is not None
))
if error is not None:
return test_case, error
return test_case, error
def json_or_python_to_attributes(value, state=None):
if value is None:
return value, None
if state is None:
state = conv.default_state
# First validation and conversion step
data, error = conv.pipe(
conv.test_isinstance(dict),
# TODO: Remove condition below, once every calls uses "period" instead of "date" & "year".
self.cleanup_period_in_json_or_python,
conv.struct(
dict(
axes=make_json_or_python_to_axes(
self.tax_benefit_system),
input_variables=conv.test_isinstance(
dict
), # Real test is done below, once period is known.
period=conv.pipe(
conv.function(periods.period),
conv.not_none,
),
test_case=conv.test_isinstance(
dict
), # Real test is done below, once period is known.
), ),
)(value, state=state)
if error is not None:
return data, error
# Second validation and conversion step
data, error = conv.struct(
dict(
input_variables=make_json_or_python_to_input_variables(
self.tax_benefit_system, data['period']),
test_case=self.make_json_or_python_to_test_case(
period=data['period'], repair=repair),
),
default=conv.noop,
)(data, state=state)
if error is not None:
return data, error
# Third validation and conversion step
errors = {}
if data['input_variables'] is not None and data[
'test_case'] is not None:
errors['input_variables'] = state._(
"Items input_variables and test_case can't both exist")
errors['test_case'] = state._(
"Items input_variables and test_case can't both exist")
elif data['axes'] is not None and data["test_case"] is None:
errors['axes'] = state._(
"Axes can't be used with input_variables.")
if errors:
return data, errors
if data['axes'] is not None:
for parallel_axes_index, parallel_axes in enumerate(
data['axes']):
first_axis = parallel_axes[0]
axis_count = first_axis['count']
axis_entity_key = tbs.get_variable(
first_axis['name']).entity.key
first_axis_period = first_axis['period'] or data['period']
for axis_index, axis in enumerate(parallel_axes):
if axis['min'] >= axis['max']:
errors.setdefault('axes', {}).setdefault(
parallel_axes_index, {}
).setdefault(axis_index, {})['max'] = state._(
"Max value must be greater than min value")
column = tbs.get_variable(axis['name'])
if axis['index'] >= len(
data['test_case'][column.entity.plural]):
errors.setdefault('axes', {}).setdefault(
parallel_axes_index, {}).setdefault(
axis_index, {})['index'] = state._(
"Index must be lower than {}").format(
len(data['test_case'][
column.entity.plural]))
if axis_index > 0:
if axis['count'] != axis_count:
errors.setdefault('axes', {}).setdefault(
parallel_axes_index, {}
).setdefault(
axis_index, {}
)['count'] = state._(
"Parallel indexes must have the same count"
)
if column.entity.key != axis_entity_key:
errors.setdefault('axes', {}).setdefault(
parallel_axes_index, {}
).setdefault(
axis_index, {}
)['period'] = state._(
"Parallel indexes must belong to the same entity"
)
axis_period = axis['period'] or data['period']
if axis_period.unit != first_axis_period.unit:
errors.setdefault('axes', {}).setdefault(
parallel_axes_index, {}
).setdefault(
axis_index, {}
)['period'] = state._(
"Parallel indexes must have the same period unit"
)
elif axis_period.size != first_axis_period.size:
errors.setdefault('axes', {}).setdefault(
parallel_axes_index, {}
).setdefault(
axis_index, {}
)['period'] = state._(
"Parallel indexes must have the same period size"
)
if errors:
return data, errors
self.axes = data['axes']
if data['input_variables'] is not None:
self.input_variables = data['input_variables']
self.period = data['period']
if data['test_case'] is not None:
self.test_case = data['test_case']
return self, None
def validate_value_xml_json(value, state = None):
if value is None:
return None, None
container = state.ancestors[-1]
value_converter = dict(
bool = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
conv.test_in([u'0', u'1']),
),
float = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
conv.test_conv(conv.anything_to_float),
),
integer = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
conv.test_conv(conv.anything_to_strict_int),
),
percent = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
conv.test_conv(conv.anything_to_float),
),
date = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
#TODO: add a new conv ?
conv.test_conv(conv.anything_to_strict_int),
),
)[container.get('format') or 'float'] # Only CODE have a "format".
state = conv.add_ancestor_to_state(state, value)
validated_value, errors = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
code = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
),
deb = conv.pipe(
conv.test_isinstance(basestring),
conv.iso8601_input_to_date,
conv.date_to_iso8601_str,
conv.not_none,
),
fin = conv.pipe(
conv.test_isinstance(basestring),
conv.iso8601_input_to_date,
conv.date_to_iso8601_str,
conv.not_none,
),
format = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in(xml_json_formats),
conv.test_equals(container.get('format')),
),
tail = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
text = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
type = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in([
'age',
'days',
'hours',
'monetary',
'months',
]),
conv.test_equals(container.get('type')),
),
valeur = conv.pipe(
value_converter,
conv.not_none,
),
),
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
),
)(value, state = state)
conv.remove_ancestor_from_state(state, value)
return validated_value, errors
def post_process_test_case(self, test_case, period, state):
individu_by_id = {
individu['id']: individu
for individu in test_case['individus']
}
parents_id = set(
parent_id
for foyer_fiscal in test_case['foyers_fiscaux']
for parent_id in foyer_fiscal['declarants']
)
test_case, error = conv.struct(
dict(
# foyers_fiscaux = conv.pipe(
# conv.uniform_sequence(
# conv.struct(
# dict(
# enfants = conv.uniform_sequence(
# conv.test(
# lambda individu_id:
# individu_by_id[individu_id].get('invalide', False)
# or find_age(individu_by_id[individu_id], period.start.date,
# default = 0) <= 25,
# error = u"Une personne à charge d'un foyer fiscal doit avoir moins de"
# u" 25 ans ou être handicapée",
# ),
# ),
# parents = conv.pipe(
# conv.empty_to_none,
# conv.not_none,
# conv.test(lambda parents: len(parents) <= 2,
# error = N_(u'A "famille" must have at most 2 "parents"'))
# ),
# ),
# default = conv.noop,
# ),
# ),
# conv.empty_to_none,
# conv.not_none,
# ),
foyers_fiscaux = conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(
declarants = conv.pipe(
conv.empty_to_none,
conv.not_none,
conv.test(
lambda declarants: len(declarants) <= 2,
error = N_(u'A "foyer_fiscal" must have at most 2 "declarants"'),
),
conv.uniform_sequence(conv.pipe(
)),
),
personnes_a_charge = conv.uniform_sequence(
conv.test(
lambda individu_id:
individu_by_id[individu_id].get('handicap', False)
or find_age(individu_by_id[individu_id], period.start.date,
default = 0) <= 25,
error = u"Une personne à charge d'un foyer fiscal doit avoir moins de"
u" 25 ans ou être handicapée",
),
),
),
default = conv.noop,
),
),
conv.empty_to_none,
conv.not_none,
),
menages = conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(
personne_de_reference = conv.not_none,
),
default = conv.noop,
),
),
conv.empty_to_none,
conv.not_none,
),
),
default = conv.noop,
)(test_case, state = state)
return test_case, error
# First validation and conversion step
test_case, error = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
foyers_fiscaux = conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items = True,
),
conv.uniform_sequence(
conv.struct(
dict(itertools.chain(
dict(
declarants = conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.default([]),
),
id = conv.pipe(
conv.test_isinstance((basestring, int)),
conv.not_none,
),
personnes_a_charge = conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.default([]),
),
).iteritems(),
)),
drop_none_values = True,
),
drop_none_items = True,
),
conv.default([]),
),
menages = conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items = True,
),
conv.uniform_sequence(
conv.struct(
dict(itertools.chain(
dict(
autres = conv.pipe(
# personnes ayant un lien autre avec la personne de référence
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.default([]),
),
# conjoint de la personne de référence
conjoint = conv.test_isinstance((basestring, int)),
enfants = conv.pipe(
# enfants de la personne de référence ou de son conjoint
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.default([]),
),
id = conv.pipe(
conv.test_isinstance((basestring, int)),
conv.not_none,
),
personne_de_reference = conv.test_isinstance((basestring, int)),
).iteritems(),
)),
drop_none_values = True,
),
drop_none_items = True,
),
conv.default([]),
),
),
),
)(test_case, state = state)
# test_case, error = conv.struct(
# dict(
# foyers_fiscaux = conv.uniform_sequence(
# conv.struct(
# dict(
# declarants = conv.uniform_sequence(conv.test_in_pop(foyers_fiscaux_individus_id)),
# personnes_a_charge = conv.uniform_sequence(conv.test_in_pop(
# foyers_fiscaux_individus_id)),
# ),
# default = conv.noop,
# ),
# ),
# menages = conv.uniform_sequence(
# conv.struct(
# dict(
# autres = conv.uniform_sequence(conv.test_in_pop(menages_individus_id)),
# conjoint = conv.test_in_pop(menages_individus_id),
# enfants = conv.uniform_sequence(conv.test_in_pop(menages_individus_id)),
# personne_de_reference = conv.test_in_pop(menages_individus_id),
# ),
# default = conv.noop,
# ),
# ),
# ),
# default = conv.noop,
# )(test_case, state = state)
return test_case, error
def json_to_dated_python(self):
return conv.pipe(
conv.test_isinstance((float, int, basestring_type)),
conv.make_anything_to_float(accept_expression=True),
)
def validate_slice_xml_json(slice, state = None):
if slice is None:
return None, None
state = conv.add_ancestor_to_state(state, slice)
validated_slice, errors = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
ASSIETTE = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_values_holder_xml_json,
drop_none_items = True,
),
conv.empty_to_none,
conv.test(lambda l: len(l) == 1, error = N_(u"List must contain one and only one item")),
),
code = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
),
MONTANT = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_values_holder_xml_json,
drop_none_items = True,
),
conv.empty_to_none,
conv.test(lambda l: len(l) == 1, error = N_(u"List must contain one and only one item")),
),
SEUIL = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_values_holder_xml_json,
drop_none_items = True,
),
conv.empty_to_none,
conv.test(lambda l: len(l) == 1, error = N_(u"List must contain one and only one item")),
conv.not_none,
),
tail = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
TAUX = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_values_holder_xml_json,
drop_none_items = True,
),
conv.empty_to_none,
conv.test(lambda l: len(l) == 1, error = N_(u"List must contain one and only one item")),
),
text = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
),
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
),
conv.test(lambda slice: bool(slice.get('MONTANT')) ^ bool(slice.get('TAUX')),
error = N_(u"Either MONTANT or TAUX must be provided")),
)(slice, state = state)
conv.remove_ancestor_from_state(state, slice)
return validated_slice, errors
def json_or_python_to_test_case(value, state=None):
if value is None:
return value, None
if state is None:
state = conv.default_state
column_by_name = self.tax_benefit_system.column_by_name
# First validation and conversion step
test_case, error = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
individus=conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(conv.test_isinstance(dict), drop_none_items=True),
conv.function(scenarios.set_entities_json_id),
conv.uniform_sequence(
conv.struct(
dict(
itertools.chain(
dict(
id=conv.pipe(conv.test_isinstance((basestring, int)), conv.not_none)
).iteritems(),
(
(column.name, column.json_to_python)
for column in column_by_name.itervalues()
if column.entity == "ind" and column.name not in ("idmen", "quimen")
),
)
),
drop_none_values=True,
),
drop_none_items=True,
),
conv.empty_to_none,
conv.not_none,
),
menages=conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(conv.test_isinstance(dict), drop_none_items=True),
conv.function(scenarios.set_entities_json_id),
conv.uniform_sequence(
conv.struct(
dict(
itertools.chain(
dict(
autres=conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)), drop_none_items=True
),
conv.default([]),
),
conjoint=conv.pipe(
conv.test_isinstance(basestring, int), conv.default(None)
),
enfants=conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)), drop_none_items=True
),
conv.default([]),
),
id=conv.pipe(conv.test_isinstance((basestring, int)), conv.not_none),
personne_de_reference=conv.pipe(
conv.test_isinstance(basestring, int), conv.default(None)
),
).iteritems(),
(
(column.name, column.json_to_python)
for column in column_by_name.itervalues()
if column.entity == "men"
),
)
),
drop_none_values=True,
),
drop_none_items=True,
),
conv.default({}),
),
)
),
)(value, state=state)
if error is not None:
return test_case, error
# Second validation step
menages_individus_id = [individu["id"] for individu in test_case["individus"]]
test_case, error = conv.struct(
dict(
menages=conv.uniform_sequence(
conv.struct(
dict(
autres=conv.uniform_sequence(conv.test_in_pop(menages_individus_id)),
conjoint=conv.test_in_pop(menages_individus_id),
enfants=conv.uniform_sequence(conv.test_in_pop(menages_individus_id)),
personne_de_reference=conv.test_in_pop(menages_individus_id),
),
default=conv.noop,
)
)
),
default=conv.noop,
)(test_case, state=state)
remaining_individus_id = set(menages_individus_id)
if remaining_individus_id:
individu_index_by_id = {
individu[u"id"]: individu_index for individu_index, individu in enumerate(test_case[u"individus"])
}
if error is None:
error = {}
for individu_id in remaining_individus_id:
error.setdefault("individus", {})[individu_index_by_id[individu_id]] = state._(
u"Individual is missing from {}"
).format(
state._(u" & ").join(
word
for word in [u"menages" if individu_id in menages_individus_id else None]
if word is not None
)
)
if error is not None:
return test_case, error
return test_case, error
