def validate_slice_json(slice, state = None):
if slice is None:
return None, None
state = conv.add_ancestor_to_state(state, slice)
validated_slice, errors = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
base = validate_values_holder_json,
comment = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
rate = conv.pipe(
validate_values_holder_json,
conv.not_none,
),
threshold = conv.pipe(
validate_values_holder_json,
conv.not_none,
),
),
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
),
)(slice, state = state)
conv.remove_ancestor_from_state(state, slice)
return validated_slice, errors
def validate_dated_legislation_json(dated_legislation_json, state = None):
if dated_legislation_json is None:
return None, None
if state is None:
state = conv.default_state
dated_legislation_json, error = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
datesim = conv.pipe(
conv.test_isinstance(basestring),
conv.iso8601_input_to_date,
conv.date_to_iso8601_str,
conv.not_none,
),
),
constructor = collections.OrderedDict,
default = conv.noop,
drop_none_values = 'missing',
keep_value_order = True,
),
)(dated_legislation_json, state = state)
if error is not None:
return dated_legislation_json, error
datesim = dated_legislation_json.pop('datesim')
dated_legislation_json, error = validate_dated_node_json(dated_legislation_json, state = state)
dated_legislation_json['datesim'] = datesim
return dated_legislation_json, error
def validate_dated_legislation_json(dated_legislation_json, state=None):
if dated_legislation_json is None:
return None, None
if state is None:
state = conv.default_state
dated_legislation_json, error = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(datesim=conv.pipe(
conv.test_isinstance(basestring),
conv.iso8601_input_to_date,
conv.date_to_iso8601_str,
conv.not_none,
), ),
constructor=collections.OrderedDict,
default=conv.noop,
drop_none_values='missing',
keep_value_order=True,
),
)(dated_legislation_json, state=state)
if error is not None:
return dated_legislation_json, error
datesim = dated_legislation_json.pop('datesim')
dated_legislation_json, error = validate_dated_node_json(
dated_legislation_json, state=state)
dated_legislation_json['datesim'] = datesim
return dated_legislation_json, error
def input_to_json_data(value, state=None):
return conv.pipe(
conv.make_input_to_json(),
conv.test_isinstance(dict),
conv.struct(
dict(
resultat_officiel=conv.pipe(
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(
conv.test_isinstance(basestring),
conv.not_none,
),
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
code=conv.pipe(
conv.test_isinstance(basestring),
conv.not_none,
),
name=conv.pipe(
conv.test_isinstance(basestring),
conv.not_none,
),
value=conv.pipe(
conv.test_isinstance(float),
conv.not_none,
),
), ),
conv.not_none,
),
),
conv.not_none,
),
scenario=conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
test_case=conv.pipe(
conv.test_isinstance(dict),
# For each entity convert its members from a dict to a list.
conv.uniform_mapping(
conv.noop,
conv.pipe(
conv.test_isinstance(dict),
conv.function(
transform_entity_member_by_id_to_members
),
),
),
), ),
default=conv.noop,
),
tax_benefit_system.Scenario.make_json_to_instance(
tax_benefit_system=tax_benefit_system),
conv.not_none,
),
), ),
)(value, state=state or conv.default_state)
def make_json_to_array_by_period(self, period):
return conv.condition(
conv.test_isinstance(dict),
conv.pipe(
# Value is a dict of (period, value) couples.
conv.uniform_mapping(
conv.pipe(
conv.function(periods.period),
conv.not_none,
),
conv.pipe(
conv.make_item_to_singleton(),
conv.uniform_sequence(self.json_to_dated_python, ),
conv.empty_to_none,
conv.function(lambda cells_list: np.array(
cells_list, dtype=self.dtype)),
),
drop_none_values=True,
),
conv.empty_to_none,
),
conv.pipe(
conv.make_item_to_singleton(),
conv.uniform_sequence(self.json_to_dated_python, ),
conv.empty_to_none,
conv.function(
lambda cells_list: np.array(cells_list, dtype=self.dtype)),
conv.function(lambda array: {period: array}),
),
)
def validate_slice_json(slice, state=None):
if slice is None:
return None, None
state = conv.add_ancestor_to_state(state, slice)
validated_slice, errors = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
base=validate_values_holder_json,
comment=conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
rate=conv.pipe(
validate_values_holder_json,
conv.not_none,
),
threshold=conv.pipe(
validate_values_holder_json,
conv.not_none,
),
),
constructor=collections.OrderedDict,
drop_none_values='missing',
keep_value_order=True,
),
)(slice, state=state)
conv.remove_ancestor_from_state(state, slice)
return validated_slice, errors
def validate_node_json(node, state=None):
if node is None:
return None, None
if state is None:
state = conv.default_state
validated_node, errors = conv.pipe(
conv.condition(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_node_json,
drop_none_items=True,
),
conv.pipe(
conv.condition(
conv.test_isinstance(basestring_type),
conv.function(lambda code: dict(code=code)),
conv.test_isinstance(dict),
),
conv.struct(
dict(
children=conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_node_json,
drop_none_items=True,
),
conv.empty_to_none,
),
code=conv.pipe(
conv.test_isinstance(basestring_type),
conv.cleanup_line,
),
),
constructor=collections.OrderedDict,
default=conv.noop,
drop_none_values='missing',
keep_value_order=True,
),
),
),
conv.empty_to_none,
)(node, state=state)
if validated_node is None or errors is not None:
return validated_node, errors
if isinstance(validated_node,
dict) and not validated_node.get('children'):
validated_node, errors = conv.struct(
dict(code=conv.pipe(
conv.test_in(tax_benefit_system.variables),
conv.not_none,
), ),
default=conv.noop,
)(validated_node, state=state)
return validated_node, errors
def validate_parameter_xml_json(parameter, state = None):
if parameter is None:
return None, None
state = conv.add_ancestor_to_state(state, parameter)
validated_parameter, errors = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
code = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
conv.not_none,
),
description = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
),
format = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in(xml_json_formats),
),
tail = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
taille = conv.pipe(
conv.test_isinstance(basestring),
conv.test_in([
'moinsde20',
'plusde20',
]),
),
text = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
type = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in(json_unit_by_xml_json_type),
),
VALUE = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_value_xml_json,
drop_none_items = True,
),
make_validate_values_xml_json_dates(require_consecutive_dates = True),
conv.empty_to_none,
conv.not_none,
),
),
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
),
)(parameter, state = state)
conv.remove_ancestor_from_state(state, parameter)
return validated_parameter, errors
def json_or_python_to_input_variables(value, state=None):
if value is None:
return value, None
if state is None:
state = conv.default_state
input_variables, errors = conv.pipe(
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(
conv.test_isinstance(basestring_type),
conv.not_none,
),
conv.noop,
),
)(value, state=state)
if errors is not None:
return input_variables, errors
for input_variable_name in input_variables.keys():
if input_variable_name not in variables_name:
# We only import VariableNotFound here to avoid a circular dependency in imports
from .taxbenefitsystems import VariableNotFound
raise VariableNotFound(input_variable_name, tax_benefit_system)
input_variables, errors = conv.pipe(
make_json_or_python_to_array_by_period_by_variable_name(
tax_benefit_system, period),
conv.empty_to_none,
)(value, state=state)
if errors is not None:
return input_variables, errors
if input_variables is None:
return input_variables, None
count_by_entity_key = {}
errors = {}
for variable_name, array_by_period in input_variables.items():
column = tax_benefit_system.get_variable(variable_name)
entity_key = column.entity.key
entity_count = count_by_entity_key.get(entity_key, 0)
for variable_period, variable_array in array_by_period.items():
if entity_count == 0:
count_by_entity_key[entity_key] = entity_count = len(
variable_array)
elif len(variable_array) != entity_count:
errors[column.name] = state._(
"Array has not the same length as other variables of entity {}: {} instead of {}"
).format(column.name, len(variable_array), entity_count)
return input_variables, errors or None
def json_to_dated_python(self):
enum = self.variable.possible_values
possible_names = [item.name for item in list(enum)]
if enum is None:
return conv.pipe(conv.test_isinstance(basestring_type))
return conv.pipe(
conv.test_isinstance(basestring_type),
conv.pipe(
# Verify that item belongs to enumeration.
conv.test_in(possible_names),
# Transform that item into enum object.
conv.function(lambda enum_name: enum[enum_name])))
def json_to_python(self):
return conv.condition(
conv.test_isinstance(dict),
conv.pipe(
# Value is a dict of (period, value) couples.
conv.uniform_mapping(
conv.pipe(
conv.function(periods.period),
conv.not_none,
),
self.json_to_dated_python,
), ),
self.json_to_dated_python,
)
def validate_scale_xml_json(scale, state = None):
if scale is None:
return None, None
state = conv.add_ancestor_to_state(state, scale)
validated_scale, errors = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
code = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
conv.not_none,
),
description = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
),
option = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in((
'contrib',
'main-d-oeuvre',
'noncontrib',
)),
),
TRANCHE = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_slice_xml_json,
drop_none_items = True,
),
validate_slices_xml_json_dates,
conv.empty_to_none,
conv.not_none,
),
tail = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
text = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
type = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in((
'monetary',
)),
),
),
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
),
)(scale, state = state)
conv.remove_ancestor_from_state(state, scale)
return validated_scale, errors
def input_to_json_data(value, state=None):
return conv.pipe(
conv.make_input_to_json(),
conv.test_isinstance(dict),
conv.struct(
dict(
resultat_officiel=conv.pipe(
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(conv.test_isinstance(basestring), conv.not_none),
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
code=conv.pipe(conv.test_isinstance(basestring), conv.not_none),
name=conv.pipe(conv.test_isinstance(basestring), conv.not_none),
value=conv.pipe(conv.test_isinstance(float), conv.not_none),
)
),
conv.not_none,
),
),
conv.not_none,
),
scenario=conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
test_case=conv.pipe(
conv.test_isinstance(dict),
# For each entity convert its members from a dict to a list.
conv.uniform_mapping(
conv.noop,
conv.pipe(
conv.test_isinstance(dict),
conv.function(transform_entity_member_by_id_to_members),
),
),
)
),
default=conv.noop,
),
tax_benefit_system.Scenario.make_json_to_instance(tax_benefit_system=tax_benefit_system),
conv.not_none,
),
)
),
)(value, state=state or conv.default_state)
def validate_dated_value_json(value, state = None):
if value is None:
return None, None
container = state.ancestors[-1]
value_converter = dict(
boolean = conv.condition(
conv.test_isinstance(int),
conv.test_in((0, 1)),
conv.test_isinstance(bool),
),
float = conv.condition(
conv.test_isinstance(int),
conv.anything_to_float,
conv.test_isinstance(float),
),
integer = conv.condition(
conv.test_isinstance(float),
conv.pipe(
conv.test(lambda number: round(number) == number),
conv.function(int),
),
conv.test_isinstance(int),
),
rate = conv.condition(
conv.test_isinstance(int),
conv.anything_to_float,
conv.test_isinstance(float),
),
)[container.get('format') or 'float'] # Only parameters have a "format".
return value_converter(value, state = state or conv.default_state)
def input_to_dated_python(self):
enum = self.variable.possible_values
if enum is None:
return conv.test_isinstance(basestring_type)
return conv.pipe(
# Verify that item index belongs to enumeration.
conv.test_in([item.name for item in list(enum)]))
def test():
dir_path = os.path.join(os.path.dirname(__file__), 'formulas')
for filename in sorted(os.listdir(dir_path)):
if not filename.endswith('.yaml'):
continue
filename_core = os.path.splitext(filename)[0]
with open(os.path.join(dir_path, filename)) as yaml_file:
tests = yaml.load(yaml_file)
tests, error = conv.pipe(
conv.make_item_to_singleton(),
conv.uniform_sequence(
conv.noop,
drop_none_items = True,
),
)(tests)
if error is not None:
embedding_error = conv.embed_error(tests, u'errors', error)
assert embedding_error is None, embedding_error
conv.check((tests, error)) # Generate an error.
for test in tests:
test, error = scenarios.make_json_or_python_to_test(tax_benefit_system)(test)
if error is not None:
embedding_error = conv.embed_error(test, u'errors', error)
assert embedding_error is None, embedding_error
conv.check((test, error)) # Generate an error.
if test.get(u'ignore', False):
continue
yield check, test.get('name') or filename_core, unicode(test['scenario'].period), test
def validate_dated_value_json(value, state=None):
if value is None:
return None, None
container = state.ancestors[-1]
value_converter = dict(
boolean=conv.condition(
conv.test_isinstance(int),
conv.test_in((0, 1)),
conv.test_isinstance(bool),
),
float=conv.condition(
conv.test_isinstance(int),
conv.anything_to_float,
conv.test_isinstance(float),
),
integer=conv.condition(
conv.test_isinstance(float),
conv.pipe(
conv.test(lambda number: round(number) == number),
conv.function(int),
),
conv.test_isinstance(int),
),
rate=conv.condition(
conv.test_isinstance(int),
conv.anything_to_float,
conv.test_isinstance(float),
),
)[container.get('format') or 'float'] # Only parameters have a "format".
return value_converter(value, state=state or conv.default_state)
def _parse_yaml_file(tax_benefit_system, yaml_path):
filename = os.path.splitext(os.path.basename(yaml_path))[0]
with open(yaml_path) as yaml_file:
tests = yaml.load(yaml_file)
tests, error = conv.pipe(
conv.make_item_to_singleton(),
conv.uniform_sequence(
conv.noop,
drop_none_items = True,
),
)(tests)
if error is not None:
embedding_error = conv.embed_error(tests, u'errors', error)
assert embedding_error is None, embedding_error
raise ValueError("Error in test {}:\n{}".format(yaml_path, yaml.dump(tests, allow_unicode = True,
default_flow_style = False, indent = 2, width = 120)))
for test in tests:
test, error = scenarios.make_json_or_python_to_test(
tax_benefit_system = tax_benefit_system
)(test)
if error is not None:
embedding_error = conv.embed_error(test, u'errors', error)
assert embedding_error is None, embedding_error
raise ValueError("Error in test {}:\n{}\nYaml test content: \n{}\n".format(
yaml_path, error, yaml.dump(test, allow_unicode = True,
default_flow_style = False, indent = 2, width = 120)))
yield yaml_path, test.get('name') or filename, unicode(test['scenario'].period), test
def input_to_dated_python(self):
return conv.pipe(
conv.test(year_or_month_or_day_re.match, error=N_('Invalid date')),
conv.function(lambda birth: '-'.join(
(birth.split('-') + ['01', '01'])[:3])),
conv.iso8601_input_to_date,
)
def json_or_python_to_test(value, state=None):
if value is None:
return value, None
if state is None:
state = conv.default_state
value, error = conv.pipe(
conv.test_isinstance(dict),
validate,
)(value, state=state)
if error is not None:
return value, error
value, error = conv.struct(
dict(output_variables=make_json_or_python_to_input_variables(
tax_benefit_system, value['period']), ),
default=conv.noop,
)(value, state=state)
if error is not None:
return value, error
test_case = value.copy()
absolute_error_margin = test_case.pop('absolute_error_margin')
axes = test_case.pop('axes')
description = test_case.pop('description')
input_variables = test_case.pop('input_variables')
keywords = test_case.pop('keywords')
name = test_case.pop('name')
output_variables = test_case.pop('output_variables')
period = test_case.pop('period')
relative_error_margin = test_case.pop('relative_error_margin')
if test_case is not None and all(
entity_members is None
for entity_members in test_case.values()):
test_case = None
scenario, error = tax_benefit_system.Scenario.make_json_to_instance(
repair=True, tax_benefit_system=tax_benefit_system)(dict(
axes=axes,
input_variables=input_variables,
period=period,
test_case=test_case,
),
state=state)
if error is not None:
return scenario, error
return {
key: value
for key, value in dict(
absolute_error_margin=absolute_error_margin,
description=description,
keywords=keywords,
name=name,
output_variables=output_variables,
relative_error_margin=relative_error_margin,
scenario=scenario,
).items() if value is not None
}, None
def json_to_instance(cls, value, state=None):
attributes, error = conv.pipe(
cls.json_to_attributes,
conv.default({}),
)(value, state=state or conv.default_state)
if error is not None:
return attributes, error
return cls(**attributes), None
def json_to_dated_python(self):
return conv.pipe(
super(AgeCol, self).json_to_dated_python,
conv.first_match(
conv.test_greater_or_equal(0),
conv.test_equals(-9999),
),
)
def json_to_dated_python(self):
return conv.pipe(
conv.condition(
conv.test_isinstance((float, int)),
# YAML stores strings containing only digits as numbers.
conv.function(str),
),
conv.test_isinstance(basestring_type),
)
def json_to_dated_python(self):
return conv.pipe(
conv.condition(
conv.test_isinstance((float, int)),
# YAML stores strings containing only digits as numbers.
conv.function(str),
),
conv.test_isinstance(basestring_type),
conv.test(lambda value: len(value) <= self.variable.max_length),
)
def make_json_or_python_to_axes(tax_benefit_system):
column_by_name = tax_benefit_system.variables
return conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.pipe(
conv.item_or_sequence(
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
count=conv.pipe(
conv.test_isinstance(int),
conv.test_greater_or_equal(1),
conv.not_none,
),
index=conv.pipe(
conv.test_isinstance(int),
conv.test_greater_or_equal(0),
conv.default(0),
),
max=conv.pipe(
conv.test_isinstance((float, int)),
conv.not_none,
),
min=conv.pipe(
conv.test_isinstance((float, int)),
conv.not_none,
),
name=conv.pipe(
conv.test_isinstance(basestring_type),
conv.test_in(column_by_name),
conv.test(
lambda column_name: tax_benefit_system.
get_variable(column_name).dtype in
(np.float32, np.int16, np.int32),
error=N_(
'Invalid type for axe: integer or float expected'
)),
conv.not_none,
),
period=conv.function(periods.period),
), ),
),
drop_none_items=True,
),
conv.make_item_to_singleton(),
),
drop_none_items=True,
),
conv.empty_to_none,
)
def test(force = False, name_filter = None, options_by_path = None):
if isinstance(name_filter, str):
name_filter = name_filter.decode('utf-8')
if options_by_path is None:
options_by_path = options_by_dir
for path, options in options_by_path.iteritems():
if not force and options.get('ignore', False):
log.info(u'Ignoring {}'.format(path))
continue
if not os.path.exists(path):
log.warning(u'Skipping missing {}'.format(path))
continue
if os.path.isdir(path):
yaml_paths = [
os.path.join(path, filename)
for filename in sorted(os.listdir(path))
if filename.endswith('.yaml')
]
else:
yaml_paths = [path]
for yaml_path in yaml_paths:
filename_core = os.path.splitext(os.path.basename(yaml_path))[0]
with open(yaml_path) as yaml_file:
tests = yaml.load(yaml_file)
tests, error = conv.pipe(
conv.make_item_to_singleton(),
conv.uniform_sequence(
conv.noop,
drop_none_items = True,
),
)(tests)
if error is not None:
embedding_error = conv.embed_error(tests, u'errors', error)
assert embedding_error is None, embedding_error
raise ValueError("Error in test {}:\n{}".format(yaml_path, yaml.dump(tests, allow_unicode = True,
default_flow_style = False, indent = 2, width = 120)))
for test in tests:
test, error = scenarios.make_json_or_python_to_test(get_tax_benefit_system(options.get('reform')),
default_absolute_error_margin = options['default_absolute_error_margin'])(test)
if error is not None:
embedding_error = conv.embed_error(test, u'errors', error)
assert embedding_error is None, embedding_error
raise ValueError("Error in test {}:\n{}".format(yaml_path, yaml.dump(test, allow_unicode = True,
default_flow_style = False, indent = 2, width = 120)))
if not force and test.get(u'ignore', False):
continue
if name_filter is not None and name_filter not in filename_core \
and name_filter not in (test.get('name', u'')) \
and name_filter not in (test.get('keywords', [])):
continue
checker = check_any_period if options['accept_other_period'] else check
yield checker, test.get('name') or filename_core, unicode(test['scenario'].period), test, force
def json_to_dated_python(self):
return conv.pipe(
conv.condition(
conv.test_isinstance(datetime.date),
conv.noop,
conv.condition(
conv.test_isinstance(int),
conv.pipe(
conv.test_between(1870, 2099),
conv.function(lambda year: datetime.date(year, 1, 1)),
),
conv.pipe(
conv.test_isinstance(basestring_type),
conv.test(year_or_month_or_day_re.match, error = N_('Invalid date')),
conv.function(lambda birth: '-'.join((birth.split('-') + ['01', '01'])[:3])),
conv.iso8601_input_to_date,
),
),
),
conv.test_between(datetime.date(1870, 1, 1), datetime.date(2099, 12, 31)),
)
def check_entities_and_role(test_case, tax_benefit_system, state):
"""
Check that the test_case describes entities consistent with the tax and benefit system.
Will raise an error if :
- An entity is not recognized
- An entity role is not recognized
- A variable is declared for an entity it is not defined for (e.g. salary for a family)
"""
test_case = deepcopy(
test_case) # Avoid side-effects on other references to test_case
entity_classes = {
entity_class.plural: entity_class
for entity_class in tax_benefit_system.entities
}
for entity_type_name, entities in test_case.items():
if entity_classes.get(entity_type_name) is None:
raise ValueError("Invalid entity name: {}".format(
entity_type_name).encode('utf-8'))
entities, error = conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items=True,
),
conv.function(set_entities_json_id),
)(entities)
if error is not None:
raise ValueError("Invalid list of {}: {}. Error: {}".format(
entity_type_name, entities, error).encode('utf-8'))
if entities is None:
entities = test_case[entity_type_name] = [
] # YAML test runner may set these values to None
entity_class = entity_classes[entity_type_name]
valid_roles = dict(
(role.key, role) if (role.max == 1) else (role.plural, role) for
role in entity_class.roles) if not entity_class.is_person else {}
for entity_json in entities:
check_entity_fields(entity_json, entity_class, valid_roles,
tax_benefit_system)
for entity_class in entity_classes.values():
if test_case.get(entity_class.plural) is None:
test_case[entity_class.plural] = [
] # by convention, all entities must be declared in the test_case
return test_case
def _parse_test_file(tax_benefit_system, yaml_path):
filename = os.path.splitext(os.path.basename(yaml_path))[0]
with open(yaml_path) as yaml_file:
try:
tests = yaml.load(yaml_file, Loader=Loader)
except yaml.scanner.ScannerError:
log.error("{} is not a valid YAML file".format(yaml_path).encode('utf-8'))
raise
tests, error = conv.pipe(
conv.make_item_to_singleton(),
conv.uniform_sequence(
conv.noop,
drop_none_items = True,
),
)(tests)
if error is not None:
embedding_error = conv.embed_error(tests, 'errors', error)
assert embedding_error is None, embedding_error
raise ValueError("Error in test {}:\n{}".format(yaml_path, yaml.dump(tests, Dumper=Dumper, allow_unicode = True,
default_flow_style = False, indent = 2, width = 120)))
for test in tests:
current_tax_benefit_system = tax_benefit_system
if test.get('reforms'):
reforms = test.pop('reforms')
if not isinstance(reforms, list):
reforms = [reforms]
for reform_path in reforms:
current_tax_benefit_system = current_tax_benefit_system.apply_reform(reform_path)
try:
test, error = scenarios.make_json_or_python_to_test(
tax_benefit_system = current_tax_benefit_system
)(test)
except Exception:
log.error("{} is not a valid OpenFisca test file".format(yaml_path).encode('utf-8'))
raise
if error is not None:
embedding_error = conv.embed_error(test, 'errors', error)
assert embedding_error is None, embedding_error
raise ValueError("Error in test {}:\n{}\nYaml test content: \n{}\n".format(
yaml_path, error, yaml.dump(test, Dumper=Dumper, allow_unicode = True,
default_flow_style = False, indent = 2, width = 120)))
yield yaml_path, test.get('name') or filename, to_unicode(test['scenario'].period), test
def check_role(value, key):
role = valid_roles.get(key)
if role.max == 1:
value, error = conv.test_isinstance((basestring_type, int))(value)
else:
value, error = conv.pipe(
conv.make_item_to_singleton(), conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring_type, int)),
drop_none_items=True,
))(value)
if error is not None:
raise ValueError("Invalid description of {}: {}. Error: {}".format(
entity_class.key, entity_json, error).encode('utf-8'))
entity_json[key] = value
def test(current_options_by_dir = None, force = False, name_filter = None):
if current_options_by_dir is None:
current_options_by_dir = options_by_dir
for dir, options in sorted(current_options_by_dir.iteritems()):
if not force and options.get('ignore', False):
log.info(u'Ignoring directory: {}'.format(dir))
continue
if not os.path.isdir(dir):
log.warning(u'Skipping missing directory: {}'.format(dir))
continue
if isinstance(name_filter, str):
name_filter = name_filter.decode('utf-8')
for filename in sorted(os.listdir(dir)):
if not filename.endswith('.yaml'):
continue
filename_core = os.path.splitext(filename)[0]
with open(os.path.join(dir, filename)) as yaml_file:
tests = yaml.load(yaml_file)
tests, error = conv.pipe(
conv.make_item_to_singleton(),
conv.uniform_sequence(
conv.noop,
drop_none_items = True,
),
)(tests)
if error is not None:
embedding_error = conv.embed_error(tests, u'errors', error)
assert embedding_error is None, embedding_error
conv.check((tests, error)) # Generate an error.
for test in tests:
test, error = scenarios.make_json_or_python_to_test(get_tax_benefit_system(options.get('reform')),
default_absolute_error_margin = options['default_absolute_error_margin'])(test)
if error is not None:
embedding_error = conv.embed_error(test, u'errors', error)
assert embedding_error is None, embedding_error
conv.check((test, error)) # Generate an error.
if not force and test.get(u'ignore', False):
continue
if name_filter is not None and name_filter not in filename_core \
and name_filter not in (test.get('name', u'')) \
and name_filter not in (test.get('keywords', [])):
continue
checker = check_any_period if options['accept_other_period'] else check
yield checker, test.get('name') or filename_core, unicode(test['scenario'].period), test, force
def test(force = False, name_filter = None):
if isinstance(name_filter, str):
name_filter = name_filter.decode('utf-8')
yaml_paths = [
os.path.join(source_file_dir_name, filename)
for filename in sorted(os.listdir(source_file_dir_name))
if filename.endswith('.yaml')
]
for yaml_path in yaml_paths:
filename_core = os.path.splitext(os.path.basename(yaml_path))[0]
with open(yaml_path) as yaml_file:
tests = yaml.load(yaml_file)
tests, error = conv.pipe(
conv.make_item_to_singleton(),
conv.uniform_sequence(
conv.noop,
drop_none_items = True,
),
)(tests)
if error is not None:
embedding_error = conv.embed_error(tests, u'errors', error)
assert embedding_error is None, embedding_error
raise ValueError("Error in test {}:\n{}".format(yaml_path, yaml.dump(tests, allow_unicode = True,
default_flow_style = False, indent = 2, width = 120)))
for test in tests:
test, error = scenarios.make_json_or_python_to_test(
tax_benefit_system = tax_benefit_system,
default_absolute_error_margin = default_absolute_error_margin,
)(test)
if error is not None:
embedding_error = conv.embed_error(test, u'errors', error)
assert embedding_error is None, embedding_error
raise ValueError("Error in test {}:\n{}\nYaml test content: \n{}\n".format(
yaml_path, error, yaml.dump(test, allow_unicode = True,
default_flow_style = False, indent = 2, width = 120)))
if not force and test.get(u'ignore', False):
continue
if name_filter is not None and name_filter not in filename_core \
and name_filter not in (test.get('name', u'')) \
and name_filter not in (test.get('keywords', [])):
continue
yield check, yaml_path, test.get('name') or filename_core, unicode(test['scenario'].period), test, force
def validate_dated_slice_json(slice, state=None):
if slice is None:
return None, None
state = conv.add_ancestor_to_state(state, slice)
validated_slice, errors = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
base=conv.pipe(
validate_dated_value_json,
conv.test_greater_or_equal(0),
),
constant_amount=validate_dated_value_json,
comment=conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
rate=conv.pipe(
validate_dated_value_json,
conv.test_between(0, 1),
),
threshold=conv.pipe(
validate_dated_value_json,
conv.test_greater_or_equal(0),
),
date=conv.pipe(
validate_dated_value_json,
conv.test_between(0, 1), #TODO: changer
),
),
constructor=collections.OrderedDict,
drop_none_values='missing',
keep_value_order=True,
),
)(slice, state=state)
conv.remove_ancestor_from_state(state, slice)
return validated_slice, errors
def validate_dated_slice_json(slice, state = None):
if slice is None:
return None, None
state = conv.add_ancestor_to_state(state, slice)
validated_slice, errors = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
base = conv.pipe(
validate_dated_value_json,
conv.test_greater_or_equal(0),
),
constant_amount = validate_dated_value_json,
comment = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
rate = conv.pipe(
validate_dated_value_json,
conv.test_between(0, 1),
),
threshold = conv.pipe(
validate_dated_value_json,
conv.test_greater_or_equal(0),
),
date = conv.pipe(
validate_dated_value_json,
conv.test_between(0, 1), #TODO: changer
),
),
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
),
)(slice, state = state)
conv.remove_ancestor_from_state(state, slice)
return validated_slice, errors
def validate_value_json(value, state = None):
if value is None:
return None, None
container = state.ancestors[-1]
value_converter = dict(
boolean = conv.condition(
conv.test_isinstance(int),
conv.test_in((0, 1)),
conv.test_isinstance(bool),
),
float = conv.condition(
conv.test_isinstance(int),
conv.anything_to_float,
conv.test_isinstance(float),
),
integer = conv.condition(
conv.test_isinstance(float),
conv.pipe(
conv.test(lambda number: round(number) == number),
conv.function(int),
),
conv.test_isinstance(int),
),
rate = conv.condition(
conv.test_isinstance(int),
conv.anything_to_float,
conv.test_isinstance(float),
),
)[container.get('format') or 'float'] # Only parameters have a "format".
state = conv.add_ancestor_to_state(state, value)
validated_value, errors = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
{
u'comment': conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
u'from': conv.pipe(
conv.test_isinstance(basestring),
conv.iso8601_input_to_date,
conv.date_to_iso8601_str,
conv.not_none,
),
u'to': conv.pipe(
conv.test_isinstance(basestring),
conv.iso8601_input_to_date,
conv.date_to_iso8601_str,
conv.not_none,
),
u'value': conv.pipe(
value_converter,
conv.not_none,
),
},
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
),
)(value, state = state)
conv.remove_ancestor_from_state(state, value)
return validated_value, errors
def make_json_or_python_to_test(tax_benefit_system):
validate = conv.struct(
dict(
itertools.chain(
dict(
absolute_error_margin=conv.pipe(
conv.test_isinstance((float, int)),
conv.test_greater_or_equal(0),
),
axes=make_json_or_python_to_axes(tax_benefit_system),
description=conv.pipe(
conv.test_isinstance(basestring_type),
conv.cleanup_line,
),
input_variables=conv.pipe(
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(
conv.test_isinstance(basestring_type),
conv.not_none,
),
conv.noop,
),
conv.empty_to_none,
),
keywords=conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.pipe(
conv.test_isinstance(basestring_type),
conv.cleanup_line,
),
drop_none_items=True,
),
conv.empty_to_none,
),
name=conv.pipe(
conv.test_isinstance(basestring_type),
conv.cleanup_line,
),
output_variables=conv.test_isinstance(dict),
period=conv.pipe(
conv.function(periods.period),
conv.not_none,
),
relative_error_margin=conv.pipe(
conv.test_isinstance((float, int)),
conv.test_greater_or_equal(0),
),
).items(),
((entity.plural,
conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
)) for entity in tax_benefit_system.entities),
)), )
def json_or_python_to_test(value, state=None):
if value is None:
return value, None
if state is None:
state = conv.default_state
value, error = conv.pipe(
conv.test_isinstance(dict),
validate,
)(value, state=state)
if error is not None:
return value, error
value, error = conv.struct(
dict(output_variables=make_json_or_python_to_input_variables(
tax_benefit_system, value['period']), ),
default=conv.noop,
)(value, state=state)
if error is not None:
return value, error
test_case = value.copy()
absolute_error_margin = test_case.pop('absolute_error_margin')
axes = test_case.pop('axes')
description = test_case.pop('description')
input_variables = test_case.pop('input_variables')
keywords = test_case.pop('keywords')
name = test_case.pop('name')
output_variables = test_case.pop('output_variables')
period = test_case.pop('period')
relative_error_margin = test_case.pop('relative_error_margin')
if test_case is not None and all(
entity_members is None
for entity_members in test_case.values()):
test_case = None
scenario, error = tax_benefit_system.Scenario.make_json_to_instance(
repair=True, tax_benefit_system=tax_benefit_system)(dict(
axes=axes,
input_variables=input_variables,
period=period,
test_case=test_case,
),
state=state)
if error is not None:
return scenario, error
return {
key: value
for key, value in dict(
absolute_error_margin=absolute_error_margin,
description=description,
keywords=keywords,
name=name,
output_variables=output_variables,
relative_error_margin=relative_error_margin,
scenario=scenario,
).items() if value is not None
}, None
return json_or_python_to_test
def json_or_python_to_attributes(value, state=None):
if value is None:
return value, None
if state is None:
state = conv.default_state
# First validation and conversion step
data, error = conv.pipe(
conv.test_isinstance(dict),
# TODO: Remove condition below, once every calls uses "period" instead of "date" & "year".
self.cleanup_period_in_json_or_python,
conv.struct(
dict(
axes=make_json_or_python_to_axes(
self.tax_benefit_system),
input_variables=conv.test_isinstance(
dict
), # Real test is done below, once period is known.
period=conv.pipe(
conv.function(periods.period),
conv.not_none,
),
test_case=conv.test_isinstance(
dict
), # Real test is done below, once period is known.
), ),
)(value, state=state)
if error is not None:
return data, error
# Second validation and conversion step
data, error = conv.struct(
dict(
input_variables=make_json_or_python_to_input_variables(
self.tax_benefit_system, data['period']),
test_case=self.make_json_or_python_to_test_case(
period=data['period'], repair=repair),
),
default=conv.noop,
)(data, state=state)
if error is not None:
return data, error
# Third validation and conversion step
errors = {}
if data['input_variables'] is not None and data[
'test_case'] is not None:
errors['input_variables'] = state._(
"Items input_variables and test_case can't both exist")
errors['test_case'] = state._(
"Items input_variables and test_case can't both exist")
elif data['axes'] is not None and data["test_case"] is None:
errors['axes'] = state._(
"Axes can't be used with input_variables.")
if errors:
return data, errors
if data['axes'] is not None:
for parallel_axes_index, parallel_axes in enumerate(
data['axes']):
first_axis = parallel_axes[0]
axis_count = first_axis['count']
axis_entity_key = tbs.get_variable(
first_axis['name']).entity.key
first_axis_period = first_axis['period'] or data['period']
for axis_index, axis in enumerate(parallel_axes):
if axis['min'] >= axis['max']:
errors.setdefault('axes', {}).setdefault(
parallel_axes_index, {}
).setdefault(axis_index, {})['max'] = state._(
"Max value must be greater than min value")
column = tbs.get_variable(axis['name'])
if axis['index'] >= len(
data['test_case'][column.entity.plural]):
errors.setdefault('axes', {}).setdefault(
parallel_axes_index, {}).setdefault(
axis_index, {})['index'] = state._(
"Index must be lower than {}").format(
len(data['test_case'][
column.entity.plural]))
if axis_index > 0:
if axis['count'] != axis_count:
errors.setdefault('axes', {}).setdefault(
parallel_axes_index, {}
).setdefault(
axis_index, {}
)['count'] = state._(
"Parallel indexes must have the same count"
)
if column.entity.key != axis_entity_key:
errors.setdefault('axes', {}).setdefault(
parallel_axes_index, {}
).setdefault(
axis_index, {}
)['period'] = state._(
"Parallel indexes must belong to the same entity"
)
axis_period = axis['period'] or data['period']
if axis_period.unit != first_axis_period.unit:
errors.setdefault('axes', {}).setdefault(
parallel_axes_index, {}
).setdefault(
axis_index, {}
)['period'] = state._(
"Parallel indexes must have the same period unit"
)
elif axis_period.size != first_axis_period.size:
errors.setdefault('axes', {}).setdefault(
parallel_axes_index, {}
).setdefault(
axis_index, {}
)['period'] = state._(
"Parallel indexes must have the same period size"
)
if errors:
return data, errors
self.axes = data['axes']
if data['input_variables'] is not None:
self.input_variables = data['input_variables']
self.period = data['period']
if data['test_case'] is not None:
self.test_case = data['test_case']
return self, None
),
},
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
),
)(value, state = state)
conv.remove_ancestor_from_state(state, value)
return validated_value, errors
validate_values_holder_json = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_value_json,
drop_none_items = True,
),
make_validate_values_json_dates(require_consecutive_dates = False),
conv.empty_to_none,
)
# Level-2 Converters
validate_any_legislation_json = conv.pipe(
conv.test_isinstance(dict),
conv.condition(
conv.test(lambda legislation_json: 'datesim' in legislation_json),
validate_dated_legislation_json,
validate_node_json,
def json_or_python_to_test_case(value, state = None):
if value is None:
return value, None
if state is None:
state = conv.default_state
column_by_name = self.tax_benefit_system.column_by_name
# First validation and conversion step
test_case, error = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
familles = conv.pipe(
conv.condition(
conv.test_isinstance(list),
conv.pipe(
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items = True,
),
conv.function(lambda values: collections.OrderedDict(
(value.pop('id', index), value)
for index, value in enumerate(values)
)),
),
),
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(
conv.test_isinstance((basestring, int)),
conv.not_none,
),
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(itertools.chain(
dict(
enfants = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.default([]),
),
parents = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.default([]),
),
).iteritems(),
(
(column.name, column.json_to_python)
for column in column_by_name.itervalues()
if column.entity == 'fam'
),
)),
drop_none_values = True,
),
),
drop_none_values = True,
),
conv.default({}),
),
individus = conv.pipe(
conv.condition(
conv.test_isinstance(list),
conv.pipe(
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items = True,
),
conv.function(lambda values: collections.OrderedDict(
(value.pop('id', index), value)
for index, value in enumerate(values)
)),
),
),
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(
conv.test_isinstance((basestring, int)),
conv.not_none,
),
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
(column.name, column.json_to_python)
for column in column_by_name.itervalues()
if column.entity == 'ind' and column.name not in (
'idfam', 'idfoy', 'idmen', 'quifam', 'quifoy', 'quimen')
),
drop_none_values = True,
),
),
drop_none_values = True,
),
conv.empty_to_none,
conv.not_none,
),
),
),
)(value, state = state)
if error is not None:
return test_case, error
# Second validation step
familles_individus_id = list(test_case['individus'].iterkeys())
test_case, error = conv.struct(
dict(
familles = conv.uniform_mapping(
conv.noop,
conv.struct(
dict(
enfants = conv.uniform_sequence(conv.test_in_pop(familles_individus_id)),
parents = conv.uniform_sequence(conv.test_in_pop(familles_individus_id)),
),
default = conv.noop,
),
),
),
default = conv.noop,
)(test_case, state = state)
remaining_individus_id = set(familles_individus_id)
if remaining_individus_id:
if error is None:
error = {}
for individu_id in remaining_individus_id:
error.setdefault('individus', {})[individu_id] = state._(u"Individual is missing from {}").format(
state._(u' & ').join(
word
for word in [
u'familles' if individu_id in familles_individus_id else None,
]
if word is not None
))
if error is not None:
return test_case, error
return test_case, error
def validate_value_xml_json(value, state = None):
if value is None:
return None, None
container = state.ancestors[-1]
value_converter = dict(
bool = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
conv.test_in([u'0', u'1']),
),
float = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
conv.test_conv(conv.anything_to_float),
),
integer = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
conv.test_conv(conv.anything_to_strict_int),
),
percent = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
conv.test_conv(conv.anything_to_float),
),
date = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
#TODO: add a new conv ?
conv.test_conv(conv.anything_to_strict_int),
),
)[container.get('format') or 'float'] # Only CODE have a "format".
state = conv.add_ancestor_to_state(state, value)
validated_value, errors = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
code = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
),
deb = conv.pipe(
conv.test_isinstance(basestring),
conv.iso8601_input_to_date,
conv.date_to_iso8601_str,
conv.not_none,
),
fin = conv.pipe(
conv.test_isinstance(basestring),
conv.iso8601_input_to_date,
conv.date_to_iso8601_str,
conv.not_none,
),
format = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in(xml_json_formats),
conv.test_equals(container.get('format')),
),
tail = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
text = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
type = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in([
'age',
'days',
'hours',
'monetary',
'months',
]),
conv.test_equals(container.get('type')),
),
valeur = conv.pipe(
value_converter,
conv.not_none,
),
),
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
),
)(value, state = state)
conv.remove_ancestor_from_state(state, value)
return validated_value, errors
def test(force = False, name_filter = None, options_by_path = None):
if isinstance(name_filter, str):
name_filter = name_filter.decode('utf-8')
if options_by_path is None:
options_by_path = options_by_dir
for path, options in options_by_path.iteritems():
if not force and options.get('ignore', False):
log.info(u'Ignoring {}'.format(path))
continue
if not os.path.exists(path):
log.warning(u'Skipping missing {}'.format(path))
continue
if os.path.isdir(path):
yaml_paths = [
os.path.join(path, filename)
for filename in sorted(os.listdir(path))
if filename.endswith('.yaml')
]
else:
yaml_paths = [path]
if options.get('requires'):
# Check if the required package was successfully imported in tests/base.py
if getattr(base, options.get('requires')) is None:
continue
reform_keys = options.get('reforms')
tax_benefit_system_for_path = base.get_cached_composed_reform(
reform_keys = reform_keys,
tax_benefit_system = base.tax_benefit_system,
) if reform_keys is not None else base.tax_benefit_system
for yaml_path in yaml_paths:
filename_core = os.path.splitext(os.path.basename(yaml_path))[0]
with open(yaml_path) as yaml_file:
tests = yaml.load(yaml_file)
tests, error = conv.pipe(
conv.make_item_to_singleton(),
conv.uniform_sequence(
conv.noop,
drop_none_items = True,
),
)(tests)
if error is not None:
embedding_error = conv.embed_error(tests, u'errors', error)
assert embedding_error is None, embedding_error
raise ValueError("Error in test {}:\n{}".format(yaml_path, yaml.dump(tests, allow_unicode = True,
default_flow_style = False, indent = 2, width = 120)))
for test in tests:
test, error = scenarios.make_json_or_python_to_test(
tax_benefit_system = tax_benefit_system_for_path,
default_absolute_error_margin = options.get('default_absolute_error_margin'),
default_relative_error_margin = options.get('default_relative_error_margin'),
)(test)
if error is not None:
embedding_error = conv.embed_error(test, u'errors', error)
assert embedding_error is None, embedding_error
raise ValueError("Error in test {}:\n{}\nYaml test content: \n{}\n".format(
yaml_path, error, yaml.dump(test, allow_unicode = True,
default_flow_style = False, indent = 2, width = 120)))
if not force and test.get(u'ignore', False):
continue
if name_filter is not None and name_filter not in filename_core \
and name_filter not in (test.get('name', u'')) \
and name_filter not in (test.get('keywords', [])):
continue
checker = check_calculate_output if options['calculate_output'] else check
yield checker, yaml_path, test.get('name') or filename_core, unicode(test['scenario'].period), test, \
force
def post_process_test_case(self, test_case, period, state):
individu_by_id = {
individu['id']: individu
for individu in test_case['individus']
}
parents_id = set( # noqa F841
parent_id
for famille in test_case['familles']
for parent_id in famille['parents']
)
test_case, error = conv.struct(
dict(
familles = conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(
enfants = conv.uniform_sequence(
conv.test(
lambda individu_id:
individu_by_id[individu_id].get('handicap', False)
or find_age(individu_by_id[individu_id], period.start.date,
default = 0) <= 25,
error = u"Une personne à charge d'un foyer fiscal doit avoir moins de"
u" 25 ans ou être handicapée",
),
),
parents = conv.pipe(
conv.empty_to_none,
conv.not_none,
conv.test(lambda parents: len(parents) <= 2,
error = N_(u'A "famille" must have at most 2 "parents"'))
),
),
default = conv.noop,
),
),
conv.empty_to_none,
conv.not_none,
),
foyers_fiscaux = conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(
declarants = conv.pipe(
conv.empty_to_none,
conv.not_none,
conv.test(
lambda declarants: len(declarants) <= 2,
error = N_(u'A "foyer_fiscal" must have at most 2 "declarants"'),
),
conv.uniform_sequence(conv.pipe(
)),
),
personnes_a_charge = conv.uniform_sequence(
conv.test(
lambda individu_id:
individu_by_id[individu_id].get('handicap', False)
or find_age(individu_by_id[individu_id], period.start.date,
default = 0) <= 25,
error = u"Une personne à charge d'un foyer fiscal doit avoir moins de"
u" 25 ans ou être handicapée",
),
),
),
default = conv.noop,
),
),
conv.empty_to_none,
conv.not_none,
),
menages = conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(
personne_de_reference = conv.not_none,
),
default = conv.noop,
),
),
conv.empty_to_none,
conv.not_none,
),
),
default = conv.noop,
)(test_case, state = state)
return test_case, error
def validate_node_json(node, state = None):
if node is None:
return None, None
state = conv.add_ancestor_to_state(state, node)
validated_node, error = conv.test_isinstance(dict)(node, state = state)
if error is not None:
conv.remove_ancestor_from_state(state, node)
return validated_node, error
validated_node, errors = conv.struct(
{
'@context': conv.pipe(
conv.test_isinstance(basestring),
conv.make_input_to_url(full = True),
conv.test_equals(u'http://openfisca.fr/contexts/legislation.jsonld'),
),
'@type': conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
conv.test_in((u'Node', u'Parameter', u'Scale')),
conv.not_none,
),
'comment': conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
'description': conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
),
},
constructor = collections.OrderedDict,
default = conv.noop,
drop_none_values = 'missing',
keep_value_order = True,
)(validated_node, state = state)
if errors is not None:
conv.remove_ancestor_from_state(state, node)
return validated_node, errors
validated_node.pop('@context', None) # Remove optional @context everywhere. It will be added to root node later.
node_converters = {
'@type': conv.noop,
'comment': conv.noop,
'description': conv.noop,
}
node_type = validated_node['@type']
if node_type == u'Node':
node_converters.update(dict(
children = conv.pipe(
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
conv.not_none,
),
conv.pipe(
validate_node_json,
conv.not_none,
),
),
conv.empty_to_none,
conv.not_none,
),
))
elif node_type == u'Parameter':
node_converters.update(dict(
format = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in([
'boolean',
'float',
'integer',
'rate',
]),
),
unit = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in(units),
),
values = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_value_json,
drop_none_items = True,
),
make_validate_values_json_dates(require_consecutive_dates = True),
conv.empty_to_none,
conv.not_none,
),
))
else:
assert node_type == u'Scale'
node_converters.update(dict(
option = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in((
'contrib',
'main-d-oeuvre',
'noncontrib',
)),
),
slices = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_slice_json,
drop_none_items = True,
),
validate_slices_json_dates,
conv.empty_to_none,
conv.not_none,
),
unit = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in((
'currency',
)),
),
))
validated_node, errors = conv.struct(
node_converters,
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
)(validated_node, state = state)
conv.remove_ancestor_from_state(state, node)
return validated_node, errors
def attribute_groupless_persons_to_entities(self, test_case, period, groupless_individus):
individus_without_menage = groupless_individus['menages']
individus_without_foyer_fiscal = groupless_individus['foyers_fiscaux']
individu_by_id = {
individu['id']: individu
for individu in test_case['individus']
}
# Affecte à un foyer fiscal chaque individu qui n'appartient à aucun d'entre eux.
new_foyer_fiscal = dict(
declarants = [],
personnes_a_charge = [],
)
new_foyer_fiscal_id = None
for individu_id in individus_without_foyer_fiscal[:]:
# Tente d'affecter l'individu à un foyer fiscal d'après son ménage.
menage, menage_role = find_menage_and_role(test_case, individu_id)
if menage_role == u'personne_de_reference':
conjoint_id = menage[u'conjoint']
if conjoint_id is not None:
foyer_fiscal, other_role = find_foyer_fiscal_and_role(test_case, conjoint_id)
if other_role == u'declarants' and len(foyer_fiscal[u'declarants']) == 1:
# Quand l'individu n'est pas encore dans un foyer fiscal, mais qu'il est personne de
# référence dans un ménage, qu'il y a un conjoint dans ce ménage et que ce
# conjoint est seul déclarant dans un foyer fiscal, alors ajoute l'individu comme
# autre déclarant de ce foyer fiscal.
foyer_fiscal[u'declarants'].append(individu_id)
individus_without_foyer_fiscal.remove(individu_id)
elif menage_role == u'conjoint':
personne_de_reference_id = menage[u'personne_de_reference']
if personne_de_reference_id is not None:
foyer_fiscal, other_role = find_foyer_fiscal_and_role(test_case, personne_de_reference_id)
if other_role == u'declarants' and len(foyer_fiscal[u'declarants']) == 1:
# Quand l'individu n'est pas encore dans un foyer fiscal, mais qu'il est conjoint
# dans un ménage, qu'il y a une personne de référence dans ce ménage et que
# cette personne est seul déclarant dans un foyer fiscal, alors ajoute l'individu
# comme autre déclarant de ce foyer fiscal.
foyer_fiscal[u'declarants'].append(individu_id)
individus_without_foyer_fiscal.remove(individu_id)
elif menage_role == u'enfants' and (
menage['personne_de_reference'] is not None or menage[u'conjoint'] is not None):
for other_id in (menage['personne_de_reference'], menage[u'conjoint']):
if other_id is None:
continue
foyer_fiscal, other_role = find_foyer_fiscal_and_role(test_case, other_id)
if other_role == u'declarants':
# Quand l'individu n'est pas encore dans un foyer fiscal, mais qu'il est enfant dans
# un ménage, qu'il y a une personne à charge ou un conjoint dans ce ménage et que
# celui-ci est déclarant dans un foyer fiscal, alors ajoute l'individu comme
# personne à charge de ce foyer fiscal.
foyer_fiscal[u'personnes_a_charge'].append(individu_id)
individus_without_foyer_fiscal.remove(individu_id)
break
if individu_id in individus_without_foyer_fiscal:
# L'individu n'est toujours pas affecté à un foyer fiscal.
individu = individu_by_id[individu_id]
age = find_age(individu, period.start.date)
if len(new_foyer_fiscal[u'declarants']) < 2 and (age is None or age >= 18):
new_foyer_fiscal[u'declarants'].append(individu_id)
else:
new_foyer_fiscal[u'personnes_a_charge'].append(individu_id)
if new_foyer_fiscal_id is None:
new_foyer_fiscal[u'id'] = new_foyer_fiscal_id = unicode(uuid.uuid4())
test_case[u'foyers_fiscaux'].append(new_foyer_fiscal)
individus_without_foyer_fiscal.remove(individu_id)
# Affecte à un ménage chaque individu qui n'appartient à aucun d'entre eux.
new_menage = dict(
autres = [],
conjoint = None,
enfants = [],
personne_de_reference = None,
)
new_menage_id = None
for individu_id in menages_individus_id[:]:
# Tente d'affecter l'individu à un ménage d'après son foyer fiscal.
foyer_fiscal, foyer_fiscal_role = find_foyer_fiscal_and_role(test_case, individu_id)
if foyer_fiscal_role == u'declarants' and len(foyer_fiscal[u'declarants']) == 2:
for declarant_id in foyer_fiscal[u'declarants']:
if declarant_id != individu_id:
menage, other_role = find_menage_and_role(test_case, declarant_id)
if other_role == u'personne_de_reference' and menage[u'conjoint'] is None:
# Quand l'individu n'est pas encore dans un ménage, mais qu'il est déclarant
# dans un foyer fiscal, qu'il y a un autre déclarant dans ce foyer fiscal et que
# cet autre déclarant est personne de référence dans un ménage et qu'il n'y a
# pas de conjoint dans ce ménage, alors ajoute l'individu comme conjoint de ce
# ménage.
menage[u'conjoint'] = individu_id
menages_individus_id.remove(individu_id)
elif other_role == u'conjoint' and menage[u'personne_de_reference'] is None:
# Quand l'individu n'est pas encore dans un ménage, mais qu'il est déclarant
# dans une foyer fiscal, qu'il y a un autre déclarant dans ce foyer fiscal et
# que cet autre déclarant est conjoint dans un ménage et qu'il n'y a pas de
# personne de référence dans ce ménage, alors ajoute l'individu comme personne
# de référence de ce ménage.
menage[u'personne_de_reference'] = individu_id
menages_individus_id.remove(individu_id)
break
elif foyer_fiscal_role == u'personnes_a_charge' and foyer_fiscal[u'declarants']:
for declarant_id in foyer_fiscal[u'declarants']:
menage, other_role = find_menage_and_role(test_case, declarant_id)
if other_role in (u'personne_de_reference', u'conjoint'):
# Quand l'individu n'est pas encore dans un ménage, mais qu'il est personne à charge
# dans un foyer fiscal, qu'il y a un déclarant dans ce foyer fiscal et que ce
# déclarant est personne de référence ou conjoint dans un ménage, alors ajoute
# l'individu comme enfant de ce ménage.
menage[u'enfants'].append(individu_id)
menages_individus_id.remove(individu_id)
break
if individu_id in menages_individus_id:
# L'individu n'est toujours pas affecté à un ménage.
if new_menage[u'personne_de_reference'] is None:
new_menage[u'personne_de_reference'] = individu_id
elif new_menage[u'conjoint'] is None:
new_menage[u'conjoint'] = individu_id
else:
new_menage[u'enfants'].append(individu_id)
if new_menage_id is None:
new_menage[u'id'] = new_menage_id = unicode(uuid.uuid4())
test_case[u'menages'].append(new_menage)
menages_individus_id.remove(individu_id)
remaining_individus_id = set(individus_without_foyer_fiscal).union(menages_individus_id)
if remaining_individus_id:
individu_index_by_id = {
individu[u'id']: individu_index
for individu_index, individu in enumerate(test_case[u'individus'])
}
if error is None:
error = {}
for individu_id in remaining_individus_id:
error.setdefault('individus', {})[individu_index_by_id[individu_id]] = state._(
u"Individual is missing from {}").format(
state._(u' & ').join(
word
for word in [
u'foyers_fiscaux' if individu_id in individus_without_foyer_fiscal else None,
u'menages' if individu_id in menages_individus_id else None,
]
if word is not None
))
if error is not None:
return test_case, error
# Third validation step
individu_by_id = test_case['individus']
test_case, error = conv.struct(
dict(
foyers_fiscaux = conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(
declarants = conv.pipe(
conv.empty_to_none,
conv.not_none,
conv.test(
lambda declarants: len(declarants) <= 2,
error = N_(u'A "foyer_fiscal" must have at most 2 "declarants"'),
),
# conv.uniform_sequence(conv.pipe(
# conv.test(lambda individu_id:
# find_age(individu_by_id[individu_id], period.start.date,
# default = 100) >= 18,
# error = u"Un déclarant d'un foyer fiscal doit être agé d'au moins 18"
# u" ans",
# ),
# conv.test(
# lambda individu_id: individu_id in parents_id,
# error = u"Un déclarant ou un conjoint sur la déclaration d'impôt, doit"
# u" être un parent dans sa famille",
# ),
# )),
),
personnes_a_charge = conv.uniform_sequence(
conv.test(
lambda individu_id:
individu_by_id[individu_id].get('invalide', False)
or find_age(individu_by_id[individu_id], period.start.date,
default = 0) < 25,
error = u"Une personne à charge d'un foyer fiscal doit avoir moins de"
u" 25 ans ou être invalide",
),
),
),
default = conv.noop,
),
),
conv.empty_to_none,
conv.not_none,
),
# individus = conv.uniform_sequence(
# conv.struct(
# dict(
# date_naissance = conv.test(
# lambda date_naissance: period.start.date - date_naissance >= datetime.timedelta(0),
# error = u"L'individu doit être né au plus tard le jour de la simulation",
# ),
# ),
# default = conv.noop,
# drop_none_values = 'missing',
# ),
# ),
menages = conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(
personne_de_reference = conv.not_none,
),
default = conv.noop,
),
),
conv.empty_to_none,
conv.not_none,
),
),
default = conv.noop,
)(test_case, state = state)
return test_case, error
return json_or_python_to_test_case
def json_or_python_to_test_case(value, state=None):
if value is None:
return value, None
if state is None:
state = conv.default_state
column_by_name = self.tax_benefit_system.column_by_name
# First validation and conversion step
test_case, error = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
individus=conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(conv.test_isinstance(dict), drop_none_items=True),
conv.function(scenarios.set_entities_json_id),
conv.uniform_sequence(
conv.struct(
dict(
itertools.chain(
dict(
id=conv.pipe(conv.test_isinstance((basestring, int)), conv.not_none)
).iteritems(),
(
(column.name, column.json_to_python)
for column in column_by_name.itervalues()
if column.entity == "ind" and column.name not in ("idmen", "quimen")
),
)
),
drop_none_values=True,
),
drop_none_items=True,
),
conv.empty_to_none,
conv.not_none,
),
menages=conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(conv.test_isinstance(dict), drop_none_items=True),
conv.function(scenarios.set_entities_json_id),
conv.uniform_sequence(
conv.struct(
dict(
itertools.chain(
dict(
autres=conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)), drop_none_items=True
),
conv.default([]),
),
conjoint=conv.pipe(
conv.test_isinstance(basestring, int), conv.default(None)
),
enfants=conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)), drop_none_items=True
),
conv.default([]),
),
id=conv.pipe(conv.test_isinstance((basestring, int)), conv.not_none),
personne_de_reference=conv.pipe(
conv.test_isinstance(basestring, int), conv.default(None)
),
).iteritems(),
(
(column.name, column.json_to_python)
for column in column_by_name.itervalues()
if column.entity == "men"
),
)
),
drop_none_values=True,
),
drop_none_items=True,
),
conv.default({}),
),
)
),
)(value, state=state)
if error is not None:
return test_case, error
# Second validation step
menages_individus_id = [individu["id"] for individu in test_case["individus"]]
test_case, error = conv.struct(
dict(
menages=conv.uniform_sequence(
conv.struct(
dict(
autres=conv.uniform_sequence(conv.test_in_pop(menages_individus_id)),
conjoint=conv.test_in_pop(menages_individus_id),
enfants=conv.uniform_sequence(conv.test_in_pop(menages_individus_id)),
personne_de_reference=conv.test_in_pop(menages_individus_id),
),
default=conv.noop,
)
)
),
default=conv.noop,
)(test_case, state=state)
remaining_individus_id = set(menages_individus_id)
if remaining_individus_id:
individu_index_by_id = {
individu[u"id"]: individu_index for individu_index, individu in enumerate(test_case[u"individus"])
}
if error is None:
error = {}
for individu_id in remaining_individus_id:
error.setdefault("individus", {})[individu_index_by_id[individu_id]] = state._(
u"Individual is missing from {}"
).format(
state._(u" & ").join(
word
for word in [u"menages" if individu_id in menages_individus_id else None]
if word is not None
)
)
if error is not None:
return test_case, error
return test_case, error
def validate_slice_xml_json(slice, state = None):
if slice is None:
return None, None
state = conv.add_ancestor_to_state(state, slice)
validated_slice, errors = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
ASSIETTE = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_values_holder_xml_json,
drop_none_items = True,
),
conv.empty_to_none,
conv.test(lambda l: len(l) == 1, error = N_(u"List must contain one and only one item")),
),
code = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
),
MONTANT = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_values_holder_xml_json,
drop_none_items = True,
),
conv.empty_to_none,
conv.test(lambda l: len(l) == 1, error = N_(u"List must contain one and only one item")),
),
SEUIL = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_values_holder_xml_json,
drop_none_items = True,
),
conv.empty_to_none,
conv.test(lambda l: len(l) == 1, error = N_(u"List must contain one and only one item")),
conv.not_none,
),
tail = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
TAUX = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_values_holder_xml_json,
drop_none_items = True,
),
conv.empty_to_none,
conv.test(lambda l: len(l) == 1, error = N_(u"List must contain one and only one item")),
),
text = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
),
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
),
conv.test(lambda slice: bool(slice.get('MONTANT')) ^ bool(slice.get('TAUX')),
error = N_(u"Either MONTANT or TAUX must be provided")),
)(slice, state = state)
conv.remove_ancestor_from_state(state, slice)
return validated_slice, errors
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
),
)(value, state = state)
conv.remove_ancestor_from_state(state, value)
return validated_value, errors
validate_values_holder_xml_json = conv.struct(
dict(
VALUE = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_value_xml_json,
drop_none_items = True,
),
make_validate_values_xml_json_dates(require_consecutive_dates = False),
conv.empty_to_none,
conv.not_none,
),
),
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
)
def xml_legislation_to_json(xml_element, state = None):
if xml_element is None:
return None, None
json_key, json_element = translate_xml_element_to_json_item(xml_element)
def json_to_dated_python(self):
return conv.pipe(conv.make_anything_to_int(accept_expression=True), )
def validate_node_xml_json(node, state = None):
if node is None:
return None, None
state = conv.add_ancestor_to_state(state, node)
validated_node, errors = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
BAREME = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_scale_xml_json,
drop_none_items = True,
),
conv.empty_to_none,
),
CODE = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_parameter_xml_json,
drop_none_items = True,
),
conv.empty_to_none,
),
code = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
conv.not_none,
),
description = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
),
NODE = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_node_xml_json,
drop_none_items = True,
),
conv.empty_to_none,
),
tail = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
text = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
),
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
),
)(node, state = state)
if errors is None:
children_groups_key = ('BAREME', 'CODE', 'NODE', 'VALBYTRANCHES')
if all(
validated_node.get(key) is None
for key in children_groups_key
):
error = state._(u"At least one of the following items must be present: {}").format(state._(u', ').join(
u'"{}"'.format(key)
for key in children_groups_key
))
errors = dict(
(key, error)
for key in children_groups_key
)
else:
errors = {}
children_code = set()
for key in children_groups_key:
for child_index, child in enumerate(validated_node.get(key) or []):
child_code = child['code']
if child_code in children_code:
errors.setdefault(key, {}).setdefault(child_index, {})['code'] = state._(u"Duplicate value")
else:
children_code.add(child_code)
conv.remove_ancestor_from_state(state, node)
return validated_node, errors or None
def input_to_period_tuple(value, state=None):
"""Convert an input string to a period tuple.
.. note:: This function doesn't return a period, but a tuple that allows to construct a period.
>>> input_to_period_tuple('2014')
(('year', 2014), None)
>>> input_to_period_tuple('2014:2')
(('year', 2014, 2), None)
>>> input_to_period_tuple('2014-2')
(('month', (2014, 2)), None)
>>> input_to_period_tuple('2014-3:12')
(('month', (2014, 3), 12), None)
>>> input_to_period_tuple('2014-2-3')
(('day', (2014, 2, 3)), None)
>>> input_to_period_tuple('2014-3-4:2')
(('day', (2014, 3, 4), 2), None)
>>> input_to_period_tuple('year:2014')
(('year', '2014'), None)
>>> input_to_period_tuple('year:2014:2')
(('year', '2014', '2'), None)
>>> input_to_period_tuple('year:2014-2:2')
(('year', '2014-2', '2'), None)
"""
if value is None:
return value, None
if state is None:
state = conv.default_state
split_value = tuple(
clean_fragment for clean_fragment in (fragment.strip()
for fragment in value.split(':'))
if clean_fragment)
if not split_value:
return None, None
if len(split_value) == 1:
split_value = tuple(
clean_fragment
for clean_fragment in (fragment.strip()
for fragment in split_value[0].split('-'))
if clean_fragment)
if len(split_value) == 1:
return conv.pipe(
conv.input_to_strict_int,
conv.test_greater_or_equal(0),
conv.function(lambda year: ('year', year)),
)(split_value[0], state=state)
if len(split_value) == 2:
return conv.pipe(
conv.struct((
conv.pipe(
conv.input_to_strict_int,
conv.test_greater_or_equal(0),
),
conv.pipe(
conv.input_to_strict_int,
conv.test_between(1, 12),
),
), ),
conv.function(lambda month_tuple: ('month', month_tuple)),
)(split_value, state=state)
if len(split_value) == 3:
return conv.pipe(
conv.struct((
conv.pipe(
conv.input_to_strict_int,
conv.test_greater_or_equal(0),
),
conv.pipe(
conv.input_to_strict_int,
conv.test_between(1, 12),
),
conv.pipe(
conv.input_to_strict_int,
conv.test_between(1, 31),
),
), ),
conv.function(lambda day_tuple: ('day', day_tuple)),
)(split_value, state=state)
return split_value, state._(
'Instant string contains too much "-" for a year, a month or a day'
)
if len(split_value) == 2:
split_start = tuple(
clean_fragment
for clean_fragment in (fragment.strip()
for fragment in split_value[0].split('-'))
if clean_fragment)
size, error = conv.input_to_int(split_value[1], state=state)
if error is None:
if len(split_start) == 1:
start, error = conv.pipe(
conv.input_to_strict_int,
conv.test_greater_or_equal(0),
)(split_start[0], state=state)
if error is None:
return ('year', start, size), None
elif len(split_start) == 2:
start, error = conv.struct((
conv.pipe(
conv.input_to_strict_int,
conv.test_greater_or_equal(0),
),
conv.pipe(
conv.input_to_strict_int,
conv.test_between(1, 12),
),
), )(split_start, state=state)
if error is None:
return ('month', start, size), None
elif len(split_start) == 3:
start, error = conv.struct((
conv.pipe(
conv.input_to_strict_int,
conv.test_greater_or_equal(0),
),
conv.pipe(
conv.input_to_strict_int,
conv.test_between(1, 12),
),
conv.pipe(
conv.input_to_strict_int,
conv.test_between(1, 31),
),
), )(split_start, state=state)
if error is None:
return ('day', start, size), None
return split_value, None
def post_process_test_case(self, test_case, period, state):
individu_by_id = {
individu['id']: individu
for individu in test_case['individus']
}
parents_id = set(
parent_id
for foyer_fiscal in test_case['foyers_fiscaux']
for parent_id in foyer_fiscal['declarants']
)
test_case, error = conv.struct(
dict(
# foyers_fiscaux = conv.pipe(
# conv.uniform_sequence(
# conv.struct(
# dict(
# enfants = conv.uniform_sequence(
# conv.test(
# lambda individu_id:
# individu_by_id[individu_id].get('invalide', False)
# or find_age(individu_by_id[individu_id], period.start.date,
# default = 0) <= 25,
# error = u"Une personne à charge d'un foyer fiscal doit avoir moins de"
# u" 25 ans ou être handicapée",
# ),
# ),
# parents = conv.pipe(
# conv.empty_to_none,
# conv.not_none,
# conv.test(lambda parents: len(parents) <= 2,
# error = N_(u'A "famille" must have at most 2 "parents"'))
# ),
# ),
# default = conv.noop,
# ),
# ),
# conv.empty_to_none,
# conv.not_none,
# ),
foyers_fiscaux = conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(
declarants = conv.pipe(
conv.empty_to_none,
conv.not_none,
conv.test(
lambda declarants: len(declarants) <= 2,
error = N_(u'A "foyer_fiscal" must have at most 2 "declarants"'),
),
conv.uniform_sequence(conv.pipe(
)),
),
personnes_a_charge = conv.uniform_sequence(
conv.test(
lambda individu_id:
individu_by_id[individu_id].get('handicap', False)
or find_age(individu_by_id[individu_id], period.start.date,
default = 0) <= 25,
error = u"Une personne à charge d'un foyer fiscal doit avoir moins de"
u" 25 ans ou être handicapée",
),
),
),
default = conv.noop,
),
),
conv.empty_to_none,
conv.not_none,
),
menages = conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(
personne_de_reference = conv.not_none,
),
default = conv.noop,
),
),
conv.empty_to_none,
conv.not_none,
),
),
default = conv.noop,
)(test_case, state = state)
return test_case, error
# First validation and conversion step
test_case, error = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
foyers_fiscaux = conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items = True,
),
conv.uniform_sequence(
conv.struct(
dict(itertools.chain(
dict(
declarants = conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.default([]),
),
id = conv.pipe(
conv.test_isinstance((basestring, int)),
conv.not_none,
),
personnes_a_charge = conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.default([]),
),
).iteritems(),
)),
drop_none_values = True,
),
drop_none_items = True,
),
conv.default([]),
),
menages = conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items = True,
),
conv.uniform_sequence(
conv.struct(
dict(itertools.chain(
dict(
autres = conv.pipe(
# personnes ayant un lien autre avec la personne de référence
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.default([]),
),
# conjoint de la personne de référence
conjoint = conv.test_isinstance((basestring, int)),
enfants = conv.pipe(
# enfants de la personne de référence ou de son conjoint
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.default([]),
),
id = conv.pipe(
conv.test_isinstance((basestring, int)),
conv.not_none,
),
personne_de_reference = conv.test_isinstance((basestring, int)),
).iteritems(),
)),
drop_none_values = True,
),
drop_none_items = True,
),
conv.default([]),
),
),
),
)(test_case, state = state)
# test_case, error = conv.struct(
# dict(
# foyers_fiscaux = conv.uniform_sequence(
# conv.struct(
# dict(
# declarants = conv.uniform_sequence(conv.test_in_pop(foyers_fiscaux_individus_id)),
# personnes_a_charge = conv.uniform_sequence(conv.test_in_pop(
# foyers_fiscaux_individus_id)),
# ),
# default = conv.noop,
# ),
# ),
# menages = conv.uniform_sequence(
# conv.struct(
# dict(
# autres = conv.uniform_sequence(conv.test_in_pop(menages_individus_id)),
# conjoint = conv.test_in_pop(menages_individus_id),
# enfants = conv.uniform_sequence(conv.test_in_pop(menages_individus_id)),
# personne_de_reference = conv.test_in_pop(menages_individus_id),
# ),
# default = conv.noop,
# ),
# ),
# ),
# default = conv.noop,
# )(test_case, state = state)
return test_case, error