def make_json_to_array_by_period(self, period):
return conv.condition(
conv.test_isinstance(dict),
conv.pipe(
# Value is a dict of (period, value) couples.
conv.uniform_mapping(
conv.pipe(
conv.function(periods.period),
conv.not_none,
),
conv.pipe(
conv.make_item_to_singleton(),
conv.uniform_sequence(self.json_to_dated_python, ),
conv.empty_to_none,
conv.function(lambda cells_list: np.array(
cells_list, dtype=self.dtype)),
),
drop_none_values=True,
),
conv.empty_to_none,
),
conv.pipe(
conv.make_item_to_singleton(),
conv.uniform_sequence(self.json_to_dated_python, ),
conv.empty_to_none,
conv.function(
lambda cells_list: np.array(cells_list, dtype=self.dtype)),
conv.function(lambda array: {period: array}),
),
)
def validate_node_json(node, state=None):
if node is None:
return None, None
if state is None:
state = conv.default_state
validated_node, errors = conv.pipe(
conv.condition(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_node_json,
drop_none_items=True,
),
conv.pipe(
conv.condition(
conv.test_isinstance(basestring_type),
conv.function(lambda code: dict(code=code)),
conv.test_isinstance(dict),
),
conv.struct(
dict(
children=conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_node_json,
drop_none_items=True,
),
conv.empty_to_none,
),
code=conv.pipe(
conv.test_isinstance(basestring_type),
conv.cleanup_line,
),
),
constructor=collections.OrderedDict,
default=conv.noop,
drop_none_values='missing',
keep_value_order=True,
),
),
),
conv.empty_to_none,
)(node, state=state)
if validated_node is None or errors is not None:
return validated_node, errors
if isinstance(validated_node,
dict) and not validated_node.get('children'):
validated_node, errors = conv.struct(
dict(code=conv.pipe(
conv.test_in(tax_benefit_system.variables),
conv.not_none,
), ),
default=conv.noop,
)(validated_node, state=state)
return validated_node, errors
def validate_parameter_xml_json(parameter, state = None):
if parameter is None:
return None, None
state = conv.add_ancestor_to_state(state, parameter)
validated_parameter, errors = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
code = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
conv.not_none,
),
description = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
),
format = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in(xml_json_formats),
),
tail = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
taille = conv.pipe(
conv.test_isinstance(basestring),
conv.test_in([
'moinsde20',
'plusde20',
]),
),
text = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
type = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in(json_unit_by_xml_json_type),
),
VALUE = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_value_xml_json,
drop_none_items = True,
),
make_validate_values_xml_json_dates(require_consecutive_dates = True),
conv.empty_to_none,
conv.not_none,
),
),
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
),
)(parameter, state = state)
conv.remove_ancestor_from_state(state, parameter)
return validated_parameter, errors
def _parse_yaml_file(tax_benefit_system, yaml_path):
filename = os.path.splitext(os.path.basename(yaml_path))[0]
with open(yaml_path) as yaml_file:
tests = yaml.load(yaml_file)
tests, error = conv.pipe(
conv.make_item_to_singleton(),
conv.uniform_sequence(
conv.noop,
drop_none_items = True,
),
)(tests)
if error is not None:
embedding_error = conv.embed_error(tests, u'errors', error)
assert embedding_error is None, embedding_error
raise ValueError("Error in test {}:\n{}".format(yaml_path, yaml.dump(tests, allow_unicode = True,
default_flow_style = False, indent = 2, width = 120)))
for test in tests:
test, error = scenarios.make_json_or_python_to_test(
tax_benefit_system = tax_benefit_system
)(test)
if error is not None:
embedding_error = conv.embed_error(test, u'errors', error)
assert embedding_error is None, embedding_error
raise ValueError("Error in test {}:\n{}\nYaml test content: \n{}\n".format(
yaml_path, error, yaml.dump(test, allow_unicode = True,
default_flow_style = False, indent = 2, width = 120)))
yield yaml_path, test.get('name') or filename, unicode(test['scenario'].period), test
def test():
dir_path = os.path.join(os.path.dirname(__file__), 'formulas')
for filename in sorted(os.listdir(dir_path)):
if not filename.endswith('.yaml'):
continue
filename_core = os.path.splitext(filename)[0]
with open(os.path.join(dir_path, filename)) as yaml_file:
tests = yaml.load(yaml_file)
tests, error = conv.pipe(
conv.make_item_to_singleton(),
conv.uniform_sequence(
conv.noop,
drop_none_items = True,
),
)(tests)
if error is not None:
embedding_error = conv.embed_error(tests, u'errors', error)
assert embedding_error is None, embedding_error
conv.check((tests, error)) # Generate an error.
for test in tests:
test, error = scenarios.make_json_or_python_to_test(tax_benefit_system)(test)
if error is not None:
embedding_error = conv.embed_error(test, u'errors', error)
assert embedding_error is None, embedding_error
conv.check((test, error)) # Generate an error.
if test.get(u'ignore', False):
continue
yield check, test.get('name') or filename_core, unicode(test['scenario'].period), test
def validate_scale_xml_json(scale, state = None):
if scale is None:
return None, None
state = conv.add_ancestor_to_state(state, scale)
validated_scale, errors = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
code = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
conv.not_none,
),
description = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
),
option = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in((
'contrib',
'main-d-oeuvre',
'noncontrib',
)),
),
TRANCHE = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_slice_xml_json,
drop_none_items = True,
),
validate_slices_xml_json_dates,
conv.empty_to_none,
conv.not_none,
),
tail = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
text = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
type = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in((
'monetary',
)),
),
),
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
),
)(scale, state = state)
conv.remove_ancestor_from_state(state, scale)
return validated_scale, errors
def test(force = False, name_filter = None, options_by_path = None):
if isinstance(name_filter, str):
name_filter = name_filter.decode('utf-8')
if options_by_path is None:
options_by_path = options_by_dir
for path, options in options_by_path.iteritems():
if not force and options.get('ignore', False):
log.info(u'Ignoring {}'.format(path))
continue
if not os.path.exists(path):
log.warning(u'Skipping missing {}'.format(path))
continue
if os.path.isdir(path):
yaml_paths = [
os.path.join(path, filename)
for filename in sorted(os.listdir(path))
if filename.endswith('.yaml')
]
else:
yaml_paths = [path]
for yaml_path in yaml_paths:
filename_core = os.path.splitext(os.path.basename(yaml_path))[0]
with open(yaml_path) as yaml_file:
tests = yaml.load(yaml_file)
tests, error = conv.pipe(
conv.make_item_to_singleton(),
conv.uniform_sequence(
conv.noop,
drop_none_items = True,
),
)(tests)
if error is not None:
embedding_error = conv.embed_error(tests, u'errors', error)
assert embedding_error is None, embedding_error
raise ValueError("Error in test {}:\n{}".format(yaml_path, yaml.dump(tests, allow_unicode = True,
default_flow_style = False, indent = 2, width = 120)))
for test in tests:
test, error = scenarios.make_json_or_python_to_test(get_tax_benefit_system(options.get('reform')),
default_absolute_error_margin = options['default_absolute_error_margin'])(test)
if error is not None:
embedding_error = conv.embed_error(test, u'errors', error)
assert embedding_error is None, embedding_error
raise ValueError("Error in test {}:\n{}".format(yaml_path, yaml.dump(test, allow_unicode = True,
default_flow_style = False, indent = 2, width = 120)))
if not force and test.get(u'ignore', False):
continue
if name_filter is not None and name_filter not in filename_core \
and name_filter not in (test.get('name', u'')) \
and name_filter not in (test.get('keywords', [])):
continue
checker = check_any_period if options['accept_other_period'] else check
yield checker, test.get('name') or filename_core, unicode(test['scenario'].period), test, force
def make_json_or_python_to_axes(tax_benefit_system):
column_by_name = tax_benefit_system.variables
return conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.pipe(
conv.item_or_sequence(
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
count=conv.pipe(
conv.test_isinstance(int),
conv.test_greater_or_equal(1),
conv.not_none,
),
index=conv.pipe(
conv.test_isinstance(int),
conv.test_greater_or_equal(0),
conv.default(0),
),
max=conv.pipe(
conv.test_isinstance((float, int)),
conv.not_none,
),
min=conv.pipe(
conv.test_isinstance((float, int)),
conv.not_none,
),
name=conv.pipe(
conv.test_isinstance(basestring_type),
conv.test_in(column_by_name),
conv.test(
lambda column_name: tax_benefit_system.
get_variable(column_name).dtype in
(np.float32, np.int16, np.int32),
error=N_(
'Invalid type for axe: integer or float expected'
)),
conv.not_none,
),
period=conv.function(periods.period),
), ),
),
drop_none_items=True,
),
conv.make_item_to_singleton(),
),
drop_none_items=True,
),
conv.empty_to_none,
)
def check_entities_and_role(test_case, tax_benefit_system, state):
"""
Check that the test_case describes entities consistent with the tax and benefit system.
Will raise an error if :
- An entity is not recognized
- An entity role is not recognized
- A variable is declared for an entity it is not defined for (e.g. salary for a family)
"""
test_case = deepcopy(
test_case) # Avoid side-effects on other references to test_case
entity_classes = {
entity_class.plural: entity_class
for entity_class in tax_benefit_system.entities
}
for entity_type_name, entities in test_case.items():
if entity_classes.get(entity_type_name) is None:
raise ValueError("Invalid entity name: {}".format(
entity_type_name).encode('utf-8'))
entities, error = conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items=True,
),
conv.function(set_entities_json_id),
)(entities)
if error is not None:
raise ValueError("Invalid list of {}: {}. Error: {}".format(
entity_type_name, entities, error).encode('utf-8'))
if entities is None:
entities = test_case[entity_type_name] = [
] # YAML test runner may set these values to None
entity_class = entity_classes[entity_type_name]
valid_roles = dict(
(role.key, role) if (role.max == 1) else (role.plural, role) for
role in entity_class.roles) if not entity_class.is_person else {}
for entity_json in entities:
check_entity_fields(entity_json, entity_class, valid_roles,
tax_benefit_system)
for entity_class in entity_classes.values():
if test_case.get(entity_class.plural) is None:
test_case[entity_class.plural] = [
] # by convention, all entities must be declared in the test_case
return test_case
def _parse_test_file(tax_benefit_system, yaml_path):
filename = os.path.splitext(os.path.basename(yaml_path))[0]
with open(yaml_path) as yaml_file:
try:
tests = yaml.load(yaml_file, Loader=Loader)
except yaml.scanner.ScannerError:
log.error("{} is not a valid YAML file".format(yaml_path).encode('utf-8'))
raise
tests, error = conv.pipe(
conv.make_item_to_singleton(),
conv.uniform_sequence(
conv.noop,
drop_none_items = True,
),
)(tests)
if error is not None:
embedding_error = conv.embed_error(tests, 'errors', error)
assert embedding_error is None, embedding_error
raise ValueError("Error in test {}:\n{}".format(yaml_path, yaml.dump(tests, Dumper=Dumper, allow_unicode = True,
default_flow_style = False, indent = 2, width = 120)))
for test in tests:
current_tax_benefit_system = tax_benefit_system
if test.get('reforms'):
reforms = test.pop('reforms')
if not isinstance(reforms, list):
reforms = [reforms]
for reform_path in reforms:
current_tax_benefit_system = current_tax_benefit_system.apply_reform(reform_path)
try:
test, error = scenarios.make_json_or_python_to_test(
tax_benefit_system = current_tax_benefit_system
)(test)
except Exception:
log.error("{} is not a valid OpenFisca test file".format(yaml_path).encode('utf-8'))
raise
if error is not None:
embedding_error = conv.embed_error(test, 'errors', error)
assert embedding_error is None, embedding_error
raise ValueError("Error in test {}:\n{}\nYaml test content: \n{}\n".format(
yaml_path, error, yaml.dump(test, Dumper=Dumper, allow_unicode = True,
default_flow_style = False, indent = 2, width = 120)))
yield yaml_path, test.get('name') or filename, to_unicode(test['scenario'].period), test
def check_role(value, key):
role = valid_roles.get(key)
if role.max == 1:
value, error = conv.test_isinstance((basestring_type, int))(value)
else:
value, error = conv.pipe(
conv.make_item_to_singleton(), conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring_type, int)),
drop_none_items=True,
))(value)
if error is not None:
raise ValueError("Invalid description of {}: {}. Error: {}".format(
entity_class.key, entity_json, error).encode('utf-8'))
entity_json[key] = value
def test(current_options_by_dir = None, force = False, name_filter = None):
if current_options_by_dir is None:
current_options_by_dir = options_by_dir
for dir, options in sorted(current_options_by_dir.iteritems()):
if not force and options.get('ignore', False):
log.info(u'Ignoring directory: {}'.format(dir))
continue
if not os.path.isdir(dir):
log.warning(u'Skipping missing directory: {}'.format(dir))
continue
if isinstance(name_filter, str):
name_filter = name_filter.decode('utf-8')
for filename in sorted(os.listdir(dir)):
if not filename.endswith('.yaml'):
continue
filename_core = os.path.splitext(filename)[0]
with open(os.path.join(dir, filename)) as yaml_file:
tests = yaml.load(yaml_file)
tests, error = conv.pipe(
conv.make_item_to_singleton(),
conv.uniform_sequence(
conv.noop,
drop_none_items = True,
),
)(tests)
if error is not None:
embedding_error = conv.embed_error(tests, u'errors', error)
assert embedding_error is None, embedding_error
conv.check((tests, error)) # Generate an error.
for test in tests:
test, error = scenarios.make_json_or_python_to_test(get_tax_benefit_system(options.get('reform')),
default_absolute_error_margin = options['default_absolute_error_margin'])(test)
if error is not None:
embedding_error = conv.embed_error(test, u'errors', error)
assert embedding_error is None, embedding_error
conv.check((test, error)) # Generate an error.
if not force and test.get(u'ignore', False):
continue
if name_filter is not None and name_filter not in filename_core \
and name_filter not in (test.get('name', u'')) \
and name_filter not in (test.get('keywords', [])):
continue
checker = check_any_period if options['accept_other_period'] else check
yield checker, test.get('name') or filename_core, unicode(test['scenario'].period), test, force
def test(force = False, name_filter = None):
if isinstance(name_filter, str):
name_filter = name_filter.decode('utf-8')
yaml_paths = [
os.path.join(source_file_dir_name, filename)
for filename in sorted(os.listdir(source_file_dir_name))
if filename.endswith('.yaml')
]
for yaml_path in yaml_paths:
filename_core = os.path.splitext(os.path.basename(yaml_path))[0]
with open(yaml_path) as yaml_file:
tests = yaml.load(yaml_file)
tests, error = conv.pipe(
conv.make_item_to_singleton(),
conv.uniform_sequence(
conv.noop,
drop_none_items = True,
),
)(tests)
if error is not None:
embedding_error = conv.embed_error(tests, u'errors', error)
assert embedding_error is None, embedding_error
raise ValueError("Error in test {}:\n{}".format(yaml_path, yaml.dump(tests, allow_unicode = True,
default_flow_style = False, indent = 2, width = 120)))
for test in tests:
test, error = scenarios.make_json_or_python_to_test(
tax_benefit_system = tax_benefit_system,
default_absolute_error_margin = default_absolute_error_margin,
)(test)
if error is not None:
embedding_error = conv.embed_error(test, u'errors', error)
assert embedding_error is None, embedding_error
raise ValueError("Error in test {}:\n{}\nYaml test content: \n{}\n".format(
yaml_path, error, yaml.dump(test, allow_unicode = True,
default_flow_style = False, indent = 2, width = 120)))
if not force and test.get(u'ignore', False):
continue
if name_filter is not None and name_filter not in filename_core \
and name_filter not in (test.get('name', u'')) \
and name_filter not in (test.get('keywords', [])):
continue
yield check, yaml_path, test.get('name') or filename_core, unicode(test['scenario'].period), test, force
def check_entities_consistency(test_case, tax_benefit_system, state):
"""
Checks that every person belongs to at most one entity of each kind
"""
remaining_persons = {
entity.plural: [
person['id']
for person in test_case[tax_benefit_system.person_entity.plural]
]
for entity in tax_benefit_system.group_entities
}
def build_role_checker(role, entity):
if role.max == 1:
return role.key, conv.test_in_pop(remaining_persons[entity.plural])
else:
return role.plural, conv.uniform_sequence(
conv.test_in_pop(remaining_persons[entity.plural]))
entity_parsing_dict = {
entity.plural: conv.uniform_sequence(
conv.struct(
dict(
build_role_checker(role, entity) for role in entity.roles),
default=conv.noop,
), )
for entity in tax_benefit_system.group_entities
}
test_case, error = conv.struct(
entity_parsing_dict,
default=conv.noop,
)(test_case, state=state)
return test_case, error, remaining_persons
def post_process_test_case(self, test_case, period, state):
individu_by_id = {
individu['id']: individu
for individu in test_case['individus']
}
parents_id = set(parent_id for famille in test_case['familles']
for parent_id in famille['parents'])
test_case, error = conv.struct(
dict(
familles=conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(
enfants=conv.uniform_sequence(
conv.test(
lambda individu_id:
individu_by_id[individu_id].get(
'handicap', False) or find_age(
individu_by_id[individu_id],
period.start.date,
default=0) <= 25,
error=
u"Une personne à charge d'un foyer fiscal doit avoir moins de"
u" 25 ans ou être handicapée",
), ),
parents=conv.pipe(
conv.empty_to_none, conv.not_none,
conv.test(
lambda parents: len(parents) <= 2,
error=N_(
u'A "famille" must have at most 2 "parents"'
))),
),
default=conv.noop,
), ),
conv.empty_to_none,
conv.not_none,
),
foyers_fiscaux=conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(
declarants=conv.pipe(
conv.empty_to_none,
conv.not_none,
conv.test(
lambda declarants: len(declarants) <=
2,
error=N_(
u'A "foyer_fiscal" must have at most 2 "declarants"'
),
),
conv.uniform_sequence(conv.pipe()),
),
personnes_a_charge=conv.uniform_sequence(
conv.test(
lambda individu_id:
individu_by_id[individu_id].get(
'handicap', False) or find_age(
individu_by_id[individu_id],
period.start.date,
default=0) <= 25,
error=
u"Une personne à charge d'un foyer fiscal doit avoir moins de"
u" 25 ans ou être handicapée",
), ),
),
default=conv.noop,
), ),
conv.empty_to_none,
conv.not_none,
),
menages=conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(personne_de_reference=conv.not_none, ),
default=conv.noop,
), ),
conv.empty_to_none,
conv.not_none,
),
),
default=conv.noop,
)(test_case, state=state)
return test_case, error
def test(force = False, name_filter = None, options_by_path = None):
if isinstance(name_filter, str):
name_filter = name_filter.decode('utf-8')
if options_by_path is None:
options_by_path = options_by_dir
for path, options in options_by_path.iteritems():
if not force and options.get('ignore', False):
log.info(u'Ignoring {}'.format(path))
continue
if not os.path.exists(path):
log.warning(u'Skipping missing {}'.format(path))
continue
if os.path.isdir(path):
yaml_paths = [
os.path.join(path, filename)
for filename in sorted(os.listdir(path))
if filename.endswith('.yaml')
]
else:
yaml_paths = [path]
if options.get('requires'):
# Check if the required package was successfully imported in tests/base.py
if getattr(base, options.get('requires')) is None:
continue
reform_keys = options.get('reforms')
tax_benefit_system_for_path = base.get_cached_composed_reform(
reform_keys = reform_keys,
tax_benefit_system = base.tax_benefit_system,
) if reform_keys is not None else base.tax_benefit_system
for yaml_path in yaml_paths:
filename_core = os.path.splitext(os.path.basename(yaml_path))[0]
with open(yaml_path) as yaml_file:
tests = yaml.load(yaml_file)
tests, error = conv.pipe(
conv.make_item_to_singleton(),
conv.uniform_sequence(
conv.noop,
drop_none_items = True,
),
)(tests)
if error is not None:
embedding_error = conv.embed_error(tests, u'errors', error)
assert embedding_error is None, embedding_error
raise ValueError("Error in test {}:\n{}".format(yaml_path, yaml.dump(tests, allow_unicode = True,
default_flow_style = False, indent = 2, width = 120)))
for test in tests:
test, error = scenarios.make_json_or_python_to_test(
tax_benefit_system = tax_benefit_system_for_path,
default_absolute_error_margin = options.get('default_absolute_error_margin'),
default_relative_error_margin = options.get('default_relative_error_margin'),
)(test)
if error is not None:
embedding_error = conv.embed_error(test, u'errors', error)
assert embedding_error is None, embedding_error
raise ValueError("Error in test {}:\n{}\nYaml test content: \n{}\n".format(
yaml_path, error, yaml.dump(test, allow_unicode = True,
default_flow_style = False, indent = 2, width = 120)))
if not force and test.get(u'ignore', False):
continue
if name_filter is not None and name_filter not in filename_core \
and name_filter not in (test.get('name', u'')) \
and name_filter not in (test.get('keywords', [])):
continue
checker = check_calculate_output if options['calculate_output'] else check
yield checker, yaml_path, test.get('name') or filename_core, unicode(test['scenario'].period), test, \
force
def test(force=False, name_filter=None, options_by_path=None):
if isinstance(name_filter, str):
name_filter = name_filter.decode('utf-8')
if options_by_path is None:
options_by_path = options_by_dir
for path, options in options_by_path.iteritems():
if not force and options.get('ignore', False):
log.info(u'Ignoring {}'.format(path))
continue
if not os.path.exists(path):
log.warning(u'Skipping missing {}'.format(path))
continue
if os.path.isdir(path):
yaml_paths = [
os.path.join(path, filename)
for filename in sorted(os.listdir(path))
if filename.endswith('.yaml')
]
else:
yaml_paths = [path]
for yaml_path in yaml_paths:
filename_core = os.path.splitext(os.path.basename(yaml_path))[0]
with open(yaml_path) as yaml_file:
tests = yaml.load(yaml_file)
tests, error = conv.pipe(
conv.make_item_to_singleton(),
conv.uniform_sequence(
conv.noop,
drop_none_items=True,
),
)(tests)
if error is not None:
embedding_error = conv.embed_error(tests, u'errors', error)
assert embedding_error is None, embedding_error
raise ValueError("Error in test {}:\n{}".format(
yaml_path,
yaml.dump(tests,
allow_unicode=True,
default_flow_style=False,
indent=2,
width=120)))
for test in tests:
test, error = scenarios.make_json_or_python_to_test(
get_tax_benefit_system(options.get('reform')),
default_absolute_error_margin=options[
'default_absolute_error_margin'])(test)
if error is not None:
embedding_error = conv.embed_error(test, u'errors', error)
assert embedding_error is None, embedding_error
raise ValueError("Error in test {}:\n{}".format(
yaml_path,
yaml.dump(test,
allow_unicode=True,
default_flow_style=False,
indent=2,
width=120)))
if not force and test.get(u'ignore', False):
continue
if name_filter is not None and name_filter not in filename_core \
and name_filter not in (test.get('name', u'')) \
and name_filter not in (test.get('keywords', [])):
continue
checker = check_any_period if options[
'accept_other_period'] else check
yield checker, test.get('name') or filename_core, unicode(
test['scenario'].period), test, force
def validate_node_json(node, state=None):
if node is None:
return None, None
state = conv.add_ancestor_to_state(state, node)
validated_node, error = conv.test_isinstance(dict)(node, state=state)
if error is not None:
conv.remove_ancestor_from_state(state, node)
return validated_node, error
validated_node, errors = conv.struct(
{
'@context':
conv.pipe(
conv.test_isinstance(basestring),
conv.make_input_to_url(full=True),
conv.test_equals(
u'http://openfisca.fr/contexts/legislation.jsonld'),
),
'@type':
conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
conv.test_in((u'Node', u'Parameter', u'Scale')),
conv.not_none,
),
'comment':
conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
'description':
conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
),
},
constructor=collections.OrderedDict,
default=conv.noop,
drop_none_values='missing',
keep_value_order=True,
)(validated_node, state=state)
if errors is not None:
conv.remove_ancestor_from_state(state, node)
return validated_node, errors
validated_node.pop(
'@context', None
) # Remove optional @context everywhere. It will be added to root node later.
node_converters = {
'@type': conv.noop,
'comment': conv.noop,
'description': conv.noop,
}
node_type = validated_node['@type']
if node_type == u'Node':
node_converters.update(
dict(children=conv.pipe(
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
conv.not_none,
),
conv.pipe(
validate_node_json,
conv.not_none,
),
),
conv.empty_to_none,
conv.not_none,
), ))
elif node_type == u'Parameter':
node_converters.update(
dict(
format=conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in([
'boolean',
'float',
'integer',
'rate',
]),
),
unit=conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in(units),
),
values=conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_value_json,
drop_none_items=True,
),
make_validate_values_json_dates(
require_consecutive_dates=True),
conv.empty_to_none,
conv.not_none,
),
))
else:
assert node_type == u'Scale'
node_converters.update(
dict(
option=conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in((
'contrib',
'main-d-oeuvre',
'noncontrib',
)),
),
slices=conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_slice_json,
drop_none_items=True,
),
validate_slices_json_dates,
conv.empty_to_none,
conv.not_none,
),
unit=conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in(('currency', )),
),
))
validated_node, errors = conv.struct(
node_converters,
constructor=collections.OrderedDict,
drop_none_values='missing',
keep_value_order=True,
)(validated_node, state=state)
conv.remove_ancestor_from_state(state, node)
return validated_node, errors
def json_or_python_to_test_case(value, state = None):
if value is None:
return value, None
if state is None:
state = conv.default_state
column_by_name = self.tax_benefit_system.column_by_name
# First validation and conversion step
test_case, error = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
familles = conv.pipe(
conv.condition(
conv.test_isinstance(list),
conv.pipe(
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items = True,
),
conv.function(lambda values: collections.OrderedDict(
(value.pop('id', index), value)
for index, value in enumerate(values)
)),
),
),
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(
conv.test_isinstance((basestring, int)),
conv.not_none,
),
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(itertools.chain(
dict(
enfants = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.default([]),
),
parents = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.default([]),
),
).iteritems(),
(
(column.name, column.json_to_python)
for column in column_by_name.itervalues()
if column.entity == 'fam'
),
)),
drop_none_values = True,
),
),
drop_none_values = True,
),
conv.default({}),
),
individus = conv.pipe(
conv.condition(
conv.test_isinstance(list),
conv.pipe(
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items = True,
),
conv.function(lambda values: collections.OrderedDict(
(value.pop('id', index), value)
for index, value in enumerate(values)
)),
),
),
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(
conv.test_isinstance((basestring, int)),
conv.not_none,
),
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
(column.name, column.json_to_python)
for column in column_by_name.itervalues()
if column.entity == 'ind' and column.name not in (
'idfam', 'idfoy', 'idmen', 'quifam', 'quifoy', 'quimen')
),
drop_none_values = True,
),
),
drop_none_values = True,
),
conv.empty_to_none,
conv.not_none,
),
),
),
)(value, state = state)
if error is not None:
return test_case, error
# Second validation step
familles_individus_id = list(test_case['individus'].iterkeys())
test_case, error = conv.struct(
dict(
familles = conv.uniform_mapping(
conv.noop,
conv.struct(
dict(
enfants = conv.uniform_sequence(conv.test_in_pop(familles_individus_id)),
parents = conv.uniform_sequence(conv.test_in_pop(familles_individus_id)),
),
default = conv.noop,
),
),
),
default = conv.noop,
)(test_case, state = state)
remaining_individus_id = set(familles_individus_id)
if remaining_individus_id:
if error is None:
error = {}
for individu_id in remaining_individus_id:
error.setdefault('individus', {})[individu_id] = state._(u"Individual is missing from {}").format(
state._(u' & ').join(
word
for word in [
u'familles' if individu_id in familles_individus_id else None,
]
if word is not None
))
if error is not None:
return test_case, error
return test_case, error
def validate_node_xml_json(node, state=None):
validated_node, errors = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
code=conv.pipe(
conv.test_isinstance(basestring_type),
conv.cleanup_line,
conv.not_none,
),
color=conv.pipe(
conv.test_isinstance(basestring_type),
conv.function(lambda colors: colors.split(',')),
conv.uniform_sequence(
conv.pipe(
conv.input_to_int,
conv.test_between(0, 255),
conv.not_none,
), ),
conv.test(
lambda colors: len(colors) == 3,
error=N_('Wrong number of colors in triplet.')),
conv.function(lambda colors: ','.join(
to_unicode(color) for color in colors)),
),
desc=conv.pipe(
conv.test_isinstance(basestring_type),
conv.cleanup_line,
conv.not_none,
),
NODE=conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_node_xml_json,
drop_none_items=True,
),
conv.empty_to_none,
),
shortname=conv.pipe(
conv.test_isinstance(basestring_type),
conv.cleanup_line,
conv.not_none,
),
tail=conv.pipe(
conv.test_isinstance(basestring_type),
conv.cleanup_text,
),
text=conv.pipe(
conv.test_isinstance(basestring_type),
conv.cleanup_text,
),
typevar=conv.pipe(
conv.test_isinstance(basestring_type),
conv.input_to_int,
conv.test_equals(2),
),
),
constructor=collections.OrderedDict,
drop_none_values='missing',
keep_value_order=True,
),
)(node, state=state or conv.default_state)
if errors is not None:
return validated_node, errors
if not validated_node.get('NODE'):
validated_node, errors = conv.struct(
dict(code=conv.test_in(tax_benefit_system.variables), ),
default=conv.noop,
)(validated_node, state=state)
return validated_node, errors
def test(force = False, name_filter = None, options_by_path = None):
if isinstance(name_filter, str):
name_filter = name_filter.decode('utf-8')
if options_by_path is None:
options_by_path = options_by_dir
for path, options in options_by_path.iteritems():
if not force and options.get('ignore', False):
log.info(u'Ignoring {}'.format(path))
continue
if not os.path.exists(path):
log.warning(u'Skipping missing {}'.format(path))
continue
if os.path.isdir(path):
yaml_paths = [
os.path.join(path, filename)
for filename in sorted(os.listdir(path))
if filename.endswith('.yaml')
]
else:
yaml_paths = [path]
if options.get('requires'):
# Check if the required package was successfully imported in tests/base.py
if getattr(base, options.get('requires')) is None:
continue
reform_keys = options.get('reforms')
tax_benefit_system_for_path = base.get_cached_composed_reform(
reform_keys = reform_keys,
tax_benefit_system = base.tax_benefit_system,
) if reform_keys is not None else base.tax_benefit_system
for yaml_path in yaml_paths:
filename_core = os.path.splitext(os.path.basename(yaml_path))[0]
with open(yaml_path) as yaml_file:
tests = yaml.load(yaml_file)
tests, error = conv.pipe(
conv.make_item_to_singleton(),
conv.uniform_sequence(
conv.noop,
drop_none_items = True,
),
)(tests)
if error is not None:
embedding_error = conv.embed_error(tests, u'errors', error)
assert embedding_error is None, embedding_error
raise ValueError("Error in test {}:\n{}".format(yaml_path, yaml.dump(tests, allow_unicode = True,
default_flow_style = False, indent = 2, width = 120)))
for test in tests:
test, error = scenarios.make_json_or_python_to_test(
tax_benefit_system = tax_benefit_system_for_path,
default_absolute_error_margin = options.get('default_absolute_error_margin'),
default_relative_error_margin = options.get('default_relative_error_margin'),
)(test)
if error is not None:
embedding_error = conv.embed_error(test, u'errors', error)
assert embedding_error is None, embedding_error
raise ValueError("Error in test {}:\n{}\nYaml test content: \n{}\n".format(
yaml_path, error, yaml.dump(test, allow_unicode = True,
default_flow_style = False, indent = 2, width = 120)))
if not force and test.get(u'ignore', False):
continue
if name_filter is not None and name_filter not in filename_core \
and name_filter not in (test.get('name', u'')) \
and name_filter not in (test.get('keywords', [])):
continue
checker = check_calculate_output if options['calculate_output'] else check
yield checker, yaml_path, test.get('name') or filename_core, unicode(test['scenario'].period), test, \
force
def validate_slice_xml_json(slice, state = None):
if slice is None:
return None, None
state = conv.add_ancestor_to_state(state, slice)
validated_slice, errors = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
ASSIETTE = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_values_holder_xml_json,
drop_none_items = True,
),
conv.empty_to_none,
conv.test(lambda l: len(l) == 1, error = N_(u"List must contain one and only one item")),
),
code = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
),
MONTANT = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_values_holder_xml_json,
drop_none_items = True,
),
conv.empty_to_none,
conv.test(lambda l: len(l) == 1, error = N_(u"List must contain one and only one item")),
),
SEUIL = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_values_holder_xml_json,
drop_none_items = True,
),
conv.empty_to_none,
conv.test(lambda l: len(l) == 1, error = N_(u"List must contain one and only one item")),
conv.not_none,
),
tail = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
TAUX = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_values_holder_xml_json,
drop_none_items = True,
),
conv.empty_to_none,
conv.test(lambda l: len(l) == 1, error = N_(u"List must contain one and only one item")),
),
text = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
),
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
),
conv.test(lambda slice: bool(slice.get('MONTANT')) ^ bool(slice.get('TAUX')),
error = N_(u"Either MONTANT or TAUX must be provided")),
)(slice, state = state)
conv.remove_ancestor_from_state(state, slice)
return validated_slice, errors
def json_or_python_to_test_case(value, state=None):
if value is None:
return value, None
if state is None:
state = conv.default_state
column_by_name = self.tax_benefit_system.column_by_name
# First validation and conversion step
test_case, error = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
individus=conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(conv.test_isinstance(dict), drop_none_items=True),
conv.function(scenarios.set_entities_json_id),
conv.uniform_sequence(
conv.struct(
dict(
itertools.chain(
dict(
id=conv.pipe(conv.test_isinstance((basestring, int)), conv.not_none)
).iteritems(),
(
(column.name, column.json_to_python)
for column in column_by_name.itervalues()
if column.entity == "ind" and column.name not in ("idmen", "quimen")
),
)
),
drop_none_values=True,
),
drop_none_items=True,
),
conv.empty_to_none,
conv.not_none,
),
menages=conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(conv.test_isinstance(dict), drop_none_items=True),
conv.function(scenarios.set_entities_json_id),
conv.uniform_sequence(
conv.struct(
dict(
itertools.chain(
dict(
autres=conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)), drop_none_items=True
),
conv.default([]),
),
conjoint=conv.pipe(
conv.test_isinstance(basestring, int), conv.default(None)
),
enfants=conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)), drop_none_items=True
),
conv.default([]),
),
id=conv.pipe(conv.test_isinstance((basestring, int)), conv.not_none),
personne_de_reference=conv.pipe(
conv.test_isinstance(basestring, int), conv.default(None)
),
).iteritems(),
(
(column.name, column.json_to_python)
for column in column_by_name.itervalues()
if column.entity == "men"
),
)
),
drop_none_values=True,
),
drop_none_items=True,
),
conv.default({}),
),
)
),
)(value, state=state)
if error is not None:
return test_case, error
# Second validation step
menages_individus_id = [individu["id"] for individu in test_case["individus"]]
test_case, error = conv.struct(
dict(
menages=conv.uniform_sequence(
conv.struct(
dict(
autres=conv.uniform_sequence(conv.test_in_pop(menages_individus_id)),
conjoint=conv.test_in_pop(menages_individus_id),
enfants=conv.uniform_sequence(conv.test_in_pop(menages_individus_id)),
personne_de_reference=conv.test_in_pop(menages_individus_id),
),
default=conv.noop,
)
)
),
default=conv.noop,
)(test_case, state=state)
remaining_individus_id = set(menages_individus_id)
if remaining_individus_id:
individu_index_by_id = {
individu[u"id"]: individu_index for individu_index, individu in enumerate(test_case[u"individus"])
}
if error is None:
error = {}
for individu_id in remaining_individus_id:
error.setdefault("individus", {})[individu_index_by_id[individu_id]] = state._(
u"Individual is missing from {}"
).format(
state._(u" & ").join(
word
for word in [u"menages" if individu_id in menages_individus_id else None]
if word is not None
)
)
if error is not None:
return test_case, error
return test_case, error
def attribute_groupless_persons_to_entities(self, test_case, period, groupless_individus):
individus_without_menage = groupless_individus['menages']
individus_without_foyer_fiscal = groupless_individus['foyers_fiscaux']
individu_by_id = {
individu['id']: individu
for individu in test_case['individus']
}
# Affecte à un foyer fiscal chaque individu qui n'appartient à aucun d'entre eux.
new_foyer_fiscal = dict(
declarants = [],
personnes_a_charge = [],
)
new_foyer_fiscal_id = None
for individu_id in individus_without_foyer_fiscal[:]:
# Tente d'affecter l'individu à un foyer fiscal d'après son ménage.
menage, menage_role = find_menage_and_role(test_case, individu_id)
if menage_role == u'personne_de_reference':
conjoint_id = menage[u'conjoint']
if conjoint_id is not None:
foyer_fiscal, other_role = find_foyer_fiscal_and_role(test_case, conjoint_id)
if other_role == u'declarants' and len(foyer_fiscal[u'declarants']) == 1:
# Quand l'individu n'est pas encore dans un foyer fiscal, mais qu'il est personne de
# référence dans un ménage, qu'il y a un conjoint dans ce ménage et que ce
# conjoint est seul déclarant dans un foyer fiscal, alors ajoute l'individu comme
# autre déclarant de ce foyer fiscal.
foyer_fiscal[u'declarants'].append(individu_id)
individus_without_foyer_fiscal.remove(individu_id)
elif menage_role == u'conjoint':
personne_de_reference_id = menage[u'personne_de_reference']
if personne_de_reference_id is not None:
foyer_fiscal, other_role = find_foyer_fiscal_and_role(test_case, personne_de_reference_id)
if other_role == u'declarants' and len(foyer_fiscal[u'declarants']) == 1:
# Quand l'individu n'est pas encore dans un foyer fiscal, mais qu'il est conjoint
# dans un ménage, qu'il y a une personne de référence dans ce ménage et que
# cette personne est seul déclarant dans un foyer fiscal, alors ajoute l'individu
# comme autre déclarant de ce foyer fiscal.
foyer_fiscal[u'declarants'].append(individu_id)
individus_without_foyer_fiscal.remove(individu_id)
elif menage_role == u'enfants' and (
menage['personne_de_reference'] is not None or menage[u'conjoint'] is not None):
for other_id in (menage['personne_de_reference'], menage[u'conjoint']):
if other_id is None:
continue
foyer_fiscal, other_role = find_foyer_fiscal_and_role(test_case, other_id)
if other_role == u'declarants':
# Quand l'individu n'est pas encore dans un foyer fiscal, mais qu'il est enfant dans
# un ménage, qu'il y a une personne à charge ou un conjoint dans ce ménage et que
# celui-ci est déclarant dans un foyer fiscal, alors ajoute l'individu comme
# personne à charge de ce foyer fiscal.
foyer_fiscal[u'personnes_a_charge'].append(individu_id)
individus_without_foyer_fiscal.remove(individu_id)
break
if individu_id in individus_without_foyer_fiscal:
# L'individu n'est toujours pas affecté à un foyer fiscal.
individu = individu_by_id[individu_id]
age = find_age(individu, period.start.date)
if len(new_foyer_fiscal[u'declarants']) < 2 and (age is None or age >= 18):
new_foyer_fiscal[u'declarants'].append(individu_id)
else:
new_foyer_fiscal[u'personnes_a_charge'].append(individu_id)
if new_foyer_fiscal_id is None:
new_foyer_fiscal[u'id'] = new_foyer_fiscal_id = unicode(uuid.uuid4())
test_case[u'foyers_fiscaux'].append(new_foyer_fiscal)
individus_without_foyer_fiscal.remove(individu_id)
# Affecte à un ménage chaque individu qui n'appartient à aucun d'entre eux.
new_menage = dict(
autres = [],
conjoint = None,
enfants = [],
personne_de_reference = None,
)
new_menage_id = None
for individu_id in menages_individus_id[:]:
# Tente d'affecter l'individu à un ménage d'après son foyer fiscal.
foyer_fiscal, foyer_fiscal_role = find_foyer_fiscal_and_role(test_case, individu_id)
if foyer_fiscal_role == u'declarants' and len(foyer_fiscal[u'declarants']) == 2:
for declarant_id in foyer_fiscal[u'declarants']:
if declarant_id != individu_id:
menage, other_role = find_menage_and_role(test_case, declarant_id)
if other_role == u'personne_de_reference' and menage[u'conjoint'] is None:
# Quand l'individu n'est pas encore dans un ménage, mais qu'il est déclarant
# dans un foyer fiscal, qu'il y a un autre déclarant dans ce foyer fiscal et que
# cet autre déclarant est personne de référence dans un ménage et qu'il n'y a
# pas de conjoint dans ce ménage, alors ajoute l'individu comme conjoint de ce
# ménage.
menage[u'conjoint'] = individu_id
menages_individus_id.remove(individu_id)
elif other_role == u'conjoint' and menage[u'personne_de_reference'] is None:
# Quand l'individu n'est pas encore dans un ménage, mais qu'il est déclarant
# dans une foyer fiscal, qu'il y a un autre déclarant dans ce foyer fiscal et
# que cet autre déclarant est conjoint dans un ménage et qu'il n'y a pas de
# personne de référence dans ce ménage, alors ajoute l'individu comme personne
# de référence de ce ménage.
menage[u'personne_de_reference'] = individu_id
menages_individus_id.remove(individu_id)
break
elif foyer_fiscal_role == u'personnes_a_charge' and foyer_fiscal[u'declarants']:
for declarant_id in foyer_fiscal[u'declarants']:
menage, other_role = find_menage_and_role(test_case, declarant_id)
if other_role in (u'personne_de_reference', u'conjoint'):
# Quand l'individu n'est pas encore dans un ménage, mais qu'il est personne à charge
# dans un foyer fiscal, qu'il y a un déclarant dans ce foyer fiscal et que ce
# déclarant est personne de référence ou conjoint dans un ménage, alors ajoute
# l'individu comme enfant de ce ménage.
menage[u'enfants'].append(individu_id)
menages_individus_id.remove(individu_id)
break
if individu_id in menages_individus_id:
# L'individu n'est toujours pas affecté à un ménage.
if new_menage[u'personne_de_reference'] is None:
new_menage[u'personne_de_reference'] = individu_id
elif new_menage[u'conjoint'] is None:
new_menage[u'conjoint'] = individu_id
else:
new_menage[u'enfants'].append(individu_id)
if new_menage_id is None:
new_menage[u'id'] = new_menage_id = unicode(uuid.uuid4())
test_case[u'menages'].append(new_menage)
menages_individus_id.remove(individu_id)
remaining_individus_id = set(individus_without_foyer_fiscal).union(menages_individus_id)
if remaining_individus_id:
individu_index_by_id = {
individu[u'id']: individu_index
for individu_index, individu in enumerate(test_case[u'individus'])
}
if error is None:
error = {}
for individu_id in remaining_individus_id:
error.setdefault('individus', {})[individu_index_by_id[individu_id]] = state._(
u"Individual is missing from {}").format(
state._(u' & ').join(
word
for word in [
u'foyers_fiscaux' if individu_id in individus_without_foyer_fiscal else None,
u'menages' if individu_id in menages_individus_id else None,
]
if word is not None
))
if error is not None:
return test_case, error
# Third validation step
individu_by_id = test_case['individus']
test_case, error = conv.struct(
dict(
foyers_fiscaux = conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(
declarants = conv.pipe(
conv.empty_to_none,
conv.not_none,
conv.test(
lambda declarants: len(declarants) <= 2,
error = N_(u'A "foyer_fiscal" must have at most 2 "declarants"'),
),
# conv.uniform_sequence(conv.pipe(
# conv.test(lambda individu_id:
# find_age(individu_by_id[individu_id], period.start.date,
# default = 100) >= 18,
# error = u"Un déclarant d'un foyer fiscal doit être agé d'au moins 18"
# u" ans",
# ),
# conv.test(
# lambda individu_id: individu_id in parents_id,
# error = u"Un déclarant ou un conjoint sur la déclaration d'impôt, doit"
# u" être un parent dans sa famille",
# ),
# )),
),
personnes_a_charge = conv.uniform_sequence(
conv.test(
lambda individu_id:
individu_by_id[individu_id].get('invalide', False)
or find_age(individu_by_id[individu_id], period.start.date,
default = 0) < 25,
error = u"Une personne à charge d'un foyer fiscal doit avoir moins de"
u" 25 ans ou être invalide",
),
),
),
default = conv.noop,
),
),
conv.empty_to_none,
conv.not_none,
),
# individus = conv.uniform_sequence(
# conv.struct(
# dict(
# date_naissance = conv.test(
# lambda date_naissance: period.start.date - date_naissance >= datetime.timedelta(0),
# error = u"L'individu doit être né au plus tard le jour de la simulation",
# ),
# ),
# default = conv.noop,
# drop_none_values = 'missing',
# ),
# ),
menages = conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(
personne_de_reference = conv.not_none,
),
default = conv.noop,
),
),
conv.empty_to_none,
conv.not_none,
),
),
default = conv.noop,
)(test_case, state = state)
return test_case, error
return json_or_python_to_test_case
def validate_node_xml_json(node, state = None):
if node is None:
return None, None
state = conv.add_ancestor_to_state(state, node)
validated_node, errors = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
BAREME = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_scale_xml_json,
drop_none_items = True,
),
conv.empty_to_none,
),
CODE = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_parameter_xml_json,
drop_none_items = True,
),
conv.empty_to_none,
),
code = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
conv.not_none,
),
description = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
),
NODE = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_node_xml_json,
drop_none_items = True,
),
conv.empty_to_none,
),
tail = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
text = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
),
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
),
)(node, state = state)
if errors is None:
children_groups_key = ('BAREME', 'CODE', 'NODE', 'VALBYTRANCHES')
if all(
validated_node.get(key) is None
for key in children_groups_key
):
error = state._(u"At least one of the following items must be present: {}").format(state._(u', ').join(
u'"{}"'.format(key)
for key in children_groups_key
))
errors = dict(
(key, error)
for key in children_groups_key
)
else:
errors = {}
children_code = set()
for key in children_groups_key:
for child_index, child in enumerate(validated_node.get(key) or []):
child_code = child['code']
if child_code in children_code:
errors.setdefault(key, {}).setdefault(child_index, {})['code'] = state._(u"Duplicate value")
else:
children_code.add(child_code)
conv.remove_ancestor_from_state(state, node)
return validated_node, errors or None
),
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
),
)(value, state = state)
conv.remove_ancestor_from_state(state, value)
return validated_value, errors
validate_values_holder_xml_json = conv.struct(
dict(
VALUE = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_value_xml_json,
drop_none_items = True,
),
make_validate_values_xml_json_dates(require_consecutive_dates = False),
conv.empty_to_none,
conv.not_none,
),
),
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
)
def xml_legislation_to_json(xml_element, state = None):
if xml_element is None:
return None, None
def json_or_python_to_test_case(value, state=None):
if value is None:
return value, None
if state is None:
state = conv.default_state
column_by_name = self.tax_benefit_system.column_by_name
# First validation and conversion step
test_case, error = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
foyers_fiscaux=conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items=True,
),
conv.function(scenarios.set_entities_json_id),
conv.uniform_sequence(
conv.struct(
dict(
itertools.chain(
dict(
declarants=conv.pipe(
conv.
make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(
(basestring, int)),
drop_none_items=True,
),
conv.default([]),
),
id=conv.pipe(
conv.test_isinstance(
(basestring, int)),
conv.not_none,
),
personnes_a_charge=conv.pipe(
conv.
make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(
(basestring, int)),
drop_none_items=True,
),
conv.default([]),
),
).iteritems(),
((column.name,
column.json_to_python) for column
in column_by_name.itervalues()
if column.entity == 'fam'),
)),
drop_none_values=True,
),
drop_none_items=True,
),
conv.default([]),
),
individus=conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items=True,
),
conv.function(scenarios.set_entities_json_id),
conv.uniform_sequence(
conv.struct(
dict(
itertools.chain(
dict(id=conv.pipe(
conv.test_isinstance(
(basestring, int)),
conv.not_none,
), ).iteritems(),
((column.name,
column.json_to_python) for column
in column_by_name.itervalues()
if column.entity == 'ind'
and column.name not in
('idfam', 'idfoy', 'idmen',
'quifam', 'quifoy', 'quimen')),
)),
drop_none_values=True,
),
drop_none_items=True,
),
conv.empty_to_none,
conv.not_none,
),
menages=conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items=True,
),
conv.function(scenarios.set_entities_json_id),
conv.uniform_sequence(
conv.struct(
dict(
itertools.chain(
dict(
autres=conv.pipe(
# personnes ayant un lien autre avec la personne de référence
conv.
make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(
(basestring, int)),
drop_none_items=True,
),
conv.default([]),
),
# conjoint de la personne de référence
conjoint=conv.test_isinstance(
(basestring, int)),
enfants=conv.pipe(
# enfants de la personne de référence ou de son conjoint
conv.
make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(
(basestring, int)),
drop_none_items=True,
),
conv.default([]),
),
id=conv.pipe(
conv.test_isinstance(
(basestring, int)),
conv.not_none,
),
personne_de_reference=conv.
test_isinstance(
(basestring, int)),
).iteritems(),
((column.name,
column.json_to_python) for column
in column_by_name.itervalues()
if column.entity == 'men'),
)),
drop_none_values=True,
),
drop_none_items=True,
),
conv.default([]),
),
), ),
)(value, state=state)
if error is not None:
return test_case, error
# Second validation step
foyers_fiscaux_individus_id = [
individu['id'] for individu in test_case['individus']
]
menages_individus_id = [
individu['id'] for individu in test_case['individus']
]
test_case, error = conv.struct(
dict(
foyers_fiscaux=conv.uniform_sequence(
conv.struct(
dict(
declarants=conv.uniform_sequence(
conv.test_in_pop(
foyers_fiscaux_individus_id)),
personnes_a_charge=conv.uniform_sequence(
conv.test_in_pop(
foyers_fiscaux_individus_id)),
),
default=conv.noop,
), ),
menages=conv.uniform_sequence(
conv.struct(
dict(
autres=conv.uniform_sequence(
conv.test_in_pop(menages_individus_id)),
conjoint=conv.test_in_pop(
menages_individus_id),
enfants=conv.uniform_sequence(
conv.test_in_pop(menages_individus_id)),
personne_de_reference=conv.test_in_pop(
menages_individus_id),
),
default=conv.noop,
), ),
),
default=conv.noop,
)(test_case, state=state)
individu_by_id = {
individu['id']: individu
for individu in test_case['individus']
}
if repair:
# Affecte à un foyer fiscal chaque individu qui n'appartient à aucun d'entre eux.
new_foyer_fiscal = dict(
declarants=[],
personnes_a_charge=[],
)
new_foyer_fiscal_id = None
for individu_id in foyers_fiscaux_individus_id[:]:
# Tente d'affecter l'individu à un foyer fiscal d'après son ménage.
menage, menage_role = find_menage_and_role(
test_case, individu_id)
if menage_role == u'personne_de_reference':
conjoint_id = menage[u'conjoint']
if conjoint_id is not None:
foyer_fiscal, other_role = find_foyer_fiscal_and_role(
test_case, conjoint_id)
if other_role == u'declarants' and len(
foyer_fiscal[u'declarants']) == 1:
# Quand l'individu n'est pas encore dans un foyer fiscal, mais qu'il est personne de
# référence dans un ménage, qu'il y a un conjoint dans ce ménage et que ce
# conjoint est seul déclarant dans un foyer fiscal, alors ajoute l'individu comme
# autre déclarant de ce foyer fiscal.
foyer_fiscal[u'declarants'].append(individu_id)
foyers_fiscaux_individus_id.remove(individu_id)
elif menage_role == u'conjoint':
personne_de_reference_id = menage[
u'personne_de_reference']
if personne_de_reference_id is not None:
foyer_fiscal, other_role = find_foyer_fiscal_and_role(
test_case, personne_de_reference_id)
if other_role == u'declarants' and len(
foyer_fiscal[u'declarants']) == 1:
# Quand l'individu n'est pas encore dans un foyer fiscal, mais qu'il est conjoint
# dans un ménage, qu'il y a une personne de référence dans ce ménage et que
# cette personne est seul déclarant dans un foyer fiscal, alors ajoute l'individu
# comme autre déclarant de ce foyer fiscal.
foyer_fiscal[u'declarants'].append(individu_id)
foyers_fiscaux_individus_id.remove(individu_id)
elif menage_role == u'enfants' and (
menage['personne_de_reference'] is not None
or menage[u'conjoint'] is not None):
for other_id in (menage['personne_de_reference'],
menage[u'conjoint']):
if other_id is None:
continue
foyer_fiscal, other_role = find_foyer_fiscal_and_role(
test_case, other_id)
if other_role == u'declarants':
# Quand l'individu n'est pas encore dans un foyer fiscal, mais qu'il est enfant dans
# un ménage, qu'il y a une personne à charge ou un conjoint dans ce ménage et que
# celui-ci est déclarant dans un foyer fiscal, alors ajoute l'individu comme
# personne à charge de ce foyer fiscal.
foyer_fiscal[u'personnes_a_charge'].append(
individu_id)
foyers_fiscaux_individus_id.remove(individu_id)
break
if individu_id in foyers_fiscaux_individus_id:
# L'individu n'est toujours pas affecté à un foyer fiscal.
individu = individu_by_id[individu_id]
age = find_age(individu, period.start.date)
if len(new_foyer_fiscal[u'declarants']) < 2 and (
age is None or age >= 18):
new_foyer_fiscal[u'declarants'].append(individu_id)
else:
new_foyer_fiscal[u'personnes_a_charge'].append(
individu_id)
if new_foyer_fiscal_id is None:
new_foyer_fiscal[
u'id'] = new_foyer_fiscal_id = unicode(
uuid.uuid4())
test_case[u'foyers_fiscaux'].append(
new_foyer_fiscal)
foyers_fiscaux_individus_id.remove(individu_id)
# Affecte à un ménage chaque individu qui n'appartient à aucun d'entre eux.
new_menage = dict(
autres=[],
conjoint=None,
enfants=[],
personne_de_reference=None,
)
new_menage_id = None
for individu_id in menages_individus_id[:]:
# Tente d'affecter l'individu à un ménage d'après son foyer fiscal.
foyer_fiscal, foyer_fiscal_role = find_foyer_fiscal_and_role(
test_case, individu_id)
if foyer_fiscal_role == u'declarants' and len(
foyer_fiscal[u'declarants']) == 2:
for declarant_id in foyer_fiscal[u'declarants']:
if declarant_id != individu_id:
menage, other_role = find_menage_and_role(
test_case, declarant_id)
if other_role == u'personne_de_reference' and menage[
u'conjoint'] is None:
# Quand l'individu n'est pas encore dans un ménage, mais qu'il est déclarant
# dans un foyer fiscal, qu'il y a un autre déclarant dans ce foyer fiscal et que
# cet autre déclarant est personne de référence dans un ménage et qu'il n'y a
# pas de conjoint dans ce ménage, alors ajoute l'individu comme conjoint de ce
# ménage.
menage[u'conjoint'] = individu_id
menages_individus_id.remove(individu_id)
elif other_role == u'conjoint' and menage[
u'personne_de_reference'] is None:
# Quand l'individu n'est pas encore dans un ménage, mais qu'il est déclarant
# dans une foyer fiscal, qu'il y a un autre déclarant dans ce foyer fiscal et
# que cet autre déclarant est conjoint dans un ménage et qu'il n'y a pas de
# personne de référence dans ce ménage, alors ajoute l'individu comme personne
# de référence de ce ménage.
menage[
u'personne_de_reference'] = individu_id
menages_individus_id.remove(individu_id)
break
elif foyer_fiscal_role == u'personnes_a_charge' and foyer_fiscal[
u'declarants']:
for declarant_id in foyer_fiscal[u'declarants']:
menage, other_role = find_menage_and_role(
test_case, declarant_id)
if other_role in (u'personne_de_reference',
u'conjoint'):
# Quand l'individu n'est pas encore dans un ménage, mais qu'il est personne à charge
# dans un foyer fiscal, qu'il y a un déclarant dans ce foyer fiscal et que ce
# déclarant est personne de référence ou conjoint dans un ménage, alors ajoute
# l'individu comme enfant de ce ménage.
menage[u'enfants'].append(individu_id)
menages_individus_id.remove(individu_id)
break
if individu_id in menages_individus_id:
# L'individu n'est toujours pas affecté à un ménage.
if new_menage[u'personne_de_reference'] is None:
new_menage[u'personne_de_reference'] = individu_id
elif new_menage[u'conjoint'] is None:
new_menage[u'conjoint'] = individu_id
else:
new_menage[u'enfants'].append(individu_id)
if new_menage_id is None:
new_menage[u'id'] = new_menage_id = unicode(
uuid.uuid4())
test_case[u'menages'].append(new_menage)
menages_individus_id.remove(individu_id)
remaining_individus_id = set(foyers_fiscaux_individus_id).union(
menages_individus_id)
if remaining_individus_id:
individu_index_by_id = {
individu[u'id']: individu_index
for individu_index, individu in enumerate(
test_case[u'individus'])
}
if error is None:
error = {}
for individu_id in remaining_individus_id:
error.setdefault(
'individus',
{})[individu_index_by_id[individu_id]] = state._(
u"Individual is missing from {}").format(
state._(u' & ').join(word for word in [
u'foyers_fiscaux' if individu_id in
foyers_fiscaux_individus_id else None,
u'menages' if individu_id in
menages_individus_id else None,
] if word is not None))
if error is not None:
return test_case, error
# Third validation step
individu_by_id = test_case['individus']
test_case, error = conv.struct(
dict(
foyers_fiscaux=conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(
declarants=conv.pipe(
conv.empty_to_none,
conv.not_none,
conv.test(
lambda declarants: len(declarants)
<= 2,
error=N_(
u'A "foyer_fiscal" must have at most 2 "declarants"'
),
),
# conv.uniform_sequence(conv.pipe(
# conv.test(lambda individu_id:
# find_age(individu_by_id[individu_id], period.start.date,
# default = 100) >= 18,
# error = u"Un déclarant d'un foyer fiscal doit être agé d'au moins 18"
# u" ans",
# ),
# conv.test(
# lambda individu_id: individu_id in parents_id,
# error = u"Un déclarant ou un conjoint sur la déclaration d'impôt, doit"
# u" être un parent dans sa famille",
# ),
# )),
),
personnes_a_charge=conv.uniform_sequence(
conv.test(
lambda individu_id: individu_by_id[
individu_id].get('inv', False)
or find_age(individu_by_id[
individu_id],
period.start.date,
default=0) < 25,
error=
u"Une personne à charge d'un foyer fiscal doit avoir moins de"
u" 25 ans ou être invalide",
), ),
),
default=conv.noop,
), ),
conv.empty_to_none,
conv.not_none,
),
# individus = conv.uniform_sequence(
# conv.struct(
# dict(
# date_naissance = conv.test(
# lambda date_naissance: period.start.date - date_naissance >= datetime.timedelta(0),
# error = u"L'individu doit être né au plus tard le jour de la simulation",
# ),
# ),
# default = conv.noop,
# drop_none_values = 'missing',
# ),
# ),
menages=conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(personne_de_reference=conv.not_none, ),
default=conv.noop,
), ),
conv.empty_to_none,
conv.not_none,
),
),
default=conv.noop,
)(test_case, state=state)
return test_case, error
def attribute_groupless_persons_to_entities(self, test_case, period,
groupless_individus):
individus_without_menage = groupless_individus['menages']
individus_without_foyer_fiscal = groupless_individus['foyers_fiscaux']
individu_by_id = {
individu['id']: individu
for individu in test_case['individus']
}
# Affecte à un foyer fiscal chaque individu qui n'appartient à aucun d'entre eux.
new_foyer_fiscal = dict(
declarants=[],
personnes_a_charge=[],
)
new_foyer_fiscal_id = None
for individu_id in individus_without_foyer_fiscal[:]:
# Tente d'affecter l'individu à un foyer fiscal d'après son ménage.
menage, menage_role = find_menage_and_role(test_case, individu_id)
if menage_role == u'personne_de_reference':
conjoint_id = menage[u'conjoint']
if conjoint_id is not None:
foyer_fiscal, other_role = find_foyer_fiscal_and_role(
test_case, conjoint_id)
if other_role == u'declarants' and len(
foyer_fiscal[u'declarants']) == 1:
# Quand l'individu n'est pas encore dans un foyer fiscal, mais qu'il est personne de
# référence dans un ménage, qu'il y a un conjoint dans ce ménage et que ce
# conjoint est seul déclarant dans un foyer fiscal, alors ajoute l'individu comme
# autre déclarant de ce foyer fiscal.
foyer_fiscal[u'declarants'].append(individu_id)
individus_without_foyer_fiscal.remove(individu_id)
elif menage_role == u'conjoint':
personne_de_reference_id = menage[u'personne_de_reference']
if personne_de_reference_id is not None:
foyer_fiscal, other_role = find_foyer_fiscal_and_role(
test_case, personne_de_reference_id)
if other_role == u'declarants' and len(
foyer_fiscal[u'declarants']) == 1:
# Quand l'individu n'est pas encore dans un foyer fiscal, mais qu'il est conjoint
# dans un ménage, qu'il y a une personne de référence dans ce ménage et que
# cette personne est seul déclarant dans un foyer fiscal, alors ajoute l'individu
# comme autre déclarant de ce foyer fiscal.
foyer_fiscal[u'declarants'].append(individu_id)
individus_without_foyer_fiscal.remove(individu_id)
elif menage_role == u'enfants' and (
menage['personne_de_reference'] is not None
or menage[u'conjoint'] is not None):
for other_id in (menage['personne_de_reference'],
menage[u'conjoint']):
if other_id is None:
continue
foyer_fiscal, other_role = find_foyer_fiscal_and_role(
test_case, other_id)
if other_role == u'declarants':
# Quand l'individu n'est pas encore dans un foyer fiscal, mais qu'il est enfant dans
# un ménage, qu'il y a une personne à charge ou un conjoint dans ce ménage et que
# celui-ci est déclarant dans un foyer fiscal, alors ajoute l'individu comme
# personne à charge de ce foyer fiscal.
foyer_fiscal[u'personnes_a_charge'].append(individu_id)
individus_without_foyer_fiscal.remove(individu_id)
break
if individu_id in individus_without_foyer_fiscal:
# L'individu n'est toujours pas affecté à un foyer fiscal.
individu = individu_by_id[individu_id]
age = find_age(individu, period.start.date)
if len(new_foyer_fiscal[u'declarants']) < 2 and (age is None
or age >= 18):
new_foyer_fiscal[u'declarants'].append(individu_id)
else:
new_foyer_fiscal[u'personnes_a_charge'].append(individu_id)
if new_foyer_fiscal_id is None:
new_foyer_fiscal[u'id'] = new_foyer_fiscal_id = unicode(
uuid.uuid4())
test_case[u'foyers_fiscaux'].append(new_foyer_fiscal)
individus_without_foyer_fiscal.remove(individu_id)
# Affecte à un ménage chaque individu qui n'appartient à aucun d'entre eux.
new_menage = dict(
autres=[],
conjoint=None,
enfants=[],
personne_de_reference=None,
)
new_menage_id = None
for individu_id in menages_individus_id[:]:
# Tente d'affecter l'individu à un ménage d'après son foyer fiscal.
foyer_fiscal, foyer_fiscal_role = find_foyer_fiscal_and_role(
test_case, individu_id)
if foyer_fiscal_role == u'declarants' and len(
foyer_fiscal[u'declarants']) == 2:
for declarant_id in foyer_fiscal[u'declarants']:
if declarant_id != individu_id:
menage, other_role = find_menage_and_role(
test_case, declarant_id)
if other_role == u'personne_de_reference' and menage[
u'conjoint'] is None:
# Quand l'individu n'est pas encore dans un ménage, mais qu'il est déclarant
# dans un foyer fiscal, qu'il y a un autre déclarant dans ce foyer fiscal et que
# cet autre déclarant est personne de référence dans un ménage et qu'il n'y a
# pas de conjoint dans ce ménage, alors ajoute l'individu comme conjoint de ce
# ménage.
menage[u'conjoint'] = individu_id
menages_individus_id.remove(individu_id)
elif other_role == u'conjoint' and menage[
u'personne_de_reference'] is None:
# Quand l'individu n'est pas encore dans un ménage, mais qu'il est déclarant
# dans une foyer fiscal, qu'il y a un autre déclarant dans ce foyer fiscal et
# que cet autre déclarant est conjoint dans un ménage et qu'il n'y a pas de
# personne de référence dans ce ménage, alors ajoute l'individu comme personne
# de référence de ce ménage.
menage[u'personne_de_reference'] = individu_id
menages_individus_id.remove(individu_id)
break
elif foyer_fiscal_role == u'personnes_a_charge' and foyer_fiscal[
u'declarants']:
for declarant_id in foyer_fiscal[u'declarants']:
menage, other_role = find_menage_and_role(
test_case, declarant_id)
if other_role in (u'personne_de_reference',
u'conjoint'):
# Quand l'individu n'est pas encore dans un ménage, mais qu'il est personne à charge
# dans un foyer fiscal, qu'il y a un déclarant dans ce foyer fiscal et que ce
# déclarant est personne de référence ou conjoint dans un ménage, alors ajoute
# l'individu comme enfant de ce ménage.
menage[u'enfants'].append(individu_id)
menages_individus_id.remove(individu_id)
break
if individu_id in menages_individus_id:
# L'individu n'est toujours pas affecté à un ménage.
if new_menage[u'personne_de_reference'] is None:
new_menage[u'personne_de_reference'] = individu_id
elif new_menage[u'conjoint'] is None:
new_menage[u'conjoint'] = individu_id
else:
new_menage[u'enfants'].append(individu_id)
if new_menage_id is None:
new_menage[u'id'] = new_menage_id = unicode(
uuid.uuid4())
test_case[u'menages'].append(new_menage)
menages_individus_id.remove(individu_id)
remaining_individus_id = set(individus_without_foyer_fiscal).union(
menages_individus_id)
if remaining_individus_id:
individu_index_by_id = {
individu[u'id']: individu_index
for individu_index, individu in enumerate(
test_case[u'individus'])
}
if error is None:
error = {}
for individu_id in remaining_individus_id:
error.setdefault(
'individus',
{})[individu_index_by_id[individu_id]] = state._(
u"Individual is missing from {}").format(
state._(u' & ').join(word for word in [
u'foyers_fiscaux' if individu_id in
individus_without_foyer_fiscal else None,
u'menages' if individu_id in
menages_individus_id else None,
] if word is not None))
if error is not None:
return test_case, error
# Third validation step
individu_by_id = test_case['individus']
test_case, error = conv.struct(
dict(
foyers_fiscaux=conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(
declarants=conv.pipe(
conv.empty_to_none,
conv.not_none,
conv.test(
lambda declarants: len(declarants)
<= 2,
error=N_(
u'A "foyer_fiscal" must have at most 2 "declarants"'
),
),
# conv.uniform_sequence(conv.pipe(
# conv.test(lambda individu_id:
# find_age(individu_by_id[individu_id], period.start.date,
# default = 100) >= 18,
# error = u"Un déclarant d'un foyer fiscal doit être agé d'au moins 18"
# u" ans",
# ),
# conv.test(
# lambda individu_id: individu_id in parents_id,
# error = u"Un déclarant ou un conjoint sur la déclaration d'impôt, doit"
# u" être un parent dans sa famille",
# ),
# )),
),
personnes_a_charge=conv.uniform_sequence(
conv.test(
lambda individu_id:
individu_by_id[individu_id].get(
'invalide', False) or find_age(
individu_by_id[individu_id
],
period.start.date,
default=0) < 25,
error=
u"Une personne à charge d'un foyer fiscal doit avoir moins de"
u" 25 ans ou être invalide",
), ),
),
default=conv.noop,
), ),
conv.empty_to_none,
conv.not_none,
),
# individus = conv.uniform_sequence(
# conv.struct(
# dict(
# date_naissance = conv.test(
# lambda date_naissance: period.start.date - date_naissance >= datetime.timedelta(0),
# error = u"L'individu doit être né au plus tard le jour de la simulation",
# ),
# ),
# default = conv.noop,
# drop_none_values = 'missing',
# ),
# ),
menages=conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(personne_de_reference=conv.not_none, ),
default=conv.noop,
), ),
conv.empty_to_none,
conv.not_none,
),
),
default=conv.noop,
)(test_case, state=state)
return test_case, error
return json_or_python_to_test_case
def json_or_python_to_test_case(value, state=None):
if value is None:
return value, None
if state is None:
state = conv.default_state
column_by_name = self.tax_benefit_system.column_by_name
# First validation and conversion step
test_case, error = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
individus=conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items=True,
),
conv.function(scenarios.set_entities_json_id),
conv.uniform_sequence(
conv.struct(
dict(
itertools.chain(
dict(id=conv.pipe(
conv.test_isinstance(
(basestring, int)),
conv.not_none,
), ).iteritems(),
((column.name,
column.json_to_python) for column
in column_by_name.itervalues()
if column.entity == 'ind' and
column.name not in ('idmen',
'quimen')),
)),
drop_none_values=True,
),
drop_none_items=True,
),
conv.empty_to_none,
conv.not_none,
),
menages=conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items=True,
),
conv.function(scenarios.set_entities_json_id),
conv.uniform_sequence(
conv.struct(
dict(
itertools.chain(
dict(
autres=conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(
(basestring, int)),
drop_none_items=True,
),
conv.default([]),
),
conjoint=conv.pipe(
conv.test_isinstance(
basestring, int),
conv.default(None),
),
enfants=conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(
(basestring, int)),
drop_none_items=True,
),
conv.default([]),
),
id=conv.pipe(
conv.test_isinstance(
(basestring, int)),
conv.not_none,
),
personne_de_reference=conv.
pipe(
conv.test_isinstance(
basestring, int),
conv.default(None),
),
).iteritems(),
((column.name,
column.json_to_python) for column
in column_by_name.itervalues()
if column.entity == 'men'),
)),
drop_none_values=True,
),
drop_none_items=True,
),
conv.default({}),
),
), ),
)(value, state=state)
if error is not None:
return test_case, error
# Second validation step
menages_individus_id = [
individu['id'] for individu in test_case['individus']
]
test_case, error = conv.struct(
dict(menages=conv.uniform_sequence(
conv.struct(
dict(
autres=conv.uniform_sequence(
conv.test_in_pop(menages_individus_id)),
conjoint=conv.test_in_pop(menages_individus_id),
enfants=conv.uniform_sequence(
conv.test_in_pop(menages_individus_id)),
personne_de_reference=conv.test_in_pop(
menages_individus_id),
),
default=conv.noop,
), ), ),
default=conv.noop,
)(test_case, state=state)
remaining_individus_id = set(menages_individus_id)
if remaining_individus_id:
individu_index_by_id = {
individu[u'id']: individu_index
for individu_index, individu in enumerate(
test_case[u'individus'])
}
if error is None:
error = {}
for individu_id in remaining_individus_id:
error.setdefault(
'individus',
{})[individu_index_by_id[individu_id]] = state._(
u"Individual is missing from {}").format(
state._(u' & ').join(word for word in [
u'menages' if individu_id in
menages_individus_id else None,
] if word is not None))
if error is not None:
return test_case, error
return test_case, error
def post_process_test_case(self, test_case, period, state):
individu_by_id = {
individu['id']: individu
for individu in test_case['individus']
}
parents_id = set(parent_id
for foyer_fiscal in test_case['foyers_fiscaux']
for parent_id in foyer_fiscal['declarants'])
test_case, error = conv.struct(
dict(
# foyers_fiscaux = conv.pipe(
# conv.uniform_sequence(
# conv.struct(
# dict(
# enfants = conv.uniform_sequence(
# conv.test(
# lambda individu_id:
# individu_by_id[individu_id].get('invalide', False)
# or find_age(individu_by_id[individu_id], period.start.date,
# default = 0) <= 25,
# error = u"Une personne à charge d'un foyer fiscal doit avoir moins de"
# u" 25 ans ou être handicapée",
# ),
# ),
# parents = conv.pipe(
# conv.empty_to_none,
# conv.not_none,
# conv.test(lambda parents: len(parents) <= 2,
# error = N_(u'A "famille" must have at most 2 "parents"'))
# ),
# ),
# default = conv.noop,
# ),
# ),
# conv.empty_to_none,
# conv.not_none,
# ),
foyers_fiscaux=conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(
declarants=conv.pipe(
conv.empty_to_none,
conv.not_none,
conv.test(
lambda declarants: len(declarants) <=
2,
error=N_(
u'A "foyer_fiscal" must have at most 2 "declarants"'
),
),
),
personnes_a_charge=conv.uniform_sequence(
conv.test(
lambda individu_id:
individu_by_id[individu_id].get(
'handicap', False) or find_age(
individu_by_id[individu_id],
period.start.date,
default=0) <= 25,
error=
u"Une personne à charge d'un foyer fiscal doit avoir moins de"
u" 25 ans ou être handicapée",
), ),
),
default=conv.noop,
), ),
conv.empty_to_none,
conv.not_none,
),
menages=conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(personne_de_reference=conv.not_none, ),
default=conv.noop,
), ),
conv.empty_to_none,
conv.not_none,
),
),
default=conv.noop,
)(test_case, state=state)
return test_case, error
# First validation and conversion step
test_case, error = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
foyers_fiscaux=conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items=True,
),
conv.uniform_sequence(
conv.struct(
dict(
itertools.chain(
dict(
declarants=conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(
(basestring, int)),
drop_none_items=True,
),
conv.default([]),
),
id=conv.pipe(
conv.test_isinstance(
(basestring, int)),
conv.not_none,
),
personnes_a_charge=conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(
(basestring, int)),
drop_none_items=True,
),
conv.default([]),
),
).iteritems(), )),
drop_none_values=True,
),
drop_none_items=True,
),
conv.default([]),
),
menages=conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items=True,
),
conv.uniform_sequence(
conv.struct(
dict(
itertools.chain(
dict(
autres=conv.pipe(
# personnes ayant un lien autre avec la personne de référence
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(
(basestring, int)),
drop_none_items=True,
),
conv.default([]),
),
# conjoint de la personne de référence
conjoint=conv.test_isinstance(
(basestring, int)),
enfants=conv.pipe(
# enfants de la personne de référence ou de son conjoint
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(
(basestring, int)),
drop_none_items=True,
),
conv.default([]),
),
id=conv.pipe(
conv.test_isinstance(
(basestring, int)),
conv.not_none,
),
personne_de_reference=conv.
test_isinstance((basestring, int)),
).iteritems(), )),
drop_none_values=True,
),
drop_none_items=True,
),
conv.default([]),
),
), ),
)(test_case, state=state)
return test_case, error
def post_process_test_case(self, test_case, period, state):
individu_by_id = {
individu['id']: individu
for individu in test_case['individus']
}
parents_id = set( # noqa F841
parent_id
for famille in test_case['familles']
for parent_id in famille['parents']
)
test_case, error = conv.struct(
dict(
familles = conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(
enfants = conv.uniform_sequence(
conv.test(
lambda individu_id:
individu_by_id[individu_id].get('handicap', False)
or find_age(individu_by_id[individu_id], period.start.date,
default = 0) <= 25,
error = u"Une personne à charge d'un foyer fiscal doit avoir moins de"
u" 25 ans ou être handicapée",
),
),
parents = conv.pipe(
conv.empty_to_none,
conv.not_none,
conv.test(lambda parents: len(parents) <= 2,
error = N_(u'A "famille" must have at most 2 "parents"'))
),
),
default = conv.noop,
),
),
conv.empty_to_none,
conv.not_none,
),
foyers_fiscaux = conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(
declarants = conv.pipe(
conv.empty_to_none,
conv.not_none,
conv.test(
lambda declarants: len(declarants) <= 2,
error = N_(u'A "foyer_fiscal" must have at most 2 "declarants"'),
),
conv.uniform_sequence(conv.pipe(
)),
),
personnes_a_charge = conv.uniform_sequence(
conv.test(
lambda individu_id:
individu_by_id[individu_id].get('handicap', False)
or find_age(individu_by_id[individu_id], period.start.date,
default = 0) <= 25,
error = u"Une personne à charge d'un foyer fiscal doit avoir moins de"
u" 25 ans ou être handicapée",
),
),
),
default = conv.noop,
),
),
conv.empty_to_none,
conv.not_none,
),
menages = conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(
personne_de_reference = conv.not_none,
),
default = conv.noop,
),
),
conv.empty_to_none,
conv.not_none,
),
),
default = conv.noop,
)(test_case, state = state)
return test_case, error
conv.not_none,
),
},
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
),
)(value, state = state)
conv.remove_ancestor_from_state(state, value)
return validated_value, errors
validate_values_holder_json = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_value_json,
drop_none_items = True,
),
make_validate_values_json_dates(require_consecutive_dates = False),
conv.empty_to_none,
)
# Level-2 Converters
validate_any_legislation_json = conv.pipe(
conv.test_isinstance(dict),
conv.condition(
conv.test(lambda legislation_json: 'datesim' in legislation_json),
validate_dated_legislation_json,
validate_node_json,
def build_role_checker(role, entity):
if role.max == 1:
return role.key, conv.test_in_pop(remaining_persons[entity.plural])
else:
return role.plural, conv.uniform_sequence(
conv.test_in_pop(remaining_persons[entity.plural]))
def validate_node_json(node, state = None):
if node is None:
return None, None
state = conv.add_ancestor_to_state(state, node)
validated_node, error = conv.test_isinstance(dict)(node, state = state)
if error is not None:
conv.remove_ancestor_from_state(state, node)
return validated_node, error
validated_node, errors = conv.struct(
{
'@context': conv.pipe(
conv.test_isinstance(basestring),
conv.make_input_to_url(full = True),
conv.test_equals(u'http://openfisca.fr/contexts/legislation.jsonld'),
),
'@type': conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
conv.test_in((u'Node', u'Parameter', u'Scale')),
conv.not_none,
),
'comment': conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_text,
),
'description': conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
),
},
constructor = collections.OrderedDict,
default = conv.noop,
drop_none_values = 'missing',
keep_value_order = True,
)(validated_node, state = state)
if errors is not None:
conv.remove_ancestor_from_state(state, node)
return validated_node, errors
validated_node.pop('@context', None) # Remove optional @context everywhere. It will be added to root node later.
node_converters = {
'@type': conv.noop,
'comment': conv.noop,
'description': conv.noop,
}
node_type = validated_node['@type']
if node_type == u'Node':
node_converters.update(dict(
children = conv.pipe(
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
conv.not_none,
),
conv.pipe(
validate_node_json,
conv.not_none,
),
),
conv.empty_to_none,
conv.not_none,
),
))
elif node_type == u'Parameter':
node_converters.update(dict(
format = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in([
'boolean',
'float',
'integer',
'rate',
]),
),
unit = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in(units),
),
values = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_value_json,
drop_none_items = True,
),
make_validate_values_json_dates(require_consecutive_dates = True),
conv.empty_to_none,
conv.not_none,
),
))
else:
assert node_type == u'Scale'
node_converters.update(dict(
option = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in((
'contrib',
'main-d-oeuvre',
'noncontrib',
)),
),
slices = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
validate_slice_json,
drop_none_items = True,
),
validate_slices_json_dates,
conv.empty_to_none,
conv.not_none,
),
unit = conv.pipe(
conv.test_isinstance(basestring),
conv.input_to_slug,
conv.test_in((
'currency',
)),
),
))
validated_node, errors = conv.struct(
node_converters,
constructor = collections.OrderedDict,
drop_none_values = 'missing',
keep_value_order = True,
)(validated_node, state = state)
conv.remove_ancestor_from_state(state, node)
return validated_node, errors
def make_json_or_python_to_test(tax_benefit_system):
validate = conv.struct(
dict(
itertools.chain(
dict(
absolute_error_margin=conv.pipe(
conv.test_isinstance((float, int)),
conv.test_greater_or_equal(0),
),
axes=make_json_or_python_to_axes(tax_benefit_system),
description=conv.pipe(
conv.test_isinstance(basestring_type),
conv.cleanup_line,
),
input_variables=conv.pipe(
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(
conv.test_isinstance(basestring_type),
conv.not_none,
),
conv.noop,
),
conv.empty_to_none,
),
keywords=conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.pipe(
conv.test_isinstance(basestring_type),
conv.cleanup_line,
),
drop_none_items=True,
),
conv.empty_to_none,
),
name=conv.pipe(
conv.test_isinstance(basestring_type),
conv.cleanup_line,
),
output_variables=conv.test_isinstance(dict),
period=conv.pipe(
conv.function(periods.period),
conv.not_none,
),
relative_error_margin=conv.pipe(
conv.test_isinstance((float, int)),
conv.test_greater_or_equal(0),
),
).items(),
((entity.plural,
conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
)) for entity in tax_benefit_system.entities),
)), )
def json_or_python_to_test(value, state=None):
if value is None:
return value, None
if state is None:
state = conv.default_state
value, error = conv.pipe(
conv.test_isinstance(dict),
validate,
)(value, state=state)
if error is not None:
return value, error
value, error = conv.struct(
dict(output_variables=make_json_or_python_to_input_variables(
tax_benefit_system, value['period']), ),
default=conv.noop,
)(value, state=state)
if error is not None:
return value, error
test_case = value.copy()
absolute_error_margin = test_case.pop('absolute_error_margin')
axes = test_case.pop('axes')
description = test_case.pop('description')
input_variables = test_case.pop('input_variables')
keywords = test_case.pop('keywords')
name = test_case.pop('name')
output_variables = test_case.pop('output_variables')
period = test_case.pop('period')
relative_error_margin = test_case.pop('relative_error_margin')
if test_case is not None and all(
entity_members is None
for entity_members in test_case.values()):
test_case = None
scenario, error = tax_benefit_system.Scenario.make_json_to_instance(
repair=True, tax_benefit_system=tax_benefit_system)(dict(
axes=axes,
input_variables=input_variables,
period=period,
test_case=test_case,
),
state=state)
if error is not None:
return scenario, error
return {
key: value
for key, value in dict(
absolute_error_margin=absolute_error_margin,
description=description,
keywords=keywords,
name=name,
output_variables=output_variables,
relative_error_margin=relative_error_margin,
scenario=scenario,
).items() if value is not None
}, None
return json_or_python_to_test
def post_process_test_case(self, test_case, period, state):
individu_by_id = {
individu['id']: individu
for individu in test_case['individus']
}
parents_id = set(
parent_id
for foyer_fiscal in test_case['foyers_fiscaux']
for parent_id in foyer_fiscal['declarants']
)
test_case, error = conv.struct(
dict(
# foyers_fiscaux = conv.pipe(
# conv.uniform_sequence(
# conv.struct(
# dict(
# enfants = conv.uniform_sequence(
# conv.test(
# lambda individu_id:
# individu_by_id[individu_id].get('invalide', False)
# or find_age(individu_by_id[individu_id], period.start.date,
# default = 0) <= 25,
# error = u"Une personne à charge d'un foyer fiscal doit avoir moins de"
# u" 25 ans ou être handicapée",
# ),
# ),
# parents = conv.pipe(
# conv.empty_to_none,
# conv.not_none,
# conv.test(lambda parents: len(parents) <= 2,
# error = N_(u'A "famille" must have at most 2 "parents"'))
# ),
# ),
# default = conv.noop,
# ),
# ),
# conv.empty_to_none,
# conv.not_none,
# ),
foyers_fiscaux = conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(
declarants = conv.pipe(
conv.empty_to_none,
conv.not_none,
conv.test(
lambda declarants: len(declarants) <= 2,
error = N_(u'A "foyer_fiscal" must have at most 2 "declarants"'),
),
conv.uniform_sequence(conv.pipe(
)),
),
personnes_a_charge = conv.uniform_sequence(
conv.test(
lambda individu_id:
individu_by_id[individu_id].get('handicap', False)
or find_age(individu_by_id[individu_id], period.start.date,
default = 0) <= 25,
error = u"Une personne à charge d'un foyer fiscal doit avoir moins de"
u" 25 ans ou être handicapée",
),
),
),
default = conv.noop,
),
),
conv.empty_to_none,
conv.not_none,
),
menages = conv.pipe(
conv.uniform_sequence(
conv.struct(
dict(
personne_de_reference = conv.not_none,
),
default = conv.noop,
),
),
conv.empty_to_none,
conv.not_none,
),
),
default = conv.noop,
)(test_case, state = state)
return test_case, error
# First validation and conversion step
test_case, error = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
foyers_fiscaux = conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items = True,
),
conv.uniform_sequence(
conv.struct(
dict(itertools.chain(
dict(
declarants = conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.default([]),
),
id = conv.pipe(
conv.test_isinstance((basestring, int)),
conv.not_none,
),
personnes_a_charge = conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.default([]),
),
).iteritems(),
)),
drop_none_values = True,
),
drop_none_items = True,
),
conv.default([]),
),
menages = conv.pipe(
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items = True,
),
conv.uniform_sequence(
conv.struct(
dict(itertools.chain(
dict(
autres = conv.pipe(
# personnes ayant un lien autre avec la personne de référence
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.default([]),
),
# conjoint de la personne de référence
conjoint = conv.test_isinstance((basestring, int)),
enfants = conv.pipe(
# enfants de la personne de référence ou de son conjoint
conv.make_item_to_singleton(),
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.default([]),
),
id = conv.pipe(
conv.test_isinstance((basestring, int)),
conv.not_none,
),
personne_de_reference = conv.test_isinstance((basestring, int)),
).iteritems(),
)),
drop_none_values = True,
),
drop_none_items = True,
),
conv.default([]),
),
),
),
)(test_case, state = state)
# test_case, error = conv.struct(
# dict(
# foyers_fiscaux = conv.uniform_sequence(
# conv.struct(
# dict(
# declarants = conv.uniform_sequence(conv.test_in_pop(foyers_fiscaux_individus_id)),
# personnes_a_charge = conv.uniform_sequence(conv.test_in_pop(
# foyers_fiscaux_individus_id)),
# ),
# default = conv.noop,
# ),
# ),
# menages = conv.uniform_sequence(
# conv.struct(
# dict(
# autres = conv.uniform_sequence(conv.test_in_pop(menages_individus_id)),
# conjoint = conv.test_in_pop(menages_individus_id),
# enfants = conv.uniform_sequence(conv.test_in_pop(menages_individus_id)),
# personne_de_reference = conv.test_in_pop(menages_individus_id),
# ),
# default = conv.noop,
# ),
# ),
# ),
# default = conv.noop,
# )(test_case, state = state)
return test_case, error