def test_legislation_xml_file():
legislation_tree = xml.etree.ElementTree.parse(model.PARAM_FILE)
legislation_xml_json = conv.check(legislationsxml.xml_legislation_to_json)(legislation_tree.getroot(),
state = conv.default_state)
legislation_xml_json, errors = legislationsxml.validate_node_xml_json(legislation_xml_json,
state = conv.default_state)
if errors is not None:
errors = conv.embed_error(legislation_xml_json, 'errors', errors)
if errors is None:
raise ValueError(unicode(json.dumps(legislation_xml_json, ensure_ascii = False,
indent = 2)).encode('utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)),
unicode(json.dumps(legislation_xml_json, ensure_ascii = False, indent = 2)),
).encode('utf-8'))
_, legislation_json = legislationsxml.transform_node_xml_json_to_json(legislation_xml_json)
legislation_json, errors = legislations.validate_node_json(legislation_json, state = conv.default_state)
if errors is not None:
errors = conv.embed_error(legislation_json, 'errors', errors)
if errors is None:
raise ValueError(unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)).encode('utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)),
unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)),
).encode('utf-8'))
dated_legislation_json = legislations.generate_dated_legislation_json(legislation_json, datetime.date(2006, 1, 1))
# raise ValueError(unicode(json.dumps(dated_legislation_json, ensure_ascii = False, indent = 2)).encode('utf-8'))
compact_legislation = legislations.compact_dated_node_json(dated_legislation_json)
def test_multiple_xml_based_tax_benefit_system():
tax_benefit_system = DummyMultipleXmlBasedTaxBenefitSystem()
legislation_json = tax_benefit_system.legislation_json
assert legislation_json is not None
assert isinstance(legislation_json, dict), legislation_json
dated_legislation_json = legislations.generate_dated_legislation_json(legislation_json, '2012-01-01')
assert isinstance(dated_legislation_json, dict), legislation_json
compact_legislation = legislations.compact_dated_node_json(dated_legislation_json)
assert_equal(compact_legislation.csg.activite.deductible.taux, 0.051)
assert_equal(compact_legislation.csg.activite.crds.activite.taux, 0.005)
def check_legislation_xml_file(year):
legislation_tree = conv.check(legislationsxml.make_xml_legislation_info_list_to_xml_element(False))(
TaxBenefitSystem.legislation_xml_info_list, state = conv.default_state)
legislation_xml_json = conv.check(legislationsxml.xml_legislation_to_json)(
legislation_tree,
state = conv.default_state,
)
legislation_xml_json, errors = legislationsxml.validate_legislation_xml_json(legislation_xml_json,
state = conv.default_state)
if errors is not None:
errors = conv.embed_error(legislation_xml_json, 'errors', errors)
if errors is None:
raise ValueError(unicode(json.dumps(legislation_xml_json, ensure_ascii = False,
indent = 2)).encode('utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)),
unicode(json.dumps(legislation_xml_json, ensure_ascii = False, indent = 2)),
).encode('utf-8'))
_, legislation_json = legislationsxml.transform_node_xml_json_to_json(legislation_xml_json)
legislation_json, errors = legislations.validate_legislation_json(legislation_json, state = conv.default_state)
if errors is not None:
errors = conv.embed_error(legislation_json, 'errors', errors)
if errors is None:
raise ValueError(unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)).encode('utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)),
unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)),
).encode('utf-8'))
# Create tax_benefit system only now, to be able to debug XML validation errors in above code.
tax_benefit_system = TaxBenefitSystem()
if tax_benefit_system.preprocess_legislation is not None:
legislation_json = tax_benefit_system.preprocess_legislation(legislation_json)
legislation_json = legislations.generate_dated_legislation_json(legislation_json, year)
legislation_json, errors = legislations.validate_dated_legislation_json(legislation_json,
state = conv.default_state)
if errors is not None:
errors = conv.embed_error(legislation_json, 'errors', errors)
if errors is None:
raise ValueError(unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)).encode(
'utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)),
unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)),
).encode('utf-8'))
compact_legislation = legislations.compact_dated_node_json(legislation_json)
assert compact_legislation is not None
def check_legislation_xml_file(year):
legislation_tree = xml.etree.ElementTree.parse(base.TaxBenefitSystem.legislation_xml_file_path)
legislation_xml_json = conv.check(legislationsxml.xml_legislation_to_json)(legislation_tree.getroot(),
state = conv.default_state)
legislation_xml_json, errors = legislationsxml.validate_legislation_xml_json(legislation_xml_json,
state = conv.default_state)
if errors is not None:
errors = conv.embed_error(legislation_xml_json, 'errors', errors)
if errors is None:
raise ValueError(unicode(json.dumps(legislation_xml_json, ensure_ascii = False,
indent = 2)).encode('utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)),
unicode(json.dumps(legislation_xml_json, ensure_ascii = False, indent = 2)),
).encode('utf-8'))
_, legislation_json = legislationsxml.transform_node_xml_json_to_json(legislation_xml_json)
legislation_json, errors = legislations.validate_legislation_json(legislation_json, state = conv.default_state)
if errors is not None:
errors = conv.embed_error(legislation_json, 'errors', errors)
if errors is None:
raise ValueError(unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)).encode('utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)),
unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)),
).encode('utf-8'))
legislation_json = legislations.generate_dated_legislation_json(legislation_json, year)
legislation_json, errors = legislations.validate_dated_legislation_json(legislation_json,
state = conv.default_state)
if errors is not None:
errors = conv.embed_error(legislation_json, 'errors', errors)
if errors is None:
raise ValueError(unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)).encode(
'utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)),
unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)),
).encode('utf-8'))
compact_legislation = legislations.compact_dated_node_json(legislation_json)
# Create tax_benefit system only now, to be able to debug XML validation errors in above code.
if base.tax_benefit_system.preprocess_compact_legislation is not None:
base.tax_benefit_system.preprocess_compact_legislation(compact_legislation)
def test_legislation_xml_file():
legislation_tree = xml.etree.ElementTree.parse(model.PARAM_FILE)
legislation_xml_json = conv.check(legislationsxml.xml_legislation_to_json)(legislation_tree.getroot(),
state = conv.default_state)
legislation_xml_json, errors = legislationsxml.validate_legislation_xml_json(legislation_xml_json,
state = conv.default_state)
if errors is not None:
errors = conv.embed_error(legislation_xml_json, 'errors', errors)
if errors is None:
raise ValueError(unicode(json.dumps(legislation_xml_json, ensure_ascii = False,
indent = 2)).encode('utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)),
unicode(json.dumps(legislation_xml_json, ensure_ascii = False, indent = 2)),
).encode('utf-8'))
_, legislation_json = legislationsxml.transform_node_xml_json_to_json(legislation_xml_json)
legislation_json, errors = legislations.validate_legislation_json(legislation_json, state = conv.default_state)
if errors is not None:
errors = conv.embed_error(legislation_json, 'errors', errors)
if errors is None:
raise ValueError(unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)).encode('utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)),
unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)),
).encode('utf-8'))
dated_legislation_json = legislations.generate_dated_legislation_json(legislation_json, datetime.date(2006, 1, 1))
dated_legislation_json, errors = legislations.validate_dated_legislation_json(dated_legislation_json,
state = conv.default_state)
if errors is not None:
errors = conv.embed_error(dated_legislation_json, 'errors', errors)
if errors is None:
raise ValueError(unicode(json.dumps(dated_legislation_json, ensure_ascii = False, indent = 2)).encode(
'utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)),
unicode(json.dumps(dated_legislation_json, ensure_ascii = False, indent = 2)),
).encode('utf-8'))
compact_legislation = legislations.compact_dated_node_json(dated_legislation_json)
def json_to_attributes(value, state=None):
if value is None:
return value, None
if state is None:
state = conv.default_state
# First validation and conversion step
data, error = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
familles=conv.pipe(
conv.condition(
conv.test_isinstance(list),
conv.pipe(
conv.uniform_sequence(conv.test_isinstance(dict), drop_none_items=True),
conv.function(
lambda values: collections.OrderedDict(
(value.pop("id", index), value) for index, value in enumerate(values)
)
),
),
),
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(conv.test_isinstance((basestring, int)), conv.not_none),
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
itertools.chain(
dict(
enfants=conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items=True,
),
conv.default([]),
),
parents=conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items=True,
),
conv.empty_to_none,
conv.not_none,
),
).iteritems(),
(
(column.name, column.json_to_python)
for column in column_by_name.itervalues()
if column.entity == "fam"
),
)
)
),
),
drop_none_values=True,
),
conv.empty_to_none,
conv.not_none,
),
foyers_fiscaux=conv.pipe(
conv.condition(
conv.test_isinstance(list),
conv.pipe(
conv.uniform_sequence(conv.test_isinstance(dict), drop_none_items=True),
conv.function(
lambda values: collections.OrderedDict(
(value.pop("id", index), value) for index, value in enumerate(values)
)
),
),
),
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(conv.test_isinstance((basestring, int)), conv.not_none),
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
itertools.chain(
dict(
declarants=conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items=True,
),
conv.empty_to_none,
conv.not_none,
),
personnes_a_charge=conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items=True,
),
conv.default([]),
),
).iteritems(),
(
(column.name, column.json_to_python)
for column in column_by_name.itervalues()
if column.entity == "foy"
),
)
)
),
),
drop_none_values=True,
),
conv.empty_to_none,
conv.not_none,
),
individus=conv.pipe(
conv.condition(
conv.test_isinstance(list),
conv.pipe(
conv.uniform_sequence(conv.test_isinstance(dict), drop_none_items=True),
conv.function(
lambda values: collections.OrderedDict(
(value.pop("id", index), value) for index, value in enumerate(values)
)
),
),
),
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(conv.test_isinstance((basestring, int)), conv.not_none),
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
itertools.chain(
dict(
birth=conv.pipe(
conv.test_isinstance(basestring),
conv.iso8601_input_to_date,
conv.not_none,
),
prenom=conv.pipe(
conv.test_isinstance(basestring), conv.cleanup_line
),
).iteritems(),
(
(column.name, column.json_to_python)
for column in column_by_name.itervalues()
if column.entity == "ind"
and column.name
not in (
"age",
"agem",
"idfam",
"idfoy",
"idmen",
"quifam",
"quifoy",
"quimen",
)
),
)
)
),
),
drop_none_values=True,
),
conv.empty_to_none,
conv.not_none,
),
legislation_url=conv.pipe(
conv.make_input_to_url(
error_if_fragment=True, full=True, schemes=("file", "http", "https")
),
conv.not_none,
),
menages=conv.pipe(
conv.condition(
conv.test_isinstance(list),
conv.pipe(
conv.uniform_sequence(conv.test_isinstance(dict), drop_none_items=True),
conv.function(
lambda values: collections.OrderedDict(
(value.pop("id", index), value) for index, value in enumerate(values)
)
),
),
),
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(conv.test_isinstance((basestring, int)), conv.not_none),
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
itertools.chain(
dict(
autres=conv.pipe(
# personnes ayant un lien autre avec la personne de référence
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items=True,
),
conv.default([]),
),
conjoint=conv.test_isinstance((basestring, int)),
# conjoint de la personne de référence
enfants=conv.pipe(
# enfants de la personne de référence ou de son conjoint
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items=True,
),
conv.default([]),
),
personne_de_reference=conv.pipe(
conv.test_isinstance((basestring, int)), conv.not_none
),
).iteritems(),
(
(column.name, column.json_to_python)
for column in column_by_name.itervalues()
if column.entity == "men"
),
)
)
),
),
drop_none_values=True,
),
conv.empty_to_none,
conv.not_none,
),
year=conv.pipe(
conv.test_isinstance(int),
conv.test_greater_or_equal(1900), # TODO: Check that year is valid in params.
conv.not_none,
),
)
),
)(value, state=state)
if error is not None:
return data, error
# Second validation step
familles_individus_id = set(data["individus"].iterkeys())
foyers_fiscaux_individus_id = set(data["individus"].iterkeys())
menages_individus_id = set(data["individus"].iterkeys())
data, error = conv.struct(
dict(
familles=conv.uniform_mapping(
conv.noop,
conv.struct(
dict(
enfants=conv.uniform_sequence(conv.test_in_pop(familles_individus_id)),
parents=conv.uniform_sequence(conv.test_in_pop(familles_individus_id)),
),
default=conv.noop,
),
),
foyers_fiscaux=conv.uniform_mapping(
conv.noop,
conv.struct(
dict(
declarants=conv.uniform_sequence(conv.test_in_pop(foyers_fiscaux_individus_id)),
personnes_a_charge=conv.uniform_sequence(conv.test_in_pop(foyers_fiscaux_individus_id)),
),
default=conv.noop,
),
),
menages=conv.uniform_mapping(
conv.noop,
conv.struct(
dict(
autres=conv.uniform_sequence(conv.test_in_pop(menages_individus_id)),
conjoint=conv.test_in_pop(menages_individus_id),
enfants=conv.uniform_sequence(conv.test_in_pop(menages_individus_id)),
personne_de_reference=conv.test_in_pop(menages_individus_id),
),
default=conv.noop,
),
),
),
default=conv.noop,
)(data, state=state)
remaining_individus_id = familles_individus_id.union(foyers_fiscaux_individus_id, menages_individus_id)
if remaining_individus_id:
if error is None:
error = {}
for individu_id in remaining_individus_id:
error.setdefault("individus", {})[individu_id] = state._(u"Individual is missing from {}").format(
u" & ".join(
word
for word in [
u"familles" if individu_id not in familles_individus_id else None,
u"foyers_fiscaux" if individu_id not in foyers_fiscaux_individus_id else None,
u"menages" if individu_id not in menages_individus_id else None,
]
if word is not None
)
)
if error is not None:
return data, error
if cache_dir is not None:
legislation_uuid_hex = uuid.uuid5(uuid.NAMESPACE_URL, data["legislation_url"].encode("utf-8")).hex
legislation_dir = os.path.join(cache_dir, "legislations", legislation_uuid_hex[:2])
legislation_filename = "{}.json".format(legislation_uuid_hex[2:])
legislation_file_path = os.path.join(legislation_dir, legislation_filename)
legislation_json = None
if (
os.path.exists(legislation_file_path)
and os.path.getmtime(legislation_file_path) > time.time() - 900
): # 15 minutes
with open(legislation_file_path) as legislation_file:
try:
legislation_json = json.load(legislation_file, object_pairs_hook=collections.OrderedDict)
except ValueError:
log.exception("Error while reading legislation JSON file: {}".format(legislation_file_path))
if legislation_json is None:
request = urllib2.Request(data["legislation_url"], headers={"User-Agent": "OpenFisca-Web-API"})
try:
response = urllib2.urlopen(request)
except urllib2.HTTPError:
return data, dict(legislation_url=state._(u"HTTP Error while retrieving legislation JSON"))
except urllib2.URLError:
return data, dict(legislation_url=state._(u"Error while retrieving legislation JSON"))
legislation_json, error = conv.pipe(
conv.make_input_to_json(object_pairs_hook=collections.OrderedDict),
legislations.validate_node_json,
conv.not_none,
)(response.read(), state=state)
if error is not None:
return data, dict(legislation_url=error)
if cache_dir is not None:
if not os.path.exists(legislation_dir):
os.makedirs(legislation_dir)
with open(legislation_file_path, "w") as legislation_file:
legislation_file.write(
unicode(
json.dumps(legislation_json, encoding="utf-8", ensure_ascii=False, indent=2)
).encode("utf-8")
)
datesim = date(data["year"], 1, 1)
dated_legislation_json = legislations.generate_dated_legislation_json(legislation_json, datesim)
compact_legislation = legislations.compact_dated_node_json(dated_legislation_json)
attributes = dict(
compact_legislation=compact_legislation, declar={}, famille={}, indiv={}, menage={}, year=data["year"]
)
indiv_index_by_id = dict(
(individu_id, individu_index)
for individu_index, individu_id in enumerate(data[u"individus"].iterkeys())
)
for individu_id, individu in data[u"individus"].iteritems():
individu.pop("prenom", None)
attributes["indiv"][indiv_index_by_id[individu_id]] = individu
for famille in data[u"familles"].itervalues():
parents_id = famille.pop(u"parents")
enfants_id = famille.pop(u"enfants")
noichef = indiv_index_by_id[parents_id[0]]
attributes["declar"][noichef] = famille
for indivu_id in itertools.chain(parents_id, enfants_id):
attributes["indiv"][indiv_index_by_id[indivu_id]]["noichef"] = noichef
attributes["indiv"][noichef]["quifam"] = u"chef"
if len(parents_id) > 1:
attributes["indiv"][indiv_index_by_id[parents_id[1]]]["quifam"] = u"part"
for enfant_number, enfant_id in enumerate(enfants_id, 1):
attributes["indiv"][indiv_index_by_id[enfant_id]]["quifam"] = u"enf{}".format(enfant_number)
for foyer_fiscal in data[u"foyers_fiscaux"].itervalues():
declarants_id = foyer_fiscal.pop(u"declarants")
personnes_a_charge_id = foyer_fiscal.pop(u"personnes_a_charge")
noidec = indiv_index_by_id[declarants_id[0]]
attributes["declar"][noidec] = foyer_fiscal
for indivu_id in itertools.chain(declarants_id, personnes_a_charge_id):
attributes["indiv"][indiv_index_by_id[indivu_id]]["noidec"] = noidec
attributes["indiv"][noidec]["quifoy"] = u"vous"
if len(declarants_id) > 1:
attributes["indiv"][indiv_index_by_id[declarants_id[1]]]["quifoy"] = u"conj"
for personne_a_charge_number, personne_a_charge_id in enumerate(personnes_a_charge_id, 1):
attributes["indiv"][indiv_index_by_id[personne_a_charge_id]]["quifoy"] = u"pac{}".format(
personne_a_charge_number
)
for menage in data[u"menages"].itervalues():
personne_de_reference_id = menage.pop(u"personne_de_reference")
conjoint_id = menage.pop(u"conjoint")
enfants_id = menage.pop(u"enfants")
autres_id = menage.pop(u"autres")
noipref = indiv_index_by_id[personne_de_reference_id]
attributes["declar"][noipref] = menage
for indivu_id in itertools.chain(
[personne_de_reference_id], [conjoint_id] if conjoint_id is not None else [], enfants_id, autres_id
):
attributes["indiv"][indiv_index_by_id[indivu_id]]["noipref"] = noipref
attributes["indiv"][noipref]["quimen"] = u"pref"
if conjoint_id is not None:
attributes["indiv"][indiv_index_by_id[conjoint_id]]["quimen"] = u"cref"
for enfant_number, enfant_id in enumerate(itertools.chain(enfants_id, autres_id), 1):
attributes["indiv"][indiv_index_by_id[enfant_id]]["quimen"] = u"enf{}".format(enfant_number)
return attributes, None
def json_to_attributes(cls, value, state = None):
if value is None:
return value, None
if state is None:
state = conv.default_state
# First validation and conversion step
data, error = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
familles = conv.pipe(
conv.condition(
conv.test_isinstance(list),
conv.pipe(
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items = True,
),
conv.function(lambda values: collections.OrderedDict(
(value.pop('id', index), value)
for index, value in enumerate(values)
)),
),
),
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(
conv.test_isinstance((basestring, int)),
conv.not_none,
),
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(itertools.chain(
dict(
enfants = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.default([]),
),
parents = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.empty_to_none,
conv.not_none,
),
).iteritems(),
(
(column.name, column.json_to_python)
for column in column_by_name.itervalues()
if column.entity == 'fam'
),
)),
),
),
drop_none_values = True,
),
conv.empty_to_none,
conv.not_none,
),
foyers_fiscaux = conv.pipe(
conv.condition(
conv.test_isinstance(list),
conv.pipe(
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items = True,
),
conv.function(lambda values: collections.OrderedDict(
(value.pop('id', index), value)
for index, value in enumerate(values)
)),
),
),
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(
conv.test_isinstance((basestring, int)),
conv.not_none,
),
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(itertools.chain(
dict(
declarants = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.empty_to_none,
conv.not_none,
),
personnes_a_charge = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.default([]),
),
).iteritems(),
(
(column.name, column.json_to_python)
for column in column_by_name.itervalues()
if column.entity == 'foy'
),
)),
),
),
drop_none_values = True,
),
conv.empty_to_none,
conv.not_none,
),
individus = conv.pipe(
conv.condition(
conv.test_isinstance(list),
conv.pipe(
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items = True,
),
conv.function(lambda values: collections.OrderedDict(
(value.pop('id', index), value)
for index, value in enumerate(values)
)),
),
),
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(
conv.test_isinstance((basestring, int)),
conv.not_none,
),
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(itertools.chain(
dict(
birth = conv.pipe(
conv.test_isinstance(basestring),
conv.iso8601_input_to_date,
conv.not_none,
),
).iteritems(),
(
(column.name, column.json_to_python)
for column in column_by_name.itervalues()
if column.entity == 'ind' and column.name not in ('age', 'agem', 'idfam',
'idfoy', 'idmen', 'quifam', 'quifoy', 'quimen')
),
)),
),
),
drop_none_values = True,
),
conv.empty_to_none,
conv.not_none,
),
legislation_url = conv.pipe(
conv.make_input_to_url(error_if_fragment = True, full = True),
conv.not_none,
),
menages = conv.pipe(
conv.condition(
conv.test_isinstance(list),
conv.pipe(
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items = True,
),
conv.function(lambda values: collections.OrderedDict(
(value.pop('id', index), value)
for index, value in enumerate(values)
)),
),
),
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(
conv.test_isinstance((basestring, int)),
conv.not_none,
),
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(itertools.chain(
dict(
autres = conv.pipe(
# personnes ayant un lien autre avec la personne de référence
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.default([]),
),
conjoint = conv.test_isinstance((basestring, int)),
# conjoint de la personne de référence
enfants = conv.pipe(
# enfants de la personne de référence ou de son conjoint
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.default([]),
),
personne_de_reference = conv.pipe(
conv.test_isinstance((basestring, int)),
conv.not_none,
),
).iteritems(),
(
(column.name, column.json_to_python)
for column in column_by_name.itervalues()
if column.entity == 'men'
),
)),
),
),
drop_none_values = True,
),
conv.empty_to_none,
conv.not_none,
),
year = conv.pipe(
conv.test_isinstance(int),
conv.test_greater_or_equal(1900), # TODO: Check that year is valid in params.
conv.not_none,
),
),
),
)(value, state = state)
if error is not None:
return data, error
# Second validation step
familles_individus_id = set(data['individus'].iterkeys())
foyers_fiscaux_individus_id = set(data['individus'].iterkeys())
menages_individus_id = set(data['individus'].iterkeys())
data, error = conv.struct(
dict(
familles = conv.uniform_mapping(
conv.noop,
conv.struct(
dict(
enfants = conv.uniform_sequence(conv.test_in_pop(familles_individus_id)),
parents = conv.uniform_sequence(conv.test_in_pop(familles_individus_id)),
),
default = conv.noop,
),
),
foyers_fiscaux = conv.uniform_mapping(
conv.noop,
conv.struct(
dict(
declarants = conv.uniform_sequence(conv.test_in_pop(foyers_fiscaux_individus_id)),
personnes_a_charge = conv.uniform_sequence(conv.test_in_pop(foyers_fiscaux_individus_id)),
),
default = conv.noop,
),
),
menages = conv.uniform_mapping(
conv.noop,
conv.struct(
dict(
autres = conv.uniform_sequence(conv.test_in_pop(menages_individus_id)),
conjoint = conv.test_in_pop(menages_individus_id),
enfants = conv.uniform_sequence(conv.test_in_pop(menages_individus_id)),
personne_de_reference = conv.test_in_pop(menages_individus_id),
),
default = conv.noop,
),
),
),
default = conv.noop,
)(data, state = state)
remaining_individus_id = familles_individus_id.union(foyers_fiscaux_individus_id, menages_individus_id)
if remaining_individus_id:
if error is None:
error = {}
for individu_id in remaining_individus_id:
error.setdefault('individus', {})[individu_id] = state._(u"Individual is missing from {}").format(
u' & '.join([
u'familles' if individu_id in familles_individus_id else None,
u'foyers_fiscaux' if individu_id in foyers_fiscaux_individus_id else None,
u'menages' if individu_id in menages_individus_id else None,
]))
if error is not None:
return data, error
request = urllib2.Request(data['legislation_url'], headers = {
'User-Agent': 'OpenFisca-Web-API',
})
try:
response = urllib2.urlopen(request)
except urllib2.HTTPError:
return data, dict(legislation_url = ctx._(u'HTTP Error while retrieving legislation JSON'))
except urllib2.URLError:
return data, dict(legislation_url = ctx._(u'Error while retrieving legislation JSON'))
legislation_json, error = conv.pipe(
conv.make_input_to_json(object_pairs_hook = collections.OrderedDict),
legislations.validate_node_json,
conv.not_none,
)(response.read(), state = state)
if error is not None:
return data, dict(legislation_url = error)
datesim = date(data['year'], 1, 1)
dated_legislation_json = legislations.generate_dated_legislation_json(legislation_json, datesim)
compact_legislation = legislations.compact_dated_node_json(dated_legislation_json)
attributes = dict(
compact_legislation = compact_legislation,
declar = {},
famille = {},
indiv = {},
menage = {},
year = data['year'],
)
indiv_index_by_id = dict(
(individu_id, individu_index)
for individu_index, individu_id in enumerate(data[u'individus'].iterkeys())
)
for individu_id, individu in data[u'individus'].iteritems():
attributes['indiv'][indiv_index_by_id[individu_id]] = individu
for famille in data[u'familles'].itervalues():
parents_id = famille.pop(u'parents')
enfants_id = famille.pop(u'enfants')
noichef = indiv_index_by_id[parents_id[0]]
attributes['declar'][noichef] = famille
for indivu_id in itertools.chain(parents_id, enfants_id):
attributes['indiv'][indiv_index_by_id[indivu_id]]['noichef'] = noichef
attributes['indiv'][noichef]['quifam'] = u'chef'
if len(parents_id) > 1:
attributes['indiv'][indiv_index_by_id[parents_id[1]]]['quifam'] = u'part'
for enfant_number, enfant_id in enumerate(enfants_id, 1):
attributes['indiv'][indiv_index_by_id[enfant_id]]['quifam'] = u'enf{}'.format(enfant_number)
for foyer_fiscal in data[u'foyers_fiscaux'].itervalues():
declarants_id = foyer_fiscal.pop(u'declarants')
personnes_a_charge_id = foyer_fiscal.pop(u'personnes_a_charge')
noidec = indiv_index_by_id[declarants_id[0]]
attributes['declar'][noidec] = foyer_fiscal
for indivu_id in itertools.chain(declarants_id, personnes_a_charge_id):
attributes['indiv'][indiv_index_by_id[indivu_id]]['noidec'] = noidec
attributes['indiv'][noidec]['quifoy'] = u'vous'
if len(declarants_id) > 1:
attributes['indiv'][indiv_index_by_id[declarants_id[1]]]['quifoy'] = u'conj'
for personne_a_charge_number, personne_a_charge_id in enumerate(personnes_a_charge_id, 1):
attributes['indiv'][indiv_index_by_id[personne_a_charge_id]]['quifoy'] = u'pac{}'.format(
personne_a_charge_number)
for menage in data[u'menages'].itervalues():
personne_de_reference_id = menage.pop(u'personne_de_reference')
conjoint_id = menage.pop(u'conjoint')
enfants_id = menage.pop(u'enfants')
autres_id = menage.pop(u'autres')
noipref = indiv_index_by_id[personne_de_reference_id]
attributes['declar'][noipref] = menage
for indivu_id in itertools.chain(
[personne_de_reference_id],
[conjoint_id] if conjoint_id is not None else [],
enfants_id,
autres_id,
):
attributes['indiv'][indiv_index_by_id[indivu_id]]['noipref'] = noipref
attributes['indiv'][noipref]['quimen'] = u'pref'
if conjoint_id is not None:
attributes['indiv'][indiv_index_by_id[conjoint_id]]['quimen'] = u'cref'
for enfant_number, enfant_id in enumerate(itertools.chain(enfants_id, autres_id), 1):
attributes['indiv'][indiv_index_by_id[enfant_id]]['quimen'] = u'enf{}'.format(enfant_number)
return attributes, None
def check_legislation_xml_file(year):
legislation_tree = conv.check(
legislationsxml.make_xml_legislation_info_list_to_xml_element(False))(
TaxBenefitSystem.legislation_xml_info_list,
state=conv.default_state)
legislation_xml_json = conv.check(legislationsxml.xml_legislation_to_json)(
legislation_tree,
state=conv.default_state,
)
legislation_xml_json, errors = legislationsxml.validate_legislation_xml_json(
legislation_xml_json, state=conv.default_state)
if errors is not None:
errors = conv.embed_error(legislation_xml_json, 'errors', errors)
if errors is None:
raise ValueError(
unicode(
json.dumps(legislation_xml_json,
ensure_ascii=False,
indent=2)).encode('utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(
json.dumps(errors,
ensure_ascii=False,
indent=2,
sort_keys=True)),
unicode(
json.dumps(legislation_xml_json, ensure_ascii=False,
indent=2)),
).encode('utf-8'))
_, legislation_json = legislationsxml.transform_node_xml_json_to_json(
legislation_xml_json)
legislation_json, errors = legislations.validate_legislation_json(
legislation_json, state=conv.default_state)
if errors is not None:
errors = conv.embed_error(legislation_json, 'errors', errors)
if errors is None:
raise ValueError(
unicode(
json.dumps(legislation_json, ensure_ascii=False,
indent=2)).encode('utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(
json.dumps(errors,
ensure_ascii=False,
indent=2,
sort_keys=True)),
unicode(json.dumps(legislation_json, ensure_ascii=False,
indent=2)),
).encode('utf-8'))
# Create tax_benefit system only now, to be able to debug XML validation errors in above code.
tax_benefit_system = TaxBenefitSystem()
if tax_benefit_system.preprocess_legislation is not None:
legislation_json = tax_benefit_system.preprocess_legislation(
legislation_json)
legislation_json = legislations.generate_dated_legislation_json(
legislation_json, year)
legislation_json, errors = legislations.validate_dated_legislation_json(
legislation_json, state=conv.default_state)
if errors is not None:
errors = conv.embed_error(legislation_json, 'errors', errors)
if errors is None:
raise ValueError(
unicode(
json.dumps(legislation_json, ensure_ascii=False,
indent=2)).encode('utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(
json.dumps(errors,
ensure_ascii=False,
indent=2,
sort_keys=True)),
unicode(json.dumps(legislation_json, ensure_ascii=False,
indent=2)),
).encode('utf-8'))
compact_legislation = legislations.compact_dated_node_json(
legislation_json)
assert compact_legislation is not None
def json_or_python_to_attributes(value, state = None):
if value is None:
return value, None
if state is None:
state = conv.default_state
# First validation and conversion step
data, error = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
axes = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
count = conv.pipe(
conv.test_isinstance(int),
conv.test_greater_or_equal(1),
conv.not_none,
),
index = conv.pipe(
conv.test_isinstance(int),
conv.test_greater_or_equal(0),
conv.default(0),
),
max = conv.pipe(
conv.test_isinstance((float, int)),
conv.not_none,
),
min = conv.pipe(
conv.test_isinstance((float, int)),
conv.not_none,
),
name = conv.pipe(
conv.test_isinstance(basestring),
conv.test_in(column_by_name),
conv.test(lambda column_name: column_by_name[column_name].dtype in (
np.float32, np.int16, np.int32),
error = N_(u'Invalid type for axe: integer or float expected')),
conv.not_none,
),
),
),
),
drop_none_items = True,
),
conv.empty_to_none,
),
legislation_url = conv.pipe(
conv.test_isinstance(basestring),
conv.make_input_to_url(error_if_fragment = True, full = True, schemes = ('http', 'https')),
),
test_case = conv.pipe(
self.json_or_python_to_test_case,
conv.not_none,
),
year = conv.pipe(
conv.test_isinstance(int),
conv.test_greater_or_equal(1900), # TODO: Check that year is valid in params.
conv.not_none,
),
),
),
)(value, state = state)
if error is not None:
return data, error
errors = {}
if data['axes'] is not None:
for axis_index, axis in enumerate(data['axes']):
if axis['min'] >= axis['max']:
errors.setdefault('axes', {}).setdefault(axis_index, {})['max'] = state._(
u"Max value must be greater than min value")
column = column_by_name[axis['name']]
entity_class = entities.entity_class_by_symbol[column.entity]
if axis['index'] >= len(data['test_case'][entity_class.key_plural]):
errors.setdefault('axes', {}).setdefault(axis_index, {})['index'] = state._(
u"Index must be lower than {}").format(len(data['test_case'][entity_class.key_plural]))
famille_by_id = data['test_case']['familles']
parents_id = set(
parent_id
for famille in data['test_case']['familles'].itervalues()
for parent_id in famille['parents']
)
individu_by_id = data['test_case']['individus']
data, errors = conv.struct(
dict(
test_case = conv.struct(
dict(
foyers_fiscaux = conv.uniform_mapping(
conv.noop,
conv.struct(
dict(
declarants = conv.uniform_sequence(conv.pipe(
conv.test(lambda individu_id:
individu_by_id[individu_id].get('birth') is None
or data['year'] - individu_by_id[individu_id]['birth'].year >= 18,
error = u"Un déclarant d'un foyer fiscal doit être agé d'au moins 18"
u" ans",
),
conv.test(lambda individu_id:
individu_by_id[individu_id].get('age') is None
or individu_by_id[individu_id]['age'] >= 18,
error = u"Un déclarant d'un foyer fiscal doit être agé d'au moins 18"
u" ans",
),
conv.test(lambda individu_id:
individu_by_id[individu_id].get('agem') is None
or individu_by_id[individu_id]['agem'] >= 18 * 12,
error = u"Un déclarant d'un foyer fiscal doit être agé d'au moins 18"
u" ans",
),
conv.test(lambda individu_id: individu_id in parents_id,
error = u"Un déclarant ou un conjoint sur la feuille d'impôt, doit être"
u" un parent dans sa famille",
),
)),
personnes_a_charge = conv.uniform_sequence(conv.pipe(
conv.test(lambda individu_id: individu_by_id[individu_id].get('inv', False)
or individu_by_id[individu_id].get('birth') is None
or data['year'] - individu_by_id[individu_id]['birth'].year <= 25,
error = u"Une personne à charge d'un foyer fiscal doit avoir moins de"
u" 25 ans ou être invalide",
),
conv.test(lambda individu_id: individu_by_id[individu_id].get('inv', False)
or individu_by_id[individu_id].get('age') is None
or individu_by_id[individu_id]['age'] <= 25,
error = u"Une personne à charge d'un foyer fiscal doit avoir moins de"
u" 25 ans ou être invalide",
),
conv.test(lambda individu_id: individu_by_id[individu_id].get('inv', False)
or individu_by_id[individu_id].get('agem') is None
or individu_by_id[individu_id]['agem'] <= 25 * 12,
error = u"Une personne à charge d'un foyer fiscal doit avoir moins de"
u" 25 ans ou être invalide",
),
)),
),
default = conv.noop,
),
),
individus = conv.uniform_mapping(
conv.noop,
conv.struct(
dict(
birth = conv.test(lambda birth: data['year'] - birth.year >= 0,
error = u"L'individu doit être né au plus tard l'année de la simulation",
),
),
default = conv.noop,
drop_none_values = 'missing',
),
),
),
default = conv.noop,
),
),
default = conv.noop,
)(data, state = state)
if errors:
return data, errors
if data['legislation_url'] is None:
compact_legislation = None
else:
legislation_json = None
if cache_dir is not None:
legislation_uuid_hex = uuid.uuid5(uuid.NAMESPACE_URL, data['legislation_url'].encode('utf-8')).hex
legislation_dir = os.path.join(cache_dir, 'legislations', legislation_uuid_hex[:2])
legislation_filename = '{}.json'.format(legislation_uuid_hex[2:])
legislation_file_path = os.path.join(legislation_dir, legislation_filename)
if os.path.exists(legislation_file_path) \
and os.path.getmtime(legislation_file_path) > time.time() - 900: # 15 minutes
with open(legislation_file_path) as legislation_file:
try:
legislation_json = json.load(legislation_file,
object_pairs_hook = collections.OrderedDict)
except ValueError:
log.exception('Error while reading legislation JSON file: {}'.format(
legislation_file_path))
if legislation_json is None:
request = urllib2.Request(data['legislation_url'], headers = {
'User-Agent': 'OpenFisca-Web-API',
})
try:
response = urllib2.urlopen(request)
except urllib2.HTTPError:
return data, dict(legislation_url = state._(u'HTTP Error while retrieving legislation JSON'))
except urllib2.URLError:
return data, dict(legislation_url = state._(u'Error while retrieving legislation JSON'))
legislation_json, error = conv.pipe(
conv.make_input_to_json(object_pairs_hook = collections.OrderedDict),
legislations.validate_any_legislation_json,
conv.not_none,
)(response.read(), state = state)
if error is not None:
return data, dict(legislation_url = error)
if cache_dir is not None:
if not os.path.exists(legislation_dir):
os.makedirs(legislation_dir)
with open(legislation_file_path, 'w') as legislation_file:
legislation_file.write(unicode(json.dumps(legislation_json, encoding = 'utf-8',
ensure_ascii = False, indent = 2)).encode('utf-8'))
datesim = datetime.date(data['year'], 1, 1)
if legislation_json.get('datesim') is None:
dated_legislation_json = legislations.generate_dated_legislation_json(legislation_json, datesim)
else:
dated_legislation_json = legislation_json
legislation_json = None
compact_legislation = legislations.compact_dated_node_json(dated_legislation_json)
if self.tax_benefit_system.preprocess_legislation_parameters is not None:
self.tax_benefit_system.preprocess_legislation_parameters(compact_legislation)
self.axes = data['axes']
self.compact_legislation = compact_legislation
self.test_case = data['test_case']
self.year = data['year']
return self, None
def json_to_attributes(value, state = None):
if value is None:
return value, None
if state is None:
state = conv.default_state
# First validation and conversion step
data, error = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
familles = conv.pipe(
conv.condition(
conv.test_isinstance(list),
conv.pipe(
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items = True,
),
conv.function(lambda values: collections.OrderedDict(
(value.pop('id', index), value)
for index, value in enumerate(values)
)),
),
),
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(
conv.test_isinstance((basestring, int)),
conv.not_none,
),
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(itertools.chain(
dict(
enfants = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.default([]),
),
parents = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.empty_to_none,
conv.not_none,
conv.test(lambda parents: len(parents) <= 2,
error = N_(u'A "famille" must have at most 2 "parents"'))
),
).iteritems(),
(
(column.name, column.json_to_python)
for column in column_by_name.itervalues()
if column.entity == 'fam'
),
)),
),
),
drop_none_values = True,
),
conv.empty_to_none,
conv.not_none,
),
foyers_fiscaux = conv.pipe(
conv.condition(
conv.test_isinstance(list),
conv.pipe(
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items = True,
),
conv.function(lambda values: collections.OrderedDict(
(value.pop('id', index), value)
for index, value in enumerate(values)
)),
),
),
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(
conv.test_isinstance((basestring, int)),
conv.not_none,
),
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(itertools.chain(
dict(
declarants = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.empty_to_none,
conv.not_none,
conv.test(lambda declarants: len(declarants) <= 2,
error = N_(u'A "foyer_fiscal" must have at most 2 "declarants"'
))
),
personnes_a_charge = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.default([]),
),
).iteritems(),
(
(column.name, column.json_to_python)
for column in column_by_name.itervalues()
if column.entity == 'foy'
),
)),
),
),
drop_none_values = True,
),
conv.empty_to_none,
conv.not_none,
),
individus = conv.pipe(
conv.condition(
conv.test_isinstance(list),
conv.pipe(
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items = True,
),
conv.function(lambda values: collections.OrderedDict(
(value.pop('id', index), value)
for index, value in enumerate(values)
)),
),
),
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(
conv.test_isinstance((basestring, int)),
conv.not_none,
),
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(itertools.chain(
dict(
birth = conv.pipe(
conv.condition(
conv.test_isinstance(int),
conv.pipe(
conv.test_between(1870, 2099),
conv.function(lambda year: date(year, 1, 1)),
),
conv.pipe(
conv.test_isinstance(basestring),
conv.test(year_or_month_or_day_re.match,
error = N_(u'Invalid year')),
conv.function(lambda birth: u'-'.join((
birth.split(u'-') + [u'01', u'01'])[:3])),
conv.iso8601_input_to_date,
conv.test_between(date(1870, 1, 1), date(2099, 12, 31)),
),
),
conv.not_none,
),
prenom = conv.pipe(
conv.test_isinstance(basestring),
conv.cleanup_line,
),
).iteritems(),
(
(column.name, column.json_to_python)
for column in column_by_name.itervalues()
if column.entity == 'ind' and column.name not in ('age', 'agem',
'idfam', 'idfoy', 'idmen', 'quifam', 'quifoy', 'quimen')
),
)),
),
),
drop_none_values = True,
),
conv.empty_to_none,
conv.not_none,
),
legislation_url = conv.pipe(
conv.test_isinstance(basestring),
conv.make_input_to_url(error_if_fragment = True, full = True, schemes = ('http', 'https')),
conv.default(legislation_url) if legislation_url is not None else conv.not_none,
),
menages = conv.pipe(
conv.condition(
conv.test_isinstance(list),
conv.pipe(
conv.uniform_sequence(
conv.test_isinstance(dict),
drop_none_items = True,
),
conv.function(lambda values: collections.OrderedDict(
(value.pop('id', index), value)
for index, value in enumerate(values)
)),
),
),
conv.test_isinstance(dict),
conv.uniform_mapping(
conv.pipe(
conv.test_isinstance((basestring, int)),
conv.not_none,
),
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(itertools.chain(
dict(
autres = conv.pipe(
# personnes ayant un lien autre avec la personne de référence
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.default([]),
),
conjoint = conv.test_isinstance((basestring, int)),
# conjoint de la personne de référence
enfants = conv.pipe(
# enfants de la personne de référence ou de son conjoint
conv.test_isinstance(list),
conv.uniform_sequence(
conv.test_isinstance((basestring, int)),
drop_none_items = True,
),
conv.default([]),
),
personne_de_reference = conv.pipe(
conv.test_isinstance((basestring, int)),
conv.not_none,
),
).iteritems(),
(
(column.name, column.json_to_python)
for column in column_by_name.itervalues()
if column.entity == 'men'
),
)),
),
),
drop_none_values = True,
),
conv.empty_to_none,
conv.not_none,
),
year = conv.pipe(
conv.test_isinstance(int),
conv.test_greater_or_equal(1900), # TODO: Check that year is valid in params.
conv.not_none,
),
),
),
)(value, state = state)
if error is not None:
return data, error
# Second validation step
familles_individus_id = set(data['individus'].iterkeys())
foyers_fiscaux_individus_id = set(data['individus'].iterkeys())
menages_individus_id = set(data['individus'].iterkeys())
data, error = conv.struct(
dict(
familles = conv.uniform_mapping(
conv.noop,
conv.struct(
dict(
enfants = conv.uniform_sequence(conv.test_in_pop(familles_individus_id)),
parents = conv.uniform_sequence(conv.test_in_pop(familles_individus_id)),
),
default = conv.noop,
),
),
foyers_fiscaux = conv.uniform_mapping(
conv.noop,
conv.struct(
dict(
declarants = conv.uniform_sequence(conv.test_in_pop(foyers_fiscaux_individus_id)),
personnes_a_charge = conv.uniform_sequence(conv.test_in_pop(foyers_fiscaux_individus_id)),
),
default = conv.noop,
),
),
menages = conv.uniform_mapping(
conv.noop,
conv.struct(
dict(
autres = conv.uniform_sequence(conv.test_in_pop(menages_individus_id)),
conjoint = conv.test_in_pop(menages_individus_id),
enfants = conv.uniform_sequence(conv.test_in_pop(menages_individus_id)),
personne_de_reference = conv.test_in_pop(menages_individus_id),
),
default = conv.noop,
),
),
),
default = conv.noop,
)(data, state = state)
remaining_individus_id = familles_individus_id.union(foyers_fiscaux_individus_id, menages_individus_id)
if remaining_individus_id:
if error is None:
error = {}
for individu_id in remaining_individus_id:
error.setdefault('individus', {})[individu_id] = state._(u"Individual is missing from {}").format(
u' & '.join(
word
for word in [
u'familles' if individu_id not in familles_individus_id else None,
u'foyers_fiscaux' if individu_id not in foyers_fiscaux_individus_id else None,
u'menages' if individu_id not in menages_individus_id else None,
]
if word is not None
))
if error is not None:
return data, error
if cache_dir is not None:
legislation_uuid_hex = uuid.uuid5(uuid.NAMESPACE_URL, data['legislation_url'].encode('utf-8')).hex
legislation_dir = os.path.join(cache_dir, 'legislations', legislation_uuid_hex[:2])
legislation_filename = '{}.json'.format(legislation_uuid_hex[2:])
legislation_file_path = os.path.join(legislation_dir, legislation_filename)
legislation_json = None
if os.path.exists(legislation_file_path) \
and os.path.getmtime(legislation_file_path) > time.time() - 900: # 15 minutes
with open(legislation_file_path) as legislation_file:
try:
legislation_json = json.load(legislation_file,
object_pairs_hook = collections.OrderedDict)
except ValueError:
log.exception('Error while reading legislation JSON file: {}'.format(
legislation_file_path))
if legislation_json is None:
request = urllib2.Request(data['legislation_url'], headers = {
'User-Agent': 'OpenFisca-Web-API',
})
try:
response = urllib2.urlopen(request)
except urllib2.HTTPError:
return data, dict(legislation_url = state._(u'HTTP Error while retrieving legislation JSON'))
except urllib2.URLError:
return data, dict(legislation_url = state._(u'Error while retrieving legislation JSON'))
legislation_json, error = conv.pipe(
conv.make_input_to_json(object_pairs_hook = collections.OrderedDict),
legislations.validate_any_legislation_json,
conv.not_none,
)(response.read(), state = state)
if error is not None:
return data, dict(legislation_url = error)
if cache_dir is not None:
if not os.path.exists(legislation_dir):
os.makedirs(legislation_dir)
with open(legislation_file_path, 'w') as legislation_file:
legislation_file.write(unicode(json.dumps(legislation_json, encoding = 'utf-8',
ensure_ascii = False, indent = 2)).encode('utf-8'))
datesim = date(data['year'], 1, 1)
if legislation_json.get('datesim') is None:
dated_legislation_json = legislations.generate_dated_legislation_json(legislation_json, datesim)
else:
dated_legislation_json = legislation_json
legislation_json = None
compact_legislation = legislations.compact_dated_node_json(dated_legislation_json)
attributes = dict(
compact_legislation = compact_legislation,
declar = {},
famille = {},
indiv = {},
menage = {},
year = data['year'],
)
indiv_index_by_id = dict(
(individu_id, individu_index)
for individu_index, individu_id in enumerate(data[u'individus'].iterkeys())
)
for individu_id, individu in data[u'individus'].iteritems():
individu['noi'] = indiv_index_by_id[individu_id]
individu.pop('prenom', None)
attributes['indiv'][indiv_index_by_id[individu_id]] = individu
for famille in data[u'familles'].itervalues():
parents_id = famille.pop(u'parents')
enfants_id = famille.pop(u'enfants')
noichef = indiv_index_by_id[parents_id[0]]
attributes['famille'][noichef] = famille
for indivu_id in itertools.chain(parents_id, enfants_id):
attributes['indiv'][indiv_index_by_id[indivu_id]]['noichef'] = noichef
attributes['indiv'][noichef]['quifam'] = u'chef'
if len(parents_id) > 1:
attributes['indiv'][indiv_index_by_id[parents_id[1]]]['quifam'] = u'part'
for enfant_number, enfant_id in enumerate(enfants_id, 1):
attributes['indiv'][indiv_index_by_id[enfant_id]]['quifam'] = u'enf{}'.format(enfant_number)
for foyer_fiscal in data[u'foyers_fiscaux'].itervalues():
declarants_id = foyer_fiscal.pop(u'declarants')
personnes_a_charge_id = foyer_fiscal.pop(u'personnes_a_charge')
noidec = indiv_index_by_id[declarants_id[0]]
attributes['declar'][noidec] = foyer_fiscal
for indivu_id in itertools.chain(declarants_id, personnes_a_charge_id):
attributes['indiv'][indiv_index_by_id[indivu_id]]['noidec'] = noidec
attributes['indiv'][noidec]['quifoy'] = u'vous'
if len(declarants_id) > 1:
attributes['indiv'][indiv_index_by_id[declarants_id[1]]]['quifoy'] = u'conj'
for personne_a_charge_number, personne_a_charge_id in enumerate(personnes_a_charge_id, 1):
attributes['indiv'][indiv_index_by_id[personne_a_charge_id]]['quifoy'] = u'pac{}'.format(
personne_a_charge_number)
for menage in data[u'menages'].itervalues():
personne_de_reference_id = menage.pop(u'personne_de_reference')
conjoint_id = menage.pop(u'conjoint')
enfants_id = menage.pop(u'enfants')
autres_id = menage.pop(u'autres')
noipref = indiv_index_by_id[personne_de_reference_id]
attributes['menage'][noipref] = menage
for indivu_id in itertools.chain(
[personne_de_reference_id],
[conjoint_id] if conjoint_id is not None else [],
enfants_id,
autres_id,
):
attributes['indiv'][indiv_index_by_id[indivu_id]]['noipref'] = noipref
attributes['indiv'][noipref]['quimen'] = u'pref'
if conjoint_id is not None:
attributes['indiv'][indiv_index_by_id[conjoint_id]]['quimen'] = u'cref'
for enfant_number, enfant_id in enumerate(itertools.chain(enfants_id, autres_id), 1):
attributes['indiv'][indiv_index_by_id[enfant_id]]['quimen'] = u'enf{}'.format(enfant_number)
return attributes, None
def json_or_python_to_attributes(value, state = None):
if value is None:
return value, None
if state is None:
state = conv.default_state
# First validation and conversion step
data, error = conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
axes = conv.pipe(
conv.test_isinstance(list),
conv.uniform_sequence(
conv.pipe(
conv.test_isinstance(dict),
conv.struct(
dict(
count = conv.pipe(
conv.test_isinstance(int),
conv.test_greater_or_equal(1),
conv.not_none,
),
index = conv.pipe(
conv.test_isinstance(int),
conv.test_greater_or_equal(0),
conv.default(0),
),
max = conv.pipe(
conv.test_isinstance((float, int)),
conv.not_none,
),
min = conv.pipe(
conv.test_isinstance((float, int)),
conv.not_none,
),
name = conv.pipe(
conv.test_isinstance(basestring),
conv.test_in(column_by_name),
conv.test(lambda column_name: column_by_name[column_name]._dtype in (
np.float32, np.int16, np.int32),
error = N_(u'Invalid type for axe: integer or float expected')),
conv.not_none,
),
),
),
),
drop_none_items = True,
),
conv.empty_to_none,
),
legislation_url = conv.pipe(
conv.test_isinstance(basestring),
conv.make_input_to_url(error_if_fragment = True, full = True, schemes = ('http', 'https')),
),
test_case = conv.pipe(
self.json_or_python_to_test_case,
conv.not_none,
),
year = conv.pipe(
conv.test_isinstance(int),
conv.test_greater_or_equal(1900), # TODO: Check that year is valid in params.
conv.not_none,
),
),
),
)(value, state = state)
if error is not None:
return data, error
if data['axes'] is not None:
errors = {}
for axis_index, axis in enumerate(data['axes']):
if axis['min'] >= axis['max']:
errors.setdefault('axes', {}).setdefault(axis_index, {})['max'] = state._(
u"Max value must be greater than min value")
column = column_by_name[axis['name']]
entity_class = entities.entity_class_by_symbol[column.entity]
if axis['index'] >= len(data['test_case'][entity_class.key_plural]):
errors.setdefault('axes', {}).setdefault(axis_index, {})['index'] = state._(
u"Index must be lower than {}").format(len(data['test_case'][entity_class.key_plural]))
if errors:
return data, errors
if data['legislation_url'] is None:
compact_legislation = None
else:
legislation_json = None
if cache_dir is not None:
legislation_uuid_hex = uuid.uuid5(uuid.NAMESPACE_URL, data['legislation_url'].encode('utf-8')).hex
legislation_dir = os.path.join(cache_dir, 'legislations', legislation_uuid_hex[:2])
legislation_filename = '{}.json'.format(legislation_uuid_hex[2:])
legislation_file_path = os.path.join(legislation_dir, legislation_filename)
if os.path.exists(legislation_file_path) \
and os.path.getmtime(legislation_file_path) > time.time() - 900: # 15 minutes
with open(legislation_file_path) as legislation_file:
try:
legislation_json = json.load(legislation_file,
object_pairs_hook = collections.OrderedDict)
except ValueError:
log.exception('Error while reading legislation JSON file: {}'.format(
legislation_file_path))
if legislation_json is None:
request = urllib2.Request(data['legislation_url'], headers = {
'User-Agent': 'OpenFisca-Web-API',
})
try:
response = urllib2.urlopen(request)
except urllib2.HTTPError:
return data, dict(legislation_url = state._(u'HTTP Error while retrieving legislation JSON'))
except urllib2.URLError:
return data, dict(legislation_url = state._(u'Error while retrieving legislation JSON'))
legislation_json, error = conv.pipe(
conv.make_input_to_json(object_pairs_hook = collections.OrderedDict),
legislations.validate_any_legislation_json,
conv.not_none,
)(response.read(), state = state)
if error is not None:
return data, dict(legislation_url = error)
if cache_dir is not None:
if not os.path.exists(legislation_dir):
os.makedirs(legislation_dir)
with open(legislation_file_path, 'w') as legislation_file:
legislation_file.write(unicode(json.dumps(legislation_json, encoding = 'utf-8',
ensure_ascii = False, indent = 2)).encode('utf-8'))
datesim = datetime.date(data['year'], 1, 1)
if legislation_json.get('datesim') is None:
dated_legislation_json = legislations.generate_dated_legislation_json(legislation_json, datesim)
else:
dated_legislation_json = legislation_json
legislation_json = None
compact_legislation = legislations.compact_dated_node_json(dated_legislation_json)
if self.tax_benefit_system.preprocess_legislation_parameters is not None:
self.tax_benefit_system.preprocess_legislation_parameters(compact_legislation)
self.axes = data['axes']
self.compact_legislation = compact_legislation
self.test_case = data['test_case']
self.year = data['year']
return self, None
