def test_legislation_xml_file(): legislation_tree = xml.etree.ElementTree.parse(model.PARAM_FILE) legislation_xml_json = conv.check(legislationsxml.xml_legislation_to_json)(legislation_tree.getroot(), state = conv.default_state) legislation_xml_json, errors = legislationsxml.validate_node_xml_json(legislation_xml_json, state = conv.default_state) if errors is not None: errors = conv.embed_error(legislation_xml_json, 'errors', errors) if errors is None: raise ValueError(unicode(json.dumps(legislation_xml_json, ensure_ascii = False, indent = 2)).encode('utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)), unicode(json.dumps(legislation_xml_json, ensure_ascii = False, indent = 2)), ).encode('utf-8')) _, legislation_json = legislationsxml.transform_node_xml_json_to_json(legislation_xml_json) legislation_json, errors = legislations.validate_node_json(legislation_json, state = conv.default_state) if errors is not None: errors = conv.embed_error(legislation_json, 'errors', errors) if errors is None: raise ValueError(unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)).encode('utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)), unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)), ).encode('utf-8')) dated_legislation_json = legislations.generate_dated_legislation_json(legislation_json, datetime.date(2006, 1, 1)) # raise ValueError(unicode(json.dumps(dated_legislation_json, ensure_ascii = False, indent = 2)).encode('utf-8')) compact_legislation = legislations.compact_dated_node_json(dated_legislation_json)
def test_multiple_xml_based_tax_benefit_system(): tax_benefit_system = DummyMultipleXmlBasedTaxBenefitSystem() legislation_json = tax_benefit_system.legislation_json assert legislation_json is not None assert isinstance(legislation_json, dict), legislation_json dated_legislation_json = legislations.generate_dated_legislation_json(legislation_json, '2012-01-01') assert isinstance(dated_legislation_json, dict), legislation_json compact_legislation = legislations.compact_dated_node_json(dated_legislation_json) assert_equal(compact_legislation.csg.activite.deductible.taux, 0.051) assert_equal(compact_legislation.csg.activite.crds.activite.taux, 0.005)
def check_legislation_xml_file(year): legislation_tree = conv.check(legislationsxml.make_xml_legislation_info_list_to_xml_element(False))( TaxBenefitSystem.legislation_xml_info_list, state = conv.default_state) legislation_xml_json = conv.check(legislationsxml.xml_legislation_to_json)( legislation_tree, state = conv.default_state, ) legislation_xml_json, errors = legislationsxml.validate_legislation_xml_json(legislation_xml_json, state = conv.default_state) if errors is not None: errors = conv.embed_error(legislation_xml_json, 'errors', errors) if errors is None: raise ValueError(unicode(json.dumps(legislation_xml_json, ensure_ascii = False, indent = 2)).encode('utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)), unicode(json.dumps(legislation_xml_json, ensure_ascii = False, indent = 2)), ).encode('utf-8')) _, legislation_json = legislationsxml.transform_node_xml_json_to_json(legislation_xml_json) legislation_json, errors = legislations.validate_legislation_json(legislation_json, state = conv.default_state) if errors is not None: errors = conv.embed_error(legislation_json, 'errors', errors) if errors is None: raise ValueError(unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)).encode('utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)), unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)), ).encode('utf-8')) # Create tax_benefit system only now, to be able to debug XML validation errors in above code. tax_benefit_system = TaxBenefitSystem() if tax_benefit_system.preprocess_legislation is not None: legislation_json = tax_benefit_system.preprocess_legislation(legislation_json) legislation_json = legislations.generate_dated_legislation_json(legislation_json, year) legislation_json, errors = legislations.validate_dated_legislation_json(legislation_json, state = conv.default_state) if errors is not None: errors = conv.embed_error(legislation_json, 'errors', errors) if errors is None: raise ValueError(unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)).encode( 'utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)), unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)), ).encode('utf-8')) compact_legislation = legislations.compact_dated_node_json(legislation_json) assert compact_legislation is not None
def check_legislation_xml_file(year): legislation_tree = xml.etree.ElementTree.parse(base.TaxBenefitSystem.legislation_xml_file_path) legislation_xml_json = conv.check(legislationsxml.xml_legislation_to_json)(legislation_tree.getroot(), state = conv.default_state) legislation_xml_json, errors = legislationsxml.validate_legislation_xml_json(legislation_xml_json, state = conv.default_state) if errors is not None: errors = conv.embed_error(legislation_xml_json, 'errors', errors) if errors is None: raise ValueError(unicode(json.dumps(legislation_xml_json, ensure_ascii = False, indent = 2)).encode('utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)), unicode(json.dumps(legislation_xml_json, ensure_ascii = False, indent = 2)), ).encode('utf-8')) _, legislation_json = legislationsxml.transform_node_xml_json_to_json(legislation_xml_json) legislation_json, errors = legislations.validate_legislation_json(legislation_json, state = conv.default_state) if errors is not None: errors = conv.embed_error(legislation_json, 'errors', errors) if errors is None: raise ValueError(unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)).encode('utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)), unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)), ).encode('utf-8')) legislation_json = legislations.generate_dated_legislation_json(legislation_json, year) legislation_json, errors = legislations.validate_dated_legislation_json(legislation_json, state = conv.default_state) if errors is not None: errors = conv.embed_error(legislation_json, 'errors', errors) if errors is None: raise ValueError(unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)).encode( 'utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)), unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)), ).encode('utf-8')) compact_legislation = legislations.compact_dated_node_json(legislation_json) # Create tax_benefit system only now, to be able to debug XML validation errors in above code. if base.tax_benefit_system.preprocess_compact_legislation is not None: base.tax_benefit_system.preprocess_compact_legislation(compact_legislation)
def test_legislation_xml_file(): legislation_tree = xml.etree.ElementTree.parse(model.PARAM_FILE) legislation_xml_json = conv.check(legislationsxml.xml_legislation_to_json)(legislation_tree.getroot(), state = conv.default_state) legislation_xml_json, errors = legislationsxml.validate_legislation_xml_json(legislation_xml_json, state = conv.default_state) if errors is not None: errors = conv.embed_error(legislation_xml_json, 'errors', errors) if errors is None: raise ValueError(unicode(json.dumps(legislation_xml_json, ensure_ascii = False, indent = 2)).encode('utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)), unicode(json.dumps(legislation_xml_json, ensure_ascii = False, indent = 2)), ).encode('utf-8')) _, legislation_json = legislationsxml.transform_node_xml_json_to_json(legislation_xml_json) legislation_json, errors = legislations.validate_legislation_json(legislation_json, state = conv.default_state) if errors is not None: errors = conv.embed_error(legislation_json, 'errors', errors) if errors is None: raise ValueError(unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)).encode('utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)), unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)), ).encode('utf-8')) dated_legislation_json = legislations.generate_dated_legislation_json(legislation_json, datetime.date(2006, 1, 1)) dated_legislation_json, errors = legislations.validate_dated_legislation_json(dated_legislation_json, state = conv.default_state) if errors is not None: errors = conv.embed_error(dated_legislation_json, 'errors', errors) if errors is None: raise ValueError(unicode(json.dumps(dated_legislation_json, ensure_ascii = False, indent = 2)).encode( 'utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)), unicode(json.dumps(dated_legislation_json, ensure_ascii = False, indent = 2)), ).encode('utf-8')) compact_legislation = legislations.compact_dated_node_json(dated_legislation_json)
def json_to_attributes(value, state=None): if value is None: return value, None if state is None: state = conv.default_state # First validation and conversion step data, error = conv.pipe( conv.test_isinstance(dict), conv.struct( dict( familles=conv.pipe( conv.condition( conv.test_isinstance(list), conv.pipe( conv.uniform_sequence(conv.test_isinstance(dict), drop_none_items=True), conv.function( lambda values: collections.OrderedDict( (value.pop("id", index), value) for index, value in enumerate(values) ) ), ), ), conv.test_isinstance(dict), conv.uniform_mapping( conv.pipe(conv.test_isinstance((basestring, int)), conv.not_none), conv.pipe( conv.test_isinstance(dict), conv.struct( dict( itertools.chain( dict( enfants=conv.pipe( conv.test_isinstance(list), conv.uniform_sequence( conv.test_isinstance((basestring, int)), drop_none_items=True, ), conv.default([]), ), parents=conv.pipe( conv.test_isinstance(list), conv.uniform_sequence( conv.test_isinstance((basestring, int)), drop_none_items=True, ), conv.empty_to_none, conv.not_none, ), ).iteritems(), ( (column.name, column.json_to_python) for column in column_by_name.itervalues() if column.entity == "fam" ), ) ) ), ), drop_none_values=True, ), conv.empty_to_none, conv.not_none, ), foyers_fiscaux=conv.pipe( conv.condition( conv.test_isinstance(list), conv.pipe( conv.uniform_sequence(conv.test_isinstance(dict), drop_none_items=True), conv.function( lambda values: collections.OrderedDict( (value.pop("id", index), value) for index, value in enumerate(values) ) ), ), ), conv.test_isinstance(dict), conv.uniform_mapping( conv.pipe(conv.test_isinstance((basestring, int)), conv.not_none), conv.pipe( conv.test_isinstance(dict), conv.struct( dict( itertools.chain( dict( declarants=conv.pipe( conv.test_isinstance(list), conv.uniform_sequence( conv.test_isinstance((basestring, int)), drop_none_items=True, ), conv.empty_to_none, conv.not_none, ), personnes_a_charge=conv.pipe( conv.test_isinstance(list), conv.uniform_sequence( conv.test_isinstance((basestring, int)), drop_none_items=True, ), conv.default([]), ), ).iteritems(), ( (column.name, column.json_to_python) for column in column_by_name.itervalues() if column.entity == "foy" ), ) ) ), ), drop_none_values=True, ), conv.empty_to_none, conv.not_none, ), individus=conv.pipe( conv.condition( conv.test_isinstance(list), conv.pipe( conv.uniform_sequence(conv.test_isinstance(dict), drop_none_items=True), conv.function( lambda values: collections.OrderedDict( (value.pop("id", index), value) for index, value in enumerate(values) ) ), ), ), conv.test_isinstance(dict), conv.uniform_mapping( conv.pipe(conv.test_isinstance((basestring, int)), conv.not_none), conv.pipe( conv.test_isinstance(dict), conv.struct( dict( itertools.chain( dict( birth=conv.pipe( conv.test_isinstance(basestring), conv.iso8601_input_to_date, conv.not_none, ), prenom=conv.pipe( conv.test_isinstance(basestring), conv.cleanup_line ), ).iteritems(), ( (column.name, column.json_to_python) for column in column_by_name.itervalues() if column.entity == "ind" and column.name not in ( "age", "agem", "idfam", "idfoy", "idmen", "quifam", "quifoy", "quimen", ) ), ) ) ), ), drop_none_values=True, ), conv.empty_to_none, conv.not_none, ), legislation_url=conv.pipe( conv.make_input_to_url( error_if_fragment=True, full=True, schemes=("file", "http", "https") ), conv.not_none, ), menages=conv.pipe( conv.condition( conv.test_isinstance(list), conv.pipe( conv.uniform_sequence(conv.test_isinstance(dict), drop_none_items=True), conv.function( lambda values: collections.OrderedDict( (value.pop("id", index), value) for index, value in enumerate(values) ) ), ), ), conv.test_isinstance(dict), conv.uniform_mapping( conv.pipe(conv.test_isinstance((basestring, int)), conv.not_none), conv.pipe( conv.test_isinstance(dict), conv.struct( dict( itertools.chain( dict( autres=conv.pipe( # personnes ayant un lien autre avec la personne de référence conv.test_isinstance(list), conv.uniform_sequence( conv.test_isinstance((basestring, int)), drop_none_items=True, ), conv.default([]), ), conjoint=conv.test_isinstance((basestring, int)), # conjoint de la personne de référence enfants=conv.pipe( # enfants de la personne de référence ou de son conjoint conv.test_isinstance(list), conv.uniform_sequence( conv.test_isinstance((basestring, int)), drop_none_items=True, ), conv.default([]), ), personne_de_reference=conv.pipe( conv.test_isinstance((basestring, int)), conv.not_none ), ).iteritems(), ( (column.name, column.json_to_python) for column in column_by_name.itervalues() if column.entity == "men" ), ) ) ), ), drop_none_values=True, ), conv.empty_to_none, conv.not_none, ), year=conv.pipe( conv.test_isinstance(int), conv.test_greater_or_equal(1900), # TODO: Check that year is valid in params. conv.not_none, ), ) ), )(value, state=state) if error is not None: return data, error # Second validation step familles_individus_id = set(data["individus"].iterkeys()) foyers_fiscaux_individus_id = set(data["individus"].iterkeys()) menages_individus_id = set(data["individus"].iterkeys()) data, error = conv.struct( dict( familles=conv.uniform_mapping( conv.noop, conv.struct( dict( enfants=conv.uniform_sequence(conv.test_in_pop(familles_individus_id)), parents=conv.uniform_sequence(conv.test_in_pop(familles_individus_id)), ), default=conv.noop, ), ), foyers_fiscaux=conv.uniform_mapping( conv.noop, conv.struct( dict( declarants=conv.uniform_sequence(conv.test_in_pop(foyers_fiscaux_individus_id)), personnes_a_charge=conv.uniform_sequence(conv.test_in_pop(foyers_fiscaux_individus_id)), ), default=conv.noop, ), ), menages=conv.uniform_mapping( conv.noop, conv.struct( dict( autres=conv.uniform_sequence(conv.test_in_pop(menages_individus_id)), conjoint=conv.test_in_pop(menages_individus_id), enfants=conv.uniform_sequence(conv.test_in_pop(menages_individus_id)), personne_de_reference=conv.test_in_pop(menages_individus_id), ), default=conv.noop, ), ), ), default=conv.noop, )(data, state=state) remaining_individus_id = familles_individus_id.union(foyers_fiscaux_individus_id, menages_individus_id) if remaining_individus_id: if error is None: error = {} for individu_id in remaining_individus_id: error.setdefault("individus", {})[individu_id] = state._(u"Individual is missing from {}").format( u" & ".join( word for word in [ u"familles" if individu_id not in familles_individus_id else None, u"foyers_fiscaux" if individu_id not in foyers_fiscaux_individus_id else None, u"menages" if individu_id not in menages_individus_id else None, ] if word is not None ) ) if error is not None: return data, error if cache_dir is not None: legislation_uuid_hex = uuid.uuid5(uuid.NAMESPACE_URL, data["legislation_url"].encode("utf-8")).hex legislation_dir = os.path.join(cache_dir, "legislations", legislation_uuid_hex[:2]) legislation_filename = "{}.json".format(legislation_uuid_hex[2:]) legislation_file_path = os.path.join(legislation_dir, legislation_filename) legislation_json = None if ( os.path.exists(legislation_file_path) and os.path.getmtime(legislation_file_path) > time.time() - 900 ): # 15 minutes with open(legislation_file_path) as legislation_file: try: legislation_json = json.load(legislation_file, object_pairs_hook=collections.OrderedDict) except ValueError: log.exception("Error while reading legislation JSON file: {}".format(legislation_file_path)) if legislation_json is None: request = urllib2.Request(data["legislation_url"], headers={"User-Agent": "OpenFisca-Web-API"}) try: response = urllib2.urlopen(request) except urllib2.HTTPError: return data, dict(legislation_url=state._(u"HTTP Error while retrieving legislation JSON")) except urllib2.URLError: return data, dict(legislation_url=state._(u"Error while retrieving legislation JSON")) legislation_json, error = conv.pipe( conv.make_input_to_json(object_pairs_hook=collections.OrderedDict), legislations.validate_node_json, conv.not_none, )(response.read(), state=state) if error is not None: return data, dict(legislation_url=error) if cache_dir is not None: if not os.path.exists(legislation_dir): os.makedirs(legislation_dir) with open(legislation_file_path, "w") as legislation_file: legislation_file.write( unicode( json.dumps(legislation_json, encoding="utf-8", ensure_ascii=False, indent=2) ).encode("utf-8") ) datesim = date(data["year"], 1, 1) dated_legislation_json = legislations.generate_dated_legislation_json(legislation_json, datesim) compact_legislation = legislations.compact_dated_node_json(dated_legislation_json) attributes = dict( compact_legislation=compact_legislation, declar={}, famille={}, indiv={}, menage={}, year=data["year"] ) indiv_index_by_id = dict( (individu_id, individu_index) for individu_index, individu_id in enumerate(data[u"individus"].iterkeys()) ) for individu_id, individu in data[u"individus"].iteritems(): individu.pop("prenom", None) attributes["indiv"][indiv_index_by_id[individu_id]] = individu for famille in data[u"familles"].itervalues(): parents_id = famille.pop(u"parents") enfants_id = famille.pop(u"enfants") noichef = indiv_index_by_id[parents_id[0]] attributes["declar"][noichef] = famille for indivu_id in itertools.chain(parents_id, enfants_id): attributes["indiv"][indiv_index_by_id[indivu_id]]["noichef"] = noichef attributes["indiv"][noichef]["quifam"] = u"chef" if len(parents_id) > 1: attributes["indiv"][indiv_index_by_id[parents_id[1]]]["quifam"] = u"part" for enfant_number, enfant_id in enumerate(enfants_id, 1): attributes["indiv"][indiv_index_by_id[enfant_id]]["quifam"] = u"enf{}".format(enfant_number) for foyer_fiscal in data[u"foyers_fiscaux"].itervalues(): declarants_id = foyer_fiscal.pop(u"declarants") personnes_a_charge_id = foyer_fiscal.pop(u"personnes_a_charge") noidec = indiv_index_by_id[declarants_id[0]] attributes["declar"][noidec] = foyer_fiscal for indivu_id in itertools.chain(declarants_id, personnes_a_charge_id): attributes["indiv"][indiv_index_by_id[indivu_id]]["noidec"] = noidec attributes["indiv"][noidec]["quifoy"] = u"vous" if len(declarants_id) > 1: attributes["indiv"][indiv_index_by_id[declarants_id[1]]]["quifoy"] = u"conj" for personne_a_charge_number, personne_a_charge_id in enumerate(personnes_a_charge_id, 1): attributes["indiv"][indiv_index_by_id[personne_a_charge_id]]["quifoy"] = u"pac{}".format( personne_a_charge_number ) for menage in data[u"menages"].itervalues(): personne_de_reference_id = menage.pop(u"personne_de_reference") conjoint_id = menage.pop(u"conjoint") enfants_id = menage.pop(u"enfants") autres_id = menage.pop(u"autres") noipref = indiv_index_by_id[personne_de_reference_id] attributes["declar"][noipref] = menage for indivu_id in itertools.chain( [personne_de_reference_id], [conjoint_id] if conjoint_id is not None else [], enfants_id, autres_id ): attributes["indiv"][indiv_index_by_id[indivu_id]]["noipref"] = noipref attributes["indiv"][noipref]["quimen"] = u"pref" if conjoint_id is not None: attributes["indiv"][indiv_index_by_id[conjoint_id]]["quimen"] = u"cref" for enfant_number, enfant_id in enumerate(itertools.chain(enfants_id, autres_id), 1): attributes["indiv"][indiv_index_by_id[enfant_id]]["quimen"] = u"enf{}".format(enfant_number) return attributes, None
def json_to_attributes(cls, value, state = None): if value is None: return value, None if state is None: state = conv.default_state # First validation and conversion step data, error = conv.pipe( conv.test_isinstance(dict), conv.struct( dict( familles = conv.pipe( conv.condition( conv.test_isinstance(list), conv.pipe( conv.uniform_sequence( conv.test_isinstance(dict), drop_none_items = True, ), conv.function(lambda values: collections.OrderedDict( (value.pop('id', index), value) for index, value in enumerate(values) )), ), ), conv.test_isinstance(dict), conv.uniform_mapping( conv.pipe( conv.test_isinstance((basestring, int)), conv.not_none, ), conv.pipe( conv.test_isinstance(dict), conv.struct( dict(itertools.chain( dict( enfants = conv.pipe( conv.test_isinstance(list), conv.uniform_sequence( conv.test_isinstance((basestring, int)), drop_none_items = True, ), conv.default([]), ), parents = conv.pipe( conv.test_isinstance(list), conv.uniform_sequence( conv.test_isinstance((basestring, int)), drop_none_items = True, ), conv.empty_to_none, conv.not_none, ), ).iteritems(), ( (column.name, column.json_to_python) for column in column_by_name.itervalues() if column.entity == 'fam' ), )), ), ), drop_none_values = True, ), conv.empty_to_none, conv.not_none, ), foyers_fiscaux = conv.pipe( conv.condition( conv.test_isinstance(list), conv.pipe( conv.uniform_sequence( conv.test_isinstance(dict), drop_none_items = True, ), conv.function(lambda values: collections.OrderedDict( (value.pop('id', index), value) for index, value in enumerate(values) )), ), ), conv.test_isinstance(dict), conv.uniform_mapping( conv.pipe( conv.test_isinstance((basestring, int)), conv.not_none, ), conv.pipe( conv.test_isinstance(dict), conv.struct( dict(itertools.chain( dict( declarants = conv.pipe( conv.test_isinstance(list), conv.uniform_sequence( conv.test_isinstance((basestring, int)), drop_none_items = True, ), conv.empty_to_none, conv.not_none, ), personnes_a_charge = conv.pipe( conv.test_isinstance(list), conv.uniform_sequence( conv.test_isinstance((basestring, int)), drop_none_items = True, ), conv.default([]), ), ).iteritems(), ( (column.name, column.json_to_python) for column in column_by_name.itervalues() if column.entity == 'foy' ), )), ), ), drop_none_values = True, ), conv.empty_to_none, conv.not_none, ), individus = conv.pipe( conv.condition( conv.test_isinstance(list), conv.pipe( conv.uniform_sequence( conv.test_isinstance(dict), drop_none_items = True, ), conv.function(lambda values: collections.OrderedDict( (value.pop('id', index), value) for index, value in enumerate(values) )), ), ), conv.test_isinstance(dict), conv.uniform_mapping( conv.pipe( conv.test_isinstance((basestring, int)), conv.not_none, ), conv.pipe( conv.test_isinstance(dict), conv.struct( dict(itertools.chain( dict( birth = conv.pipe( conv.test_isinstance(basestring), conv.iso8601_input_to_date, conv.not_none, ), ).iteritems(), ( (column.name, column.json_to_python) for column in column_by_name.itervalues() if column.entity == 'ind' and column.name not in ('age', 'agem', 'idfam', 'idfoy', 'idmen', 'quifam', 'quifoy', 'quimen') ), )), ), ), drop_none_values = True, ), conv.empty_to_none, conv.not_none, ), legislation_url = conv.pipe( conv.make_input_to_url(error_if_fragment = True, full = True), conv.not_none, ), menages = conv.pipe( conv.condition( conv.test_isinstance(list), conv.pipe( conv.uniform_sequence( conv.test_isinstance(dict), drop_none_items = True, ), conv.function(lambda values: collections.OrderedDict( (value.pop('id', index), value) for index, value in enumerate(values) )), ), ), conv.test_isinstance(dict), conv.uniform_mapping( conv.pipe( conv.test_isinstance((basestring, int)), conv.not_none, ), conv.pipe( conv.test_isinstance(dict), conv.struct( dict(itertools.chain( dict( autres = conv.pipe( # personnes ayant un lien autre avec la personne de référence conv.test_isinstance(list), conv.uniform_sequence( conv.test_isinstance((basestring, int)), drop_none_items = True, ), conv.default([]), ), conjoint = conv.test_isinstance((basestring, int)), # conjoint de la personne de référence enfants = conv.pipe( # enfants de la personne de référence ou de son conjoint conv.test_isinstance(list), conv.uniform_sequence( conv.test_isinstance((basestring, int)), drop_none_items = True, ), conv.default([]), ), personne_de_reference = conv.pipe( conv.test_isinstance((basestring, int)), conv.not_none, ), ).iteritems(), ( (column.name, column.json_to_python) for column in column_by_name.itervalues() if column.entity == 'men' ), )), ), ), drop_none_values = True, ), conv.empty_to_none, conv.not_none, ), year = conv.pipe( conv.test_isinstance(int), conv.test_greater_or_equal(1900), # TODO: Check that year is valid in params. conv.not_none, ), ), ), )(value, state = state) if error is not None: return data, error # Second validation step familles_individus_id = set(data['individus'].iterkeys()) foyers_fiscaux_individus_id = set(data['individus'].iterkeys()) menages_individus_id = set(data['individus'].iterkeys()) data, error = conv.struct( dict( familles = conv.uniform_mapping( conv.noop, conv.struct( dict( enfants = conv.uniform_sequence(conv.test_in_pop(familles_individus_id)), parents = conv.uniform_sequence(conv.test_in_pop(familles_individus_id)), ), default = conv.noop, ), ), foyers_fiscaux = conv.uniform_mapping( conv.noop, conv.struct( dict( declarants = conv.uniform_sequence(conv.test_in_pop(foyers_fiscaux_individus_id)), personnes_a_charge = conv.uniform_sequence(conv.test_in_pop(foyers_fiscaux_individus_id)), ), default = conv.noop, ), ), menages = conv.uniform_mapping( conv.noop, conv.struct( dict( autres = conv.uniform_sequence(conv.test_in_pop(menages_individus_id)), conjoint = conv.test_in_pop(menages_individus_id), enfants = conv.uniform_sequence(conv.test_in_pop(menages_individus_id)), personne_de_reference = conv.test_in_pop(menages_individus_id), ), default = conv.noop, ), ), ), default = conv.noop, )(data, state = state) remaining_individus_id = familles_individus_id.union(foyers_fiscaux_individus_id, menages_individus_id) if remaining_individus_id: if error is None: error = {} for individu_id in remaining_individus_id: error.setdefault('individus', {})[individu_id] = state._(u"Individual is missing from {}").format( u' & '.join([ u'familles' if individu_id in familles_individus_id else None, u'foyers_fiscaux' if individu_id in foyers_fiscaux_individus_id else None, u'menages' if individu_id in menages_individus_id else None, ])) if error is not None: return data, error request = urllib2.Request(data['legislation_url'], headers = { 'User-Agent': 'OpenFisca-Web-API', }) try: response = urllib2.urlopen(request) except urllib2.HTTPError: return data, dict(legislation_url = ctx._(u'HTTP Error while retrieving legislation JSON')) except urllib2.URLError: return data, dict(legislation_url = ctx._(u'Error while retrieving legislation JSON')) legislation_json, error = conv.pipe( conv.make_input_to_json(object_pairs_hook = collections.OrderedDict), legislations.validate_node_json, conv.not_none, )(response.read(), state = state) if error is not None: return data, dict(legislation_url = error) datesim = date(data['year'], 1, 1) dated_legislation_json = legislations.generate_dated_legislation_json(legislation_json, datesim) compact_legislation = legislations.compact_dated_node_json(dated_legislation_json) attributes = dict( compact_legislation = compact_legislation, declar = {}, famille = {}, indiv = {}, menage = {}, year = data['year'], ) indiv_index_by_id = dict( (individu_id, individu_index) for individu_index, individu_id in enumerate(data[u'individus'].iterkeys()) ) for individu_id, individu in data[u'individus'].iteritems(): attributes['indiv'][indiv_index_by_id[individu_id]] = individu for famille in data[u'familles'].itervalues(): parents_id = famille.pop(u'parents') enfants_id = famille.pop(u'enfants') noichef = indiv_index_by_id[parents_id[0]] attributes['declar'][noichef] = famille for indivu_id in itertools.chain(parents_id, enfants_id): attributes['indiv'][indiv_index_by_id[indivu_id]]['noichef'] = noichef attributes['indiv'][noichef]['quifam'] = u'chef' if len(parents_id) > 1: attributes['indiv'][indiv_index_by_id[parents_id[1]]]['quifam'] = u'part' for enfant_number, enfant_id in enumerate(enfants_id, 1): attributes['indiv'][indiv_index_by_id[enfant_id]]['quifam'] = u'enf{}'.format(enfant_number) for foyer_fiscal in data[u'foyers_fiscaux'].itervalues(): declarants_id = foyer_fiscal.pop(u'declarants') personnes_a_charge_id = foyer_fiscal.pop(u'personnes_a_charge') noidec = indiv_index_by_id[declarants_id[0]] attributes['declar'][noidec] = foyer_fiscal for indivu_id in itertools.chain(declarants_id, personnes_a_charge_id): attributes['indiv'][indiv_index_by_id[indivu_id]]['noidec'] = noidec attributes['indiv'][noidec]['quifoy'] = u'vous' if len(declarants_id) > 1: attributes['indiv'][indiv_index_by_id[declarants_id[1]]]['quifoy'] = u'conj' for personne_a_charge_number, personne_a_charge_id in enumerate(personnes_a_charge_id, 1): attributes['indiv'][indiv_index_by_id[personne_a_charge_id]]['quifoy'] = u'pac{}'.format( personne_a_charge_number) for menage in data[u'menages'].itervalues(): personne_de_reference_id = menage.pop(u'personne_de_reference') conjoint_id = menage.pop(u'conjoint') enfants_id = menage.pop(u'enfants') autres_id = menage.pop(u'autres') noipref = indiv_index_by_id[personne_de_reference_id] attributes['declar'][noipref] = menage for indivu_id in itertools.chain( [personne_de_reference_id], [conjoint_id] if conjoint_id is not None else [], enfants_id, autres_id, ): attributes['indiv'][indiv_index_by_id[indivu_id]]['noipref'] = noipref attributes['indiv'][noipref]['quimen'] = u'pref' if conjoint_id is not None: attributes['indiv'][indiv_index_by_id[conjoint_id]]['quimen'] = u'cref' for enfant_number, enfant_id in enumerate(itertools.chain(enfants_id, autres_id), 1): attributes['indiv'][indiv_index_by_id[enfant_id]]['quimen'] = u'enf{}'.format(enfant_number) return attributes, None
def check_legislation_xml_file(year): legislation_tree = conv.check( legislationsxml.make_xml_legislation_info_list_to_xml_element(False))( TaxBenefitSystem.legislation_xml_info_list, state=conv.default_state) legislation_xml_json = conv.check(legislationsxml.xml_legislation_to_json)( legislation_tree, state=conv.default_state, ) legislation_xml_json, errors = legislationsxml.validate_legislation_xml_json( legislation_xml_json, state=conv.default_state) if errors is not None: errors = conv.embed_error(legislation_xml_json, 'errors', errors) if errors is None: raise ValueError( unicode( json.dumps(legislation_xml_json, ensure_ascii=False, indent=2)).encode('utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode( json.dumps(errors, ensure_ascii=False, indent=2, sort_keys=True)), unicode( json.dumps(legislation_xml_json, ensure_ascii=False, indent=2)), ).encode('utf-8')) _, legislation_json = legislationsxml.transform_node_xml_json_to_json( legislation_xml_json) legislation_json, errors = legislations.validate_legislation_json( legislation_json, state=conv.default_state) if errors is not None: errors = conv.embed_error(legislation_json, 'errors', errors) if errors is None: raise ValueError( unicode( json.dumps(legislation_json, ensure_ascii=False, indent=2)).encode('utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode( json.dumps(errors, ensure_ascii=False, indent=2, sort_keys=True)), unicode(json.dumps(legislation_json, ensure_ascii=False, indent=2)), ).encode('utf-8')) # Create tax_benefit system only now, to be able to debug XML validation errors in above code. tax_benefit_system = TaxBenefitSystem() if tax_benefit_system.preprocess_legislation is not None: legislation_json = tax_benefit_system.preprocess_legislation( legislation_json) legislation_json = legislations.generate_dated_legislation_json( legislation_json, year) legislation_json, errors = legislations.validate_dated_legislation_json( legislation_json, state=conv.default_state) if errors is not None: errors = conv.embed_error(legislation_json, 'errors', errors) if errors is None: raise ValueError( unicode( json.dumps(legislation_json, ensure_ascii=False, indent=2)).encode('utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode( json.dumps(errors, ensure_ascii=False, indent=2, sort_keys=True)), unicode(json.dumps(legislation_json, ensure_ascii=False, indent=2)), ).encode('utf-8')) compact_legislation = legislations.compact_dated_node_json( legislation_json) assert compact_legislation is not None
def json_or_python_to_attributes(value, state = None): if value is None: return value, None if state is None: state = conv.default_state # First validation and conversion step data, error = conv.pipe( conv.test_isinstance(dict), conv.struct( dict( axes = conv.pipe( conv.test_isinstance(list), conv.uniform_sequence( conv.pipe( conv.test_isinstance(dict), conv.struct( dict( count = conv.pipe( conv.test_isinstance(int), conv.test_greater_or_equal(1), conv.not_none, ), index = conv.pipe( conv.test_isinstance(int), conv.test_greater_or_equal(0), conv.default(0), ), max = conv.pipe( conv.test_isinstance((float, int)), conv.not_none, ), min = conv.pipe( conv.test_isinstance((float, int)), conv.not_none, ), name = conv.pipe( conv.test_isinstance(basestring), conv.test_in(column_by_name), conv.test(lambda column_name: column_by_name[column_name].dtype in ( np.float32, np.int16, np.int32), error = N_(u'Invalid type for axe: integer or float expected')), conv.not_none, ), ), ), ), drop_none_items = True, ), conv.empty_to_none, ), legislation_url = conv.pipe( conv.test_isinstance(basestring), conv.make_input_to_url(error_if_fragment = True, full = True, schemes = ('http', 'https')), ), test_case = conv.pipe( self.json_or_python_to_test_case, conv.not_none, ), year = conv.pipe( conv.test_isinstance(int), conv.test_greater_or_equal(1900), # TODO: Check that year is valid in params. conv.not_none, ), ), ), )(value, state = state) if error is not None: return data, error errors = {} if data['axes'] is not None: for axis_index, axis in enumerate(data['axes']): if axis['min'] >= axis['max']: errors.setdefault('axes', {}).setdefault(axis_index, {})['max'] = state._( u"Max value must be greater than min value") column = column_by_name[axis['name']] entity_class = entities.entity_class_by_symbol[column.entity] if axis['index'] >= len(data['test_case'][entity_class.key_plural]): errors.setdefault('axes', {}).setdefault(axis_index, {})['index'] = state._( u"Index must be lower than {}").format(len(data['test_case'][entity_class.key_plural])) famille_by_id = data['test_case']['familles'] parents_id = set( parent_id for famille in data['test_case']['familles'].itervalues() for parent_id in famille['parents'] ) individu_by_id = data['test_case']['individus'] data, errors = conv.struct( dict( test_case = conv.struct( dict( foyers_fiscaux = conv.uniform_mapping( conv.noop, conv.struct( dict( declarants = conv.uniform_sequence(conv.pipe( conv.test(lambda individu_id: individu_by_id[individu_id].get('birth') is None or data['year'] - individu_by_id[individu_id]['birth'].year >= 18, error = u"Un déclarant d'un foyer fiscal doit être agé d'au moins 18" u" ans", ), conv.test(lambda individu_id: individu_by_id[individu_id].get('age') is None or individu_by_id[individu_id]['age'] >= 18, error = u"Un déclarant d'un foyer fiscal doit être agé d'au moins 18" u" ans", ), conv.test(lambda individu_id: individu_by_id[individu_id].get('agem') is None or individu_by_id[individu_id]['agem'] >= 18 * 12, error = u"Un déclarant d'un foyer fiscal doit être agé d'au moins 18" u" ans", ), conv.test(lambda individu_id: individu_id in parents_id, error = u"Un déclarant ou un conjoint sur la feuille d'impôt, doit être" u" un parent dans sa famille", ), )), personnes_a_charge = conv.uniform_sequence(conv.pipe( conv.test(lambda individu_id: individu_by_id[individu_id].get('inv', False) or individu_by_id[individu_id].get('birth') is None or data['year'] - individu_by_id[individu_id]['birth'].year <= 25, error = u"Une personne à charge d'un foyer fiscal doit avoir moins de" u" 25 ans ou être invalide", ), conv.test(lambda individu_id: individu_by_id[individu_id].get('inv', False) or individu_by_id[individu_id].get('age') is None or individu_by_id[individu_id]['age'] <= 25, error = u"Une personne à charge d'un foyer fiscal doit avoir moins de" u" 25 ans ou être invalide", ), conv.test(lambda individu_id: individu_by_id[individu_id].get('inv', False) or individu_by_id[individu_id].get('agem') is None or individu_by_id[individu_id]['agem'] <= 25 * 12, error = u"Une personne à charge d'un foyer fiscal doit avoir moins de" u" 25 ans ou être invalide", ), )), ), default = conv.noop, ), ), individus = conv.uniform_mapping( conv.noop, conv.struct( dict( birth = conv.test(lambda birth: data['year'] - birth.year >= 0, error = u"L'individu doit être né au plus tard l'année de la simulation", ), ), default = conv.noop, drop_none_values = 'missing', ), ), ), default = conv.noop, ), ), default = conv.noop, )(data, state = state) if errors: return data, errors if data['legislation_url'] is None: compact_legislation = None else: legislation_json = None if cache_dir is not None: legislation_uuid_hex = uuid.uuid5(uuid.NAMESPACE_URL, data['legislation_url'].encode('utf-8')).hex legislation_dir = os.path.join(cache_dir, 'legislations', legislation_uuid_hex[:2]) legislation_filename = '{}.json'.format(legislation_uuid_hex[2:]) legislation_file_path = os.path.join(legislation_dir, legislation_filename) if os.path.exists(legislation_file_path) \ and os.path.getmtime(legislation_file_path) > time.time() - 900: # 15 minutes with open(legislation_file_path) as legislation_file: try: legislation_json = json.load(legislation_file, object_pairs_hook = collections.OrderedDict) except ValueError: log.exception('Error while reading legislation JSON file: {}'.format( legislation_file_path)) if legislation_json is None: request = urllib2.Request(data['legislation_url'], headers = { 'User-Agent': 'OpenFisca-Web-API', }) try: response = urllib2.urlopen(request) except urllib2.HTTPError: return data, dict(legislation_url = state._(u'HTTP Error while retrieving legislation JSON')) except urllib2.URLError: return data, dict(legislation_url = state._(u'Error while retrieving legislation JSON')) legislation_json, error = conv.pipe( conv.make_input_to_json(object_pairs_hook = collections.OrderedDict), legislations.validate_any_legislation_json, conv.not_none, )(response.read(), state = state) if error is not None: return data, dict(legislation_url = error) if cache_dir is not None: if not os.path.exists(legislation_dir): os.makedirs(legislation_dir) with open(legislation_file_path, 'w') as legislation_file: legislation_file.write(unicode(json.dumps(legislation_json, encoding = 'utf-8', ensure_ascii = False, indent = 2)).encode('utf-8')) datesim = datetime.date(data['year'], 1, 1) if legislation_json.get('datesim') is None: dated_legislation_json = legislations.generate_dated_legislation_json(legislation_json, datesim) else: dated_legislation_json = legislation_json legislation_json = None compact_legislation = legislations.compact_dated_node_json(dated_legislation_json) if self.tax_benefit_system.preprocess_legislation_parameters is not None: self.tax_benefit_system.preprocess_legislation_parameters(compact_legislation) self.axes = data['axes'] self.compact_legislation = compact_legislation self.test_case = data['test_case'] self.year = data['year'] return self, None
def json_to_attributes(value, state = None): if value is None: return value, None if state is None: state = conv.default_state # First validation and conversion step data, error = conv.pipe( conv.test_isinstance(dict), conv.struct( dict( familles = conv.pipe( conv.condition( conv.test_isinstance(list), conv.pipe( conv.uniform_sequence( conv.test_isinstance(dict), drop_none_items = True, ), conv.function(lambda values: collections.OrderedDict( (value.pop('id', index), value) for index, value in enumerate(values) )), ), ), conv.test_isinstance(dict), conv.uniform_mapping( conv.pipe( conv.test_isinstance((basestring, int)), conv.not_none, ), conv.pipe( conv.test_isinstance(dict), conv.struct( dict(itertools.chain( dict( enfants = conv.pipe( conv.test_isinstance(list), conv.uniform_sequence( conv.test_isinstance((basestring, int)), drop_none_items = True, ), conv.default([]), ), parents = conv.pipe( conv.test_isinstance(list), conv.uniform_sequence( conv.test_isinstance((basestring, int)), drop_none_items = True, ), conv.empty_to_none, conv.not_none, conv.test(lambda parents: len(parents) <= 2, error = N_(u'A "famille" must have at most 2 "parents"')) ), ).iteritems(), ( (column.name, column.json_to_python) for column in column_by_name.itervalues() if column.entity == 'fam' ), )), ), ), drop_none_values = True, ), conv.empty_to_none, conv.not_none, ), foyers_fiscaux = conv.pipe( conv.condition( conv.test_isinstance(list), conv.pipe( conv.uniform_sequence( conv.test_isinstance(dict), drop_none_items = True, ), conv.function(lambda values: collections.OrderedDict( (value.pop('id', index), value) for index, value in enumerate(values) )), ), ), conv.test_isinstance(dict), conv.uniform_mapping( conv.pipe( conv.test_isinstance((basestring, int)), conv.not_none, ), conv.pipe( conv.test_isinstance(dict), conv.struct( dict(itertools.chain( dict( declarants = conv.pipe( conv.test_isinstance(list), conv.uniform_sequence( conv.test_isinstance((basestring, int)), drop_none_items = True, ), conv.empty_to_none, conv.not_none, conv.test(lambda declarants: len(declarants) <= 2, error = N_(u'A "foyer_fiscal" must have at most 2 "declarants"' )) ), personnes_a_charge = conv.pipe( conv.test_isinstance(list), conv.uniform_sequence( conv.test_isinstance((basestring, int)), drop_none_items = True, ), conv.default([]), ), ).iteritems(), ( (column.name, column.json_to_python) for column in column_by_name.itervalues() if column.entity == 'foy' ), )), ), ), drop_none_values = True, ), conv.empty_to_none, conv.not_none, ), individus = conv.pipe( conv.condition( conv.test_isinstance(list), conv.pipe( conv.uniform_sequence( conv.test_isinstance(dict), drop_none_items = True, ), conv.function(lambda values: collections.OrderedDict( (value.pop('id', index), value) for index, value in enumerate(values) )), ), ), conv.test_isinstance(dict), conv.uniform_mapping( conv.pipe( conv.test_isinstance((basestring, int)), conv.not_none, ), conv.pipe( conv.test_isinstance(dict), conv.struct( dict(itertools.chain( dict( birth = conv.pipe( conv.condition( conv.test_isinstance(int), conv.pipe( conv.test_between(1870, 2099), conv.function(lambda year: date(year, 1, 1)), ), conv.pipe( conv.test_isinstance(basestring), conv.test(year_or_month_or_day_re.match, error = N_(u'Invalid year')), conv.function(lambda birth: u'-'.join(( birth.split(u'-') + [u'01', u'01'])[:3])), conv.iso8601_input_to_date, conv.test_between(date(1870, 1, 1), date(2099, 12, 31)), ), ), conv.not_none, ), prenom = conv.pipe( conv.test_isinstance(basestring), conv.cleanup_line, ), ).iteritems(), ( (column.name, column.json_to_python) for column in column_by_name.itervalues() if column.entity == 'ind' and column.name not in ('age', 'agem', 'idfam', 'idfoy', 'idmen', 'quifam', 'quifoy', 'quimen') ), )), ), ), drop_none_values = True, ), conv.empty_to_none, conv.not_none, ), legislation_url = conv.pipe( conv.test_isinstance(basestring), conv.make_input_to_url(error_if_fragment = True, full = True, schemes = ('http', 'https')), conv.default(legislation_url) if legislation_url is not None else conv.not_none, ), menages = conv.pipe( conv.condition( conv.test_isinstance(list), conv.pipe( conv.uniform_sequence( conv.test_isinstance(dict), drop_none_items = True, ), conv.function(lambda values: collections.OrderedDict( (value.pop('id', index), value) for index, value in enumerate(values) )), ), ), conv.test_isinstance(dict), conv.uniform_mapping( conv.pipe( conv.test_isinstance((basestring, int)), conv.not_none, ), conv.pipe( conv.test_isinstance(dict), conv.struct( dict(itertools.chain( dict( autres = conv.pipe( # personnes ayant un lien autre avec la personne de référence conv.test_isinstance(list), conv.uniform_sequence( conv.test_isinstance((basestring, int)), drop_none_items = True, ), conv.default([]), ), conjoint = conv.test_isinstance((basestring, int)), # conjoint de la personne de référence enfants = conv.pipe( # enfants de la personne de référence ou de son conjoint conv.test_isinstance(list), conv.uniform_sequence( conv.test_isinstance((basestring, int)), drop_none_items = True, ), conv.default([]), ), personne_de_reference = conv.pipe( conv.test_isinstance((basestring, int)), conv.not_none, ), ).iteritems(), ( (column.name, column.json_to_python) for column in column_by_name.itervalues() if column.entity == 'men' ), )), ), ), drop_none_values = True, ), conv.empty_to_none, conv.not_none, ), year = conv.pipe( conv.test_isinstance(int), conv.test_greater_or_equal(1900), # TODO: Check that year is valid in params. conv.not_none, ), ), ), )(value, state = state) if error is not None: return data, error # Second validation step familles_individus_id = set(data['individus'].iterkeys()) foyers_fiscaux_individus_id = set(data['individus'].iterkeys()) menages_individus_id = set(data['individus'].iterkeys()) data, error = conv.struct( dict( familles = conv.uniform_mapping( conv.noop, conv.struct( dict( enfants = conv.uniform_sequence(conv.test_in_pop(familles_individus_id)), parents = conv.uniform_sequence(conv.test_in_pop(familles_individus_id)), ), default = conv.noop, ), ), foyers_fiscaux = conv.uniform_mapping( conv.noop, conv.struct( dict( declarants = conv.uniform_sequence(conv.test_in_pop(foyers_fiscaux_individus_id)), personnes_a_charge = conv.uniform_sequence(conv.test_in_pop(foyers_fiscaux_individus_id)), ), default = conv.noop, ), ), menages = conv.uniform_mapping( conv.noop, conv.struct( dict( autres = conv.uniform_sequence(conv.test_in_pop(menages_individus_id)), conjoint = conv.test_in_pop(menages_individus_id), enfants = conv.uniform_sequence(conv.test_in_pop(menages_individus_id)), personne_de_reference = conv.test_in_pop(menages_individus_id), ), default = conv.noop, ), ), ), default = conv.noop, )(data, state = state) remaining_individus_id = familles_individus_id.union(foyers_fiscaux_individus_id, menages_individus_id) if remaining_individus_id: if error is None: error = {} for individu_id in remaining_individus_id: error.setdefault('individus', {})[individu_id] = state._(u"Individual is missing from {}").format( u' & '.join( word for word in [ u'familles' if individu_id not in familles_individus_id else None, u'foyers_fiscaux' if individu_id not in foyers_fiscaux_individus_id else None, u'menages' if individu_id not in menages_individus_id else None, ] if word is not None )) if error is not None: return data, error if cache_dir is not None: legislation_uuid_hex = uuid.uuid5(uuid.NAMESPACE_URL, data['legislation_url'].encode('utf-8')).hex legislation_dir = os.path.join(cache_dir, 'legislations', legislation_uuid_hex[:2]) legislation_filename = '{}.json'.format(legislation_uuid_hex[2:]) legislation_file_path = os.path.join(legislation_dir, legislation_filename) legislation_json = None if os.path.exists(legislation_file_path) \ and os.path.getmtime(legislation_file_path) > time.time() - 900: # 15 minutes with open(legislation_file_path) as legislation_file: try: legislation_json = json.load(legislation_file, object_pairs_hook = collections.OrderedDict) except ValueError: log.exception('Error while reading legislation JSON file: {}'.format( legislation_file_path)) if legislation_json is None: request = urllib2.Request(data['legislation_url'], headers = { 'User-Agent': 'OpenFisca-Web-API', }) try: response = urllib2.urlopen(request) except urllib2.HTTPError: return data, dict(legislation_url = state._(u'HTTP Error while retrieving legislation JSON')) except urllib2.URLError: return data, dict(legislation_url = state._(u'Error while retrieving legislation JSON')) legislation_json, error = conv.pipe( conv.make_input_to_json(object_pairs_hook = collections.OrderedDict), legislations.validate_any_legislation_json, conv.not_none, )(response.read(), state = state) if error is not None: return data, dict(legislation_url = error) if cache_dir is not None: if not os.path.exists(legislation_dir): os.makedirs(legislation_dir) with open(legislation_file_path, 'w') as legislation_file: legislation_file.write(unicode(json.dumps(legislation_json, encoding = 'utf-8', ensure_ascii = False, indent = 2)).encode('utf-8')) datesim = date(data['year'], 1, 1) if legislation_json.get('datesim') is None: dated_legislation_json = legislations.generate_dated_legislation_json(legislation_json, datesim) else: dated_legislation_json = legislation_json legislation_json = None compact_legislation = legislations.compact_dated_node_json(dated_legislation_json) attributes = dict( compact_legislation = compact_legislation, declar = {}, famille = {}, indiv = {}, menage = {}, year = data['year'], ) indiv_index_by_id = dict( (individu_id, individu_index) for individu_index, individu_id in enumerate(data[u'individus'].iterkeys()) ) for individu_id, individu in data[u'individus'].iteritems(): individu['noi'] = indiv_index_by_id[individu_id] individu.pop('prenom', None) attributes['indiv'][indiv_index_by_id[individu_id]] = individu for famille in data[u'familles'].itervalues(): parents_id = famille.pop(u'parents') enfants_id = famille.pop(u'enfants') noichef = indiv_index_by_id[parents_id[0]] attributes['famille'][noichef] = famille for indivu_id in itertools.chain(parents_id, enfants_id): attributes['indiv'][indiv_index_by_id[indivu_id]]['noichef'] = noichef attributes['indiv'][noichef]['quifam'] = u'chef' if len(parents_id) > 1: attributes['indiv'][indiv_index_by_id[parents_id[1]]]['quifam'] = u'part' for enfant_number, enfant_id in enumerate(enfants_id, 1): attributes['indiv'][indiv_index_by_id[enfant_id]]['quifam'] = u'enf{}'.format(enfant_number) for foyer_fiscal in data[u'foyers_fiscaux'].itervalues(): declarants_id = foyer_fiscal.pop(u'declarants') personnes_a_charge_id = foyer_fiscal.pop(u'personnes_a_charge') noidec = indiv_index_by_id[declarants_id[0]] attributes['declar'][noidec] = foyer_fiscal for indivu_id in itertools.chain(declarants_id, personnes_a_charge_id): attributes['indiv'][indiv_index_by_id[indivu_id]]['noidec'] = noidec attributes['indiv'][noidec]['quifoy'] = u'vous' if len(declarants_id) > 1: attributes['indiv'][indiv_index_by_id[declarants_id[1]]]['quifoy'] = u'conj' for personne_a_charge_number, personne_a_charge_id in enumerate(personnes_a_charge_id, 1): attributes['indiv'][indiv_index_by_id[personne_a_charge_id]]['quifoy'] = u'pac{}'.format( personne_a_charge_number) for menage in data[u'menages'].itervalues(): personne_de_reference_id = menage.pop(u'personne_de_reference') conjoint_id = menage.pop(u'conjoint') enfants_id = menage.pop(u'enfants') autres_id = menage.pop(u'autres') noipref = indiv_index_by_id[personne_de_reference_id] attributes['menage'][noipref] = menage for indivu_id in itertools.chain( [personne_de_reference_id], [conjoint_id] if conjoint_id is not None else [], enfants_id, autres_id, ): attributes['indiv'][indiv_index_by_id[indivu_id]]['noipref'] = noipref attributes['indiv'][noipref]['quimen'] = u'pref' if conjoint_id is not None: attributes['indiv'][indiv_index_by_id[conjoint_id]]['quimen'] = u'cref' for enfant_number, enfant_id in enumerate(itertools.chain(enfants_id, autres_id), 1): attributes['indiv'][indiv_index_by_id[enfant_id]]['quimen'] = u'enf{}'.format(enfant_number) return attributes, None
def json_or_python_to_attributes(value, state = None): if value is None: return value, None if state is None: state = conv.default_state # First validation and conversion step data, error = conv.pipe( conv.test_isinstance(dict), conv.struct( dict( axes = conv.pipe( conv.test_isinstance(list), conv.uniform_sequence( conv.pipe( conv.test_isinstance(dict), conv.struct( dict( count = conv.pipe( conv.test_isinstance(int), conv.test_greater_or_equal(1), conv.not_none, ), index = conv.pipe( conv.test_isinstance(int), conv.test_greater_or_equal(0), conv.default(0), ), max = conv.pipe( conv.test_isinstance((float, int)), conv.not_none, ), min = conv.pipe( conv.test_isinstance((float, int)), conv.not_none, ), name = conv.pipe( conv.test_isinstance(basestring), conv.test_in(column_by_name), conv.test(lambda column_name: column_by_name[column_name]._dtype in ( np.float32, np.int16, np.int32), error = N_(u'Invalid type for axe: integer or float expected')), conv.not_none, ), ), ), ), drop_none_items = True, ), conv.empty_to_none, ), legislation_url = conv.pipe( conv.test_isinstance(basestring), conv.make_input_to_url(error_if_fragment = True, full = True, schemes = ('http', 'https')), ), test_case = conv.pipe( self.json_or_python_to_test_case, conv.not_none, ), year = conv.pipe( conv.test_isinstance(int), conv.test_greater_or_equal(1900), # TODO: Check that year is valid in params. conv.not_none, ), ), ), )(value, state = state) if error is not None: return data, error if data['axes'] is not None: errors = {} for axis_index, axis in enumerate(data['axes']): if axis['min'] >= axis['max']: errors.setdefault('axes', {}).setdefault(axis_index, {})['max'] = state._( u"Max value must be greater than min value") column = column_by_name[axis['name']] entity_class = entities.entity_class_by_symbol[column.entity] if axis['index'] >= len(data['test_case'][entity_class.key_plural]): errors.setdefault('axes', {}).setdefault(axis_index, {})['index'] = state._( u"Index must be lower than {}").format(len(data['test_case'][entity_class.key_plural])) if errors: return data, errors if data['legislation_url'] is None: compact_legislation = None else: legislation_json = None if cache_dir is not None: legislation_uuid_hex = uuid.uuid5(uuid.NAMESPACE_URL, data['legislation_url'].encode('utf-8')).hex legislation_dir = os.path.join(cache_dir, 'legislations', legislation_uuid_hex[:2]) legislation_filename = '{}.json'.format(legislation_uuid_hex[2:]) legislation_file_path = os.path.join(legislation_dir, legislation_filename) if os.path.exists(legislation_file_path) \ and os.path.getmtime(legislation_file_path) > time.time() - 900: # 15 minutes with open(legislation_file_path) as legislation_file: try: legislation_json = json.load(legislation_file, object_pairs_hook = collections.OrderedDict) except ValueError: log.exception('Error while reading legislation JSON file: {}'.format( legislation_file_path)) if legislation_json is None: request = urllib2.Request(data['legislation_url'], headers = { 'User-Agent': 'OpenFisca-Web-API', }) try: response = urllib2.urlopen(request) except urllib2.HTTPError: return data, dict(legislation_url = state._(u'HTTP Error while retrieving legislation JSON')) except urllib2.URLError: return data, dict(legislation_url = state._(u'Error while retrieving legislation JSON')) legislation_json, error = conv.pipe( conv.make_input_to_json(object_pairs_hook = collections.OrderedDict), legislations.validate_any_legislation_json, conv.not_none, )(response.read(), state = state) if error is not None: return data, dict(legislation_url = error) if cache_dir is not None: if not os.path.exists(legislation_dir): os.makedirs(legislation_dir) with open(legislation_file_path, 'w') as legislation_file: legislation_file.write(unicode(json.dumps(legislation_json, encoding = 'utf-8', ensure_ascii = False, indent = 2)).encode('utf-8')) datesim = datetime.date(data['year'], 1, 1) if legislation_json.get('datesim') is None: dated_legislation_json = legislations.generate_dated_legislation_json(legislation_json, datesim) else: dated_legislation_json = legislation_json legislation_json = None compact_legislation = legislations.compact_dated_node_json(dated_legislation_json) if self.tax_benefit_system.preprocess_legislation_parameters is not None: self.tax_benefit_system.preprocess_legislation_parameters(compact_legislation) self.axes = data['axes'] self.compact_legislation = compact_legislation self.test_case = data['test_case'] self.year = data['year'] return self, None