def init_single_entity(self, axes = None, enfants = None, famille = None, parent1 = None, parent2 = None,
period = None):
if enfants is None:
enfants = []
assert parent1 is not None
famille = famille.copy() if famille is not None else {}
individus = []
for index, individu in enumerate([parent1, parent2] + (enfants or [])):
if individu is None:
continue
id = individu.get('id')
if id is None:
individu = individu.copy()
individu['id'] = id = 'ind{}'.format(index)
individus.append(individu)
if index <= 1:
famille.setdefault('parents', []).append(id)
else:
famille.setdefault('enfants', []).append(id)
conv.check(self.make_json_or_python_to_attributes())(dict(
axes = axes,
period = period,
test_case = dict(
familles = [famille],
individus = individus,
),
))
return self
def test():
dir_path = os.path.join(os.path.dirname(__file__), 'formulas')
for filename in sorted(os.listdir(dir_path)):
if not filename.endswith('.yaml'):
continue
filename_core = os.path.splitext(filename)[0]
with open(os.path.join(dir_path, filename)) as yaml_file:
tests = yaml.load(yaml_file)
tests, error = conv.pipe(
conv.make_item_to_singleton(),
conv.uniform_sequence(
conv.noop,
drop_none_items = True,
),
)(tests)
if error is not None:
embedding_error = conv.embed_error(tests, u'errors', error)
assert embedding_error is None, embedding_error
conv.check((tests, error)) # Generate an error.
for test in tests:
test, error = scenarios.make_json_or_python_to_test(tax_benefit_system)(test)
if error is not None:
embedding_error = conv.embed_error(test, u'errors', error)
assert embedding_error is None, embedding_error
conv.check((test, error)) # Generate an error.
if test.get(u'ignore', False):
continue
yield check, test.get('name') or filename_core, unicode(test['scenario'].period), test
def init_single_entity(
self, autres=None, axes=None, conjoint=None, enfants=None, menage=None, period=None, personne_de_reference=None
):
if enfants is None:
enfants = []
if autres is None:
autres = []
assert personne_de_reference is not None
menage = menage.copy() if menage is not None else {}
individus = []
for index, individu in enumerate([personne_de_reference, conjoint] + (enfants or []) + (autres or [])):
if individu is None:
continue
id = individu.get("id")
if id is None:
individu = individu.copy()
individu["id"] = id = "ind{}".format(index)
individus.append(individu)
if index == 0:
menage["personne_de_reference"] = id
elif index == 1:
menage["conjoint"] = id
else:
menage.setdefault("enfants", []).append(id)
conv.check(self.make_json_or_python_to_attributes())(
dict(axes=axes, period=period, test_case=dict(menages=[menage], individus=individus))
)
return self
def init_single_entity(self, axes = None, enfants = None, foyer_fiscal = None, menage = None,
parent1 = None, parent2 = None, period = None):
if enfants is None:
enfants = []
assert parent1 is not None
foyer_fiscal = foyer_fiscal.copy() if foyer_fiscal is not None else {}
individus = []
menage = menage.copy() if menage is not None else {}
for index, individu in enumerate([parent1, parent2] + (enfants or [])):
if individu is None:
continue
id = individu.get('id')
if id is None:
individu = individu.copy()
individu['id'] = id = 'ind{}'.format(index)
individus.append(individu)
if index <= 1:
foyer_fiscal.setdefault('declarants', []).append(id)
if index == 0:
menage['personne_de_reference'] = id
else:
menage['conjoint'] = id
else:
foyer_fiscal.setdefault('personnes_a_charge', []).append(id)
menage.setdefault('enfants', []).append(id)
conv.check(self.make_json_or_python_to_attributes())(dict(
axes = axes,
period = period,
test_case = dict(
foyers_fiscaux = [foyer_fiscal],
individus = individus,
menages = [menage],
),
))
return self
def init_single_entity(self,
axes=None,
enfants=None,
famille=None,
parent1=None,
parent2=None,
period=None):
if enfants is None:
enfants = []
assert parent1 is not None
famille = famille.copy() if famille is not None else {}
individus = []
for index, individu in enumerate([parent1, parent2] + (enfants or [])):
if individu is None:
continue
id = individu.get('id')
if id is None:
individu = individu.copy()
individu['id'] = id = 'ind{}'.format(index)
individus.append(individu)
if index <= 1:
famille.setdefault('parents', []).append(id)
else:
famille.setdefault('children', []).append(id)
conv.check(self.make_json_or_python_to_attributes())(dict(
axes=axes,
period=period,
test_case=dict(
households=[famille],
persons=individus,
),
))
return self
def check_legislation_xml_file(year):
legislation_tree = conv.check(legislationsxml.make_xml_legislation_info_list_to_xml_element(False))(
TaxBenefitSystem.legislation_xml_info_list, state = conv.default_state)
legislation_xml_json = conv.check(legislationsxml.xml_legislation_to_json)(
legislation_tree,
state = conv.default_state,
)
legislation_xml_json, errors = legislationsxml.validate_legislation_xml_json(legislation_xml_json,
state = conv.default_state)
if errors is not None:
errors = conv.embed_error(legislation_xml_json, 'errors', errors)
if errors is None:
raise ValueError(unicode(json.dumps(legislation_xml_json, ensure_ascii = False,
indent = 2)).encode('utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)),
unicode(json.dumps(legislation_xml_json, ensure_ascii = False, indent = 2)),
).encode('utf-8'))
_, legislation_json = legislationsxml.transform_node_xml_json_to_json(legislation_xml_json)
legislation_json, errors = legislations.validate_legislation_json(legislation_json, state = conv.default_state)
if errors is not None:
errors = conv.embed_error(legislation_json, 'errors', errors)
if errors is None:
raise ValueError(unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)).encode('utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)),
unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)),
).encode('utf-8'))
# Create tax_benefit system only now, to be able to debug XML validation errors in above code.
tax_benefit_system = TaxBenefitSystem()
if tax_benefit_system.preprocess_legislation is not None:
legislation_json = tax_benefit_system.preprocess_legislation(legislation_json)
legislation_json = legislations.generate_dated_legislation_json(legislation_json, year)
legislation_json, errors = legislations.validate_dated_legislation_json(legislation_json,
state = conv.default_state)
if errors is not None:
errors = conv.embed_error(legislation_json, 'errors', errors)
if errors is None:
raise ValueError(unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)).encode(
'utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)),
unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)),
).encode('utf-8'))
compact_legislation = legislations.compact_dated_node_json(legislation_json)
assert compact_legislation is not None
def get_decomposition_json(tax_benefit_system, xml_file_path=None):
if xml_file_path is None:
xml_file_path = tax_benefit_system.decomposition_file_path
decomposition_tree = ElementTree.parse(xml_file_path)
decomposition_xml_json = conv.check(
decompositionsxml.xml_decomposition_to_json)(
decomposition_tree.getroot(), state=conv.State)
decomposition_xml_json = conv.check(
decompositionsxml.make_validate_node_xml_json(tax_benefit_system))(
decomposition_xml_json, state=conv.State)
decomposition_json = decompositionsxml.transform_node_xml_json_to_json(
decomposition_xml_json)
return decomposition_json
def extract_formula_function_infos(function, state = None):
formula_function = conv.check(state.FormulaFunctionFileInput.parse)(function,
state = state)
definition_colon_node = formula_function.node.children[3]
assert definition_colon_node.type == tokens.COLON
comment = formula_function.node.children[3].get_suffix()
if comment:
comment = comment.decode('utf-8').strip().lstrip('#').lstrip()
if comment:
comment = u' # ' + comment
parameters_text = u', '.join(itertools.chain(
[u'self'],
(
parameter_name
for parameter_name in formula_function.positional_parameters
if parameter_name != 'self'
),
(
u'{} = {}'.format(parameter_name, parameter_value)
for parameter_name, parameter_value in formula_function.named_parameters.iteritems()
if parameter_name != 'self'
),
))
suite_node = formula_function.node.children[4]
assert suite_node.type == symbols.suite
body_text = u' ' + '\n '.join(textwrap.dedent(unicode(suite_node).strip()).split('\n'))
return dict(
body = body_text,
comment = comment,
parameters = parameters_text,
)
def test_legislation_xml_file():
legislation_tree = xml.etree.ElementTree.parse(model.PARAM_FILE)
legislation_xml_json = conv.check(legislationsxml.xml_legislation_to_json)(legislation_tree.getroot(),
state = conv.default_state)
legislation_xml_json, errors = legislationsxml.validate_node_xml_json(legislation_xml_json,
state = conv.default_state)
if errors is not None:
errors = conv.embed_error(legislation_xml_json, 'errors', errors)
if errors is None:
raise ValueError(unicode(json.dumps(legislation_xml_json, ensure_ascii = False,
indent = 2)).encode('utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)),
unicode(json.dumps(legislation_xml_json, ensure_ascii = False, indent = 2)),
).encode('utf-8'))
_, legislation_json = legislationsxml.transform_node_xml_json_to_json(legislation_xml_json)
legislation_json, errors = legislations.validate_node_json(legislation_json, state = conv.default_state)
if errors is not None:
errors = conv.embed_error(legislation_json, 'errors', errors)
if errors is None:
raise ValueError(unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)).encode('utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)),
unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)),
).encode('utf-8'))
dated_legislation_json = legislations.generate_dated_legislation_json(legislation_json, datetime.date(2006, 1, 1))
# raise ValueError(unicode(json.dumps(dated_legislation_json, ensure_ascii = False, indent = 2)).encode('utf-8'))
compact_legislation = legislations.compact_dated_node_json(dated_legislation_json)
def check_decomposition_xml_file(file_path):
decomposition_tree = xml.etree.ElementTree.parse(os.path.join(file_path))
decomposition_xml_json = conv.check(decompositionsxml.xml_decomposition_to_json)(decomposition_tree.getroot(),
state = conv.default_state)
decomposition_xml_json, errors = decompositionsxml.make_validate_node_xml_json(base.tax_benefit_system)(
decomposition_xml_json, state = conv.default_state)
if errors is not None:
errors = conv.embed_error(decomposition_xml_json, 'errors', errors)
if errors is None:
raise ValueError(unicode(json.dumps(decomposition_xml_json, ensure_ascii = False,
indent = 2)).encode('utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)),
unicode(json.dumps(decomposition_xml_json, ensure_ascii = False, indent = 2)),
).encode('utf-8'))
decomposition_json = decompositionsxml.transform_node_xml_json_to_json(decomposition_xml_json)
decomposition_json, errors = decompositions.make_validate_node_json(base.tax_benefit_system)(
decomposition_json, state = conv.default_state)
if errors is not None:
errors = conv.embed_error(decomposition_json, 'errors', errors)
if errors is None:
raise ValueError(unicode(json.dumps(decomposition_json, ensure_ascii = False, indent = 2)).encode('utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)),
unicode(json.dumps(decomposition_json, ensure_ascii = False, indent = 2)),
).encode('utf-8'))
def init_single_entity(self,
axes=None,
enfants=None,
famille=None,
foyer_fiscal=None,
menage=None,
parent1=None,
parent2=None,
period=None):
if enfants is None:
enfants = []
assert parent1 is not None
famille = famille.copy() if famille is not None else {}
foyer_fiscal = foyer_fiscal.copy() if foyer_fiscal is not None else {}
individus = []
menage = menage.copy() if menage is not None else {}
for index, individu in enumerate([parent1, parent2] + (enfants or [])):
if individu is None:
continue
id = individu.get('id')
if id is None:
individu = individu.copy()
individu['id'] = id = 'ind{}'.format(index)
individus.append(individu)
if index <= 1:
famille.setdefault('parents', []).append(id)
foyer_fiscal.setdefault('declarants', []).append(id)
if index == 0:
menage['personne_de_reference'] = id
else:
menage['conjoint'] = id
else:
famille.setdefault('enfants', []).append(id)
foyer_fiscal.setdefault('personnes_a_charge', []).append(id)
menage.setdefault('enfants', []).append(id)
conv.check(self.make_json_or_python_to_attributes())(dict(
axes=axes,
period=period,
test_case=dict(
familles=[famille],
foyers_fiscaux=[foyer_fiscal],
individus=individus,
menages=[menage],
),
))
return self
def test(current_options_by_dir = None, force = False, name_filter = None):
if current_options_by_dir is None:
current_options_by_dir = options_by_dir
for dir, options in sorted(current_options_by_dir.iteritems()):
if not force and options.get('ignore', False):
log.info(u'Ignoring directory: {}'.format(dir))
continue
if not os.path.isdir(dir):
log.warning(u'Skipping missing directory: {}'.format(dir))
continue
if isinstance(name_filter, str):
name_filter = name_filter.decode('utf-8')
for filename in sorted(os.listdir(dir)):
if not filename.endswith('.yaml'):
continue
filename_core = os.path.splitext(filename)[0]
with open(os.path.join(dir, filename)) as yaml_file:
tests = yaml.load(yaml_file)
tests, error = conv.pipe(
conv.make_item_to_singleton(),
conv.uniform_sequence(
conv.noop,
drop_none_items = True,
),
)(tests)
if error is not None:
embedding_error = conv.embed_error(tests, u'errors', error)
assert embedding_error is None, embedding_error
conv.check((tests, error)) # Generate an error.
for test in tests:
test, error = scenarios.make_json_or_python_to_test(get_tax_benefit_system(options.get('reform')),
default_absolute_error_margin = options['default_absolute_error_margin'])(test)
if error is not None:
embedding_error = conv.embed_error(test, u'errors', error)
assert embedding_error is None, embedding_error
conv.check((test, error)) # Generate an error.
if not force and test.get(u'ignore', False):
continue
if name_filter is not None and name_filter not in filename_core \
and name_filter not in (test.get('name', u'')) \
and name_filter not in (test.get('keywords', [])):
continue
checker = check_any_period if options['accept_other_period'] else check
yield checker, test.get('name') or filename_core, unicode(test['scenario'].period), test, force
def check(test_id, test_name, scenario_data, output_variables):
scenario = conv.check(tax_benefit_system.Scenario.make_json_to_instance(
tax_benefit_system = tax_benefit_system,
))(scenario_data)
scenario.suggest()
simulation = scenario.new_simulation(debug = True)
if output_variables is not None:
for variable_name, expected_value in output_variables.iteritems():
assert_near2(simulation.calculate(variable_name, accept_other_period = True), expected_value,
error_margin = 0.007, message = "{}: ".format(variable_name))
def init_single_entity(self,
autres=None,
axes=None,
conjoint=None,
enfants=None,
menage=None,
period=None,
personne_de_reference=None):
if enfants is None:
enfants = []
if autres is None:
autres = []
assert personne_de_reference is not None
menage = menage.copy() if menage is not None else {}
individus = []
for index, individu in enumerate([personne_de_reference, conjoint] +
(enfants or []) + (autres or [])):
if individu is None:
continue
id = individu.get('id')
if id is None:
individu = individu.copy()
individu['id'] = id = 'ind{}'.format(index)
individus.append(individu)
if index == 0:
menage['personne_de_reference'] = id
elif index == 1:
menage['conjoint'] = id
else:
menage.setdefault('enfants', []).append(id)
conv.check(self.make_json_or_python_to_attributes())(dict(
axes=axes,
period=period,
test_case=dict(
menages=[menage],
individus=individus,
),
))
return self
def check_legislation_xml_file(year):
legislation_tree = xml.etree.ElementTree.parse(base.TaxBenefitSystem.legislation_xml_file_path)
legislation_xml_json = conv.check(legislationsxml.xml_legislation_to_json)(legislation_tree.getroot(),
state = conv.default_state)
legislation_xml_json, errors = legislationsxml.validate_legislation_xml_json(legislation_xml_json,
state = conv.default_state)
if errors is not None:
errors = conv.embed_error(legislation_xml_json, 'errors', errors)
if errors is None:
raise ValueError(unicode(json.dumps(legislation_xml_json, ensure_ascii = False,
indent = 2)).encode('utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)),
unicode(json.dumps(legislation_xml_json, ensure_ascii = False, indent = 2)),
).encode('utf-8'))
_, legislation_json = legislationsxml.transform_node_xml_json_to_json(legislation_xml_json)
legislation_json, errors = legislations.validate_legislation_json(legislation_json, state = conv.default_state)
if errors is not None:
errors = conv.embed_error(legislation_json, 'errors', errors)
if errors is None:
raise ValueError(unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)).encode('utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)),
unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)),
).encode('utf-8'))
legislation_json = legislations.generate_dated_legislation_json(legislation_json, year)
legislation_json, errors = legislations.validate_dated_legislation_json(legislation_json,
state = conv.default_state)
if errors is not None:
errors = conv.embed_error(legislation_json, 'errors', errors)
if errors is None:
raise ValueError(unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)).encode(
'utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)),
unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)),
).encode('utf-8'))
compact_legislation = legislations.compact_dated_node_json(legislation_json)
# Create tax_benefit system only now, to be able to debug XML validation errors in above code.
if base.tax_benefit_system.preprocess_compact_legislation is not None:
base.tax_benefit_system.preprocess_compact_legislation(compact_legislation)
def test_legislation_xml_file():
legislation_tree = xml.etree.ElementTree.parse(model.PARAM_FILE)
legislation_xml_json = conv.check(legislationsxml.xml_legislation_to_json)(legislation_tree.getroot(),
state = conv.default_state)
legislation_xml_json, errors = legislationsxml.validate_legislation_xml_json(legislation_xml_json,
state = conv.default_state)
if errors is not None:
errors = conv.embed_error(legislation_xml_json, 'errors', errors)
if errors is None:
raise ValueError(unicode(json.dumps(legislation_xml_json, ensure_ascii = False,
indent = 2)).encode('utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)),
unicode(json.dumps(legislation_xml_json, ensure_ascii = False, indent = 2)),
).encode('utf-8'))
_, legislation_json = legislationsxml.transform_node_xml_json_to_json(legislation_xml_json)
legislation_json, errors = legislations.validate_legislation_json(legislation_json, state = conv.default_state)
if errors is not None:
errors = conv.embed_error(legislation_json, 'errors', errors)
if errors is None:
raise ValueError(unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)).encode('utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)),
unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)),
).encode('utf-8'))
dated_legislation_json = legislations.generate_dated_legislation_json(legislation_json, datetime.date(2006, 1, 1))
dated_legislation_json, errors = legislations.validate_dated_legislation_json(dated_legislation_json,
state = conv.default_state)
if errors is not None:
errors = conv.embed_error(dated_legislation_json, 'errors', errors)
if errors is None:
raise ValueError(unicode(json.dumps(dated_legislation_json, ensure_ascii = False, indent = 2)).encode(
'utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)),
unicode(json.dumps(dated_legislation_json, ensure_ascii = False, indent = 2)),
).encode('utf-8'))
compact_legislation = legislations.compact_dated_node_json(dated_legislation_json)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-v',
'--verbose',
action='store_true',
default=False,
help="increase output verbosity")
args = parser.parse_args()
logging.basicConfig(
level=logging.DEBUG if args.verbose else logging.WARNING,
stream=sys.stdout)
if not os.path.exists(income_taxes_test_cases_dir):
os.makedirs(income_taxes_test_cases_dir)
if os.path.exists(tests_dir):
for filename in os.listdir(tests_dir):
if filename.endswith(filename):
os.remove(os.path.join(tests_dir, filename))
else:
os.makedirs(tests_dir)
if os.path.exists(variables_name_file_path):
with open(variables_name_file_path) as variables_name_file:
name_by_year_by_code = yaml.load(variables_name_file)
else:
name_by_year_by_code = {}
name_by_year_by_code_changed = False
for json_filename in sorted(os.listdir(json_dir)):
if not json_filename.endswith('.json'):
continue
log.info(u"Converting file {}...".format(json_filename))
with open(os.path.join(json_dir, json_filename)) as json_file:
data = conv.check(input_to_json_data)(json_file.read())
scenario = data['scenario']
tax_calculator_inputs = transform_scenario_to_tax_calculator_inputs(
scenario)
tax_year = scenario.period.start.year + 1
if tax_year <= 2011:
# Tax calculator is no more available for years before 2011.
tax_calculator_outputs = collections.OrderedDict()
tax_calculator_outputs_infos = data['resultat_officiel']
for code, infos in tax_calculator_outputs_infos.iteritems():
float_value = infos['value']
int_value = int(float_value)
tax_calculator_outputs[
code] = int_value if float_value == int_value else float_value
name = infos['name'].strip().rstrip(u'*').rstrip()
name = u' '.join(name.split()) # Remove duplicate spaces.
if name not in (u'', u'?', u'nom inconnu'):
name_by_year = name_by_year_by_code.setdefault(code, {})
current_name = name_by_year.get(tax_year)
if current_name is not None and current_name != name \
and not name.lower().endswith(current_name.lower()):
log.warning(
u'Ignoring rename of variable {} for year {} from:\n {}\nto:\n {}'
.format(code, tax_year, current_name, name))
elif current_name != name:
name_by_year[tax_year] = name
name_by_year_by_code_changed = True
else:
page = call_tax_calculator(tax_year, tax_calculator_inputs)
page_doc = etree.parse(cStringIO.StringIO(page), html_parser)
codes_without_name = set()
tax_calculator_outputs = collections.OrderedDict()
for element in page_doc.xpath('//input[@type="hidden"][@name]'):
code = element.get('name')
name = None
parent = element.getparent()
parent_tag = parent.tag.lower()
if parent_tag == 'table':
tr = parent[parent.index(element) - 1]
assert tr.tag.lower() == 'tr', tr
elif parent_tag == 'tr':
tr = parent
elif code == 'NAPCR':
name = u'Contributions sociales supplémentaires'
else:
codes_without_name.add(code)
continue
if name is None:
while True:
name = etree.tostring(
tr[1], encoding=unicode,
method='text').strip().rstrip(u'*').rstrip()
if name:
name = u' '.join(
name.split()) # Remove duplicate spaces.
break
table = tr.getparent()
tr = table[table.index(tr) - 1]
codes_without_name.discard(code)
float_value = float(element.get('value').strip())
int_value = int(float_value)
tax_calculator_outputs[
code] = int_value if float_value == int_value else float_value
name_by_year = name_by_year_by_code.setdefault(code, {})
current_name = name_by_year.get(tax_year)
if current_name is not None and current_name != name \
and not name.lower().endswith(current_name.lower()):
log.warning(
u'Renaming variable {} for year {} from:\n {}\nto:\n {}'
.format(code, tax_year, current_name, name))
if current_name != name and (
current_name is None
or not name.lower().endswith(current_name.lower())):
name_by_year[tax_year] = name
name_by_year_by_code_changed = True
assert not codes_without_name, 'Output variables {} have no name in page:\n{}'.format(
sorted(codes_without_name),
page.decode('iso-8859-1').encode('utf-8'))
# Create or update test for "calculateur impôt".
sorted_tax_calculator_inputs = collections.OrderedDict(
sorted(tax_calculator_inputs.iteritems()))
income_taxes_test_case_file_path = os.path.join(
income_taxes_test_cases_dir, '{}.yaml'.format(
hashlib.md5(
json.dumps(sorted_tax_calculator_inputs)).hexdigest()))
if os.path.exists(income_taxes_test_case_file_path):
with open(income_taxes_test_case_file_path
) as income_taxes_test_case_file:
income_taxes_test_case = yaml.load(income_taxes_test_case_file)
income_taxes_test_case['output_variables'][str(
tax_year)] = collections.OrderedDict(
sorted(tax_calculator_outputs.iteritems()))
income_taxes_test_case[
'output_variables'] = collections.OrderedDict(
sorted(income_taxes_test_case['output_variables'].
iteritems()))
else:
income_taxes_test_case = collections.OrderedDict((
('input_variables', sorted_tax_calculator_inputs),
('output_variables',
collections.OrderedDict(
((str(tax_year),
collections.OrderedDict(
sorted(tax_calculator_outputs.iteritems()))), ))),
))
with open(income_taxes_test_case_file_path,
'w') as income_taxes_test_case_file:
yaml.dump(income_taxes_test_case,
income_taxes_test_case_file,
allow_unicode=True,
default_flow_style=False,
indent=2,
width=120)
# Create or update YAML file containing the names associated to each result code of "calculateur impôt".
if name_by_year_by_code_changed:
variables_name_data = collections.OrderedDict(
(code,
collections.OrderedDict(sorted(name_by_year.iteritems())))
for code, name_by_year in sorted(
name_by_year_by_code.iteritems()))
with open(variables_name_file_path, 'w') as variables_name_file:
yaml.dump(variables_name_data,
variables_name_file,
allow_unicode=True,
default_flow_style=False,
indent=2,
width=120)
name_by_year_by_code_changed = False
# Create or update YAML file containing OpenFisca test.
main_input_variable_name = json_filename.split('-', 1)[0]
test = collections.OrderedDict((('name', main_input_variable_name), ))
test.update(scenario.to_json())
test[
'period'] = scenario.period.start.year # Replace period string with an integer.
test_case = test.pop('test_case', None)
for entity_name_plural, entity_variables in test_case.iteritems():
test[entity_name_plural] = entity_variables
test['output_variables'] = collections.OrderedDict(
sorted((variable_name, variable_value)
for variable_name, variable_value in (
(openfisca_variable_name_by_tax_calculator_code[code],
value)
for code, value in tax_calculator_outputs.iteritems())
if variable_name is not None))
tests_file_path = os.path.join(
tests_dir, '{}.yaml'.format(main_input_variable_name))
if os.path.exists(tests_file_path):
with open(tests_file_path) as tests_file:
tests = yaml.load(tests_file)
tests.append(test)
tests.sort(key=lambda test: (test['name'], test['period']))
else:
tests = [test]
with open(tests_file_path, 'w') as tests_file:
yaml.dump(tests,
tests_file,
allow_unicode=True,
default_flow_style=False,
indent=2,
width=120)
return 0
def init_from_test_case(self, period, test_case):
conv.check(self.make_json_or_python_to_attributes())(dict(
period=period, test_case=test_case))
return self
def init_from_attributes(self, repair=False, **attributes):
conv.check(
self.make_json_or_python_to_attributes(repair=repair))(attributes)
return self
def __init__(self, dateleg, data):
# TODO: use all attrbutes except data in a PensionParam class
# example:
# duration = data.last_year - data.first_year
# self.param = PensionParam.builder(dateleg, data.info_ind, duration)
# or, by anticipation:
# self.param = PensionParam.builder(dateleg, data.info_ind, duration, method)
# where method give how to shift legislation from on year to an other, constant_year, constant_sequence, etc?
self.date = DateTil(dateleg)
self.P = None
self.P_longit = None
# def load_param(self):
# ''' should run after having a data '''
assert data is not None
path_pension = os.path.dirname(os.path.abspath(__file__))
param_file = os.path.join(path_pension, 'France', 'param.xml')
''' It's a simplification of an (old) openfisca program '''
legislation_tree = ElementTree.parse(param_file)
legislation_xml_json = conv.check(legislationsxml.xml_legislation_to_json)(legislation_tree.getroot(),
state = conv.default_state)
#legislation_xml_json, _ = legislationsxml.validate_node_xml_json(legislation_xml_json,
# state = conv.default_state)
_, legislation_json = legislationsxml.transform_node_xml_json_to_json(legislation_xml_json)
dated_legislation_json = legislations.generate_dated_legislation_json(legislation_json, self.date.datetime)
compact_legislation = legislations.compact_dated_node_json(dated_legislation_json, data.info_ind) #here is where data is needed
self.P = compact_legislation
long_dated_legislation_json = legislations.generate_long_legislation_json(legislation_json, self.date.datetime)
compact_legislation_long = legislations.compact_long_dated_node_json(long_dated_legislation_json)
self.P_longit = compact_legislation_long
#Travail sur Salref
#
# Salref = salaire trimestriel de référence minimum pour le régime général
#
# Note: il existe deux opinions différentes acceptables mais non compatible sur la construction du salref
# - la première c'est de considérer que le salaire de référence de chaque année est donné directement par la
# legisaltion et que même si c'est écrit comme étant égal à d'autres paramètres de la législation, c'est un
# élément indépendant avec son propre code
# - la seconde, consiste à voir l'étape de détermination du salaire de référence comme une étape de la
# législation; le fait d'indexer sur tel ou tel indice n'est pas anodin.
#
# Les deux sont valables, ici, on s'appuie sur la première, on ne fait que recalculer le niveau du salref
# à partir de sa définition. On considère en particulier que l'indexation sur le SMIC a lieu en 1972 et
# on ne peut pas, ailleur qu'ici faire comme si ce changement dépendant de l'année de la base de donnée
# ce ne serait toutefois pas difficile à faire, il suffit de l'inscrire vraiment comme une étape de calcul
#
# En résumé, on ne fait ici qu'écrire un salref qu'on aurait pu relevé dans la législation
# Note :Toute la série chronologique est exprimé en euros
# Article R351-9 du code de la sécurité sociale
# de 1949 à 1972 -> AVTS, après jusqu'en 2014, 200 fois le smic horaire de la première année, ensuite 150 fois.
param_long = self.P_longit
smic = param_long.common.smic
avts = param_long.common.avts.montant
if compare_destinie:
smic = dict((key, val / (52*35)) for key, val in param_long.common.smic_proj.iteritems())
smic_key = sorted(smic.keys())
avts_key = sorted(avts.keys())
debut_annee = '-01-01'
salref = dict()
k = -1
year = 1949 # supposer and year < self.date.year
while year < 1972 and year < self.date.year:
date = str(year) + debut_annee
while avts_key[k+1] <= date:
k +=1
salref[date] = avts[avts_key[k]]/4
if compare_destinie == True:
salref[date] = 1
year += 1
k = -1
while year < 2014 and year < self.date.year:
date = str(year) + debut_annee
while smic_key[k+1] <= date:
k += 1
salref[date] = 200*smic[smic_key[k]]
year += 1
while year < 2014 and year < self.date.year:
date = str(year) + debut_annee
while smic_key[k+1] <= date:
k += 1
salref[date] = 150*smic[smic_key[k]]
year += 1
self.P_longit.prive.RG.salref = salref
def check_legislation_xml_file(year):
legislation_tree = conv.check(
legislationsxml.make_xml_legislation_info_list_to_xml_element(False))(
TaxBenefitSystem.legislation_xml_info_list,
state=conv.default_state)
legislation_xml_json = conv.check(legislationsxml.xml_legislation_to_json)(
legislation_tree,
state=conv.default_state,
)
legislation_xml_json, errors = legislationsxml.validate_legislation_xml_json(
legislation_xml_json, state=conv.default_state)
if errors is not None:
errors = conv.embed_error(legislation_xml_json, 'errors', errors)
if errors is None:
raise ValueError(
unicode(
json.dumps(legislation_xml_json,
ensure_ascii=False,
indent=2)).encode('utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(
json.dumps(errors,
ensure_ascii=False,
indent=2,
sort_keys=True)),
unicode(
json.dumps(legislation_xml_json, ensure_ascii=False,
indent=2)),
).encode('utf-8'))
_, legislation_json = legislationsxml.transform_node_xml_json_to_json(
legislation_xml_json)
legislation_json, errors = legislations.validate_legislation_json(
legislation_json, state=conv.default_state)
if errors is not None:
errors = conv.embed_error(legislation_json, 'errors', errors)
if errors is None:
raise ValueError(
unicode(
json.dumps(legislation_json, ensure_ascii=False,
indent=2)).encode('utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(
json.dumps(errors,
ensure_ascii=False,
indent=2,
sort_keys=True)),
unicode(json.dumps(legislation_json, ensure_ascii=False,
indent=2)),
).encode('utf-8'))
# Create tax_benefit system only now, to be able to debug XML validation errors in above code.
tax_benefit_system = TaxBenefitSystem()
if tax_benefit_system.preprocess_legislation is not None:
legislation_json = tax_benefit_system.preprocess_legislation(
legislation_json)
legislation_json = legislations.generate_dated_legislation_json(
legislation_json, year)
legislation_json, errors = legislations.validate_dated_legislation_json(
legislation_json, state=conv.default_state)
if errors is not None:
errors = conv.embed_error(legislation_json, 'errors', errors)
if errors is None:
raise ValueError(
unicode(
json.dumps(legislation_json, ensure_ascii=False,
indent=2)).encode('utf-8'))
raise ValueError(u'{0} for: {1}'.format(
unicode(
json.dumps(errors,
ensure_ascii=False,
indent=2,
sort_keys=True)),
unicode(json.dumps(legislation_json, ensure_ascii=False,
indent=2)),
).encode('utf-8'))
compact_legislation = legislations.compact_dated_node_json(
legislation_json)
assert compact_legislation is not None
def __init__(self, dateleg, data):
# TODO: use all attrbutes except data in a PensionParam class
# example:
# duration = data.last_year - data.first_year
# self.param = PensionParam.builder(dateleg, data.info_ind, duration)
# or, by anticipation:
# self.param = PensionParam.builder(dateleg, data.info_ind, duration, method)
# where method give how to shift legislation from on year to an other, constant_year, constant_sequence, etc?
self.date = DateTil(dateleg)
self.P = None
self.P_longit = None
# def load_param(self):
# ''' should run after having a data '''
assert data is not None
path_pension = os.path.dirname(os.path.abspath(__file__))
param_file = os.path.join(path_pension, 'France', 'param.xml')
''' It's a simplification of an (old) openfisca program '''
legislation_tree = ElementTree.parse(param_file)
legislation_xml_json = conv.check(
legislationsxml.xml_legislation_to_json)(
legislation_tree.getroot(), state=conv.default_state)
#legislation_xml_json, _ = legislationsxml.validate_node_xml_json(legislation_xml_json,
# state = conv.default_state)
_, legislation_json = legislationsxml.transform_node_xml_json_to_json(
legislation_xml_json)
dated_legislation_json = legislations.generate_dated_legislation_json(
legislation_json, self.date.datetime)
compact_legislation = legislations.compact_dated_node_json(
dated_legislation_json,
data.info_ind) #here is where data is needed
self.P = compact_legislation
long_dated_legislation_json = legislations.generate_long_legislation_json(
legislation_json, self.date.datetime)
compact_legislation_long = legislations.compact_long_dated_node_json(
long_dated_legislation_json)
self.P_longit = compact_legislation_long
#Travail sur Salref
#
# Salref = salaire trimestriel de référence minimum pour le régime général
#
# Note: il existe deux opinions différentes acceptables mais non compatible sur la construction du salref
# - la première c'est de considérer que le salaire de référence de chaque année est donné directement par la
# legisaltion et que même si c'est écrit comme étant égal à d'autres paramètres de la législation, c'est un
# élément indépendant avec son propre code
# - la seconde, consiste à voir l'étape de détermination du salaire de référence comme une étape de la
# législation; le fait d'indexer sur tel ou tel indice n'est pas anodin.
#
# Les deux sont valables, ici, on s'appuie sur la première, on ne fait que recalculer le niveau du salref
# à partir de sa définition. On considère en particulier que l'indexation sur le SMIC a lieu en 1972 et
# on ne peut pas, ailleur qu'ici faire comme si ce changement dépendant de l'année de la base de donnée
# ce ne serait toutefois pas difficile à faire, il suffit de l'inscrire vraiment comme une étape de calcul
#
# En résumé, on ne fait ici qu'écrire un salref qu'on aurait pu relevé dans la législation
# Note :Toute la série chronologique est exprimé en euros
# Article R351-9 du code de la sécurité sociale
# de 1949 à 1972 -> AVTS, après jusqu'en 2014, 200 fois le smic horaire de la première année, ensuite 150 fois.
param_long = self.P_longit
smic = param_long.common.smic
avts = param_long.common.avts.montant
if compare_destinie:
smic = dict(
(key, val / (52 * 35))
for key, val in param_long.common.smic_proj.iteritems())
smic_key = sorted(smic.keys())
avts_key = sorted(avts.keys())
debut_annee = '-01-01'
salref = dict()
k = -1
year = 1949 # supposer and year < self.date.year
while year < 1972 and year < self.date.year:
date = str(year) + debut_annee
while avts_key[k + 1] <= date:
k += 1
salref[date] = avts[avts_key[k]] / 4
if compare_destinie == True:
salref[date] = 1
year += 1
k = -1
while year < 2014 and year < self.date.year:
date = str(year) + debut_annee
while smic_key[k + 1] <= date:
k += 1
salref[date] = 200 * smic[smic_key[k]]
year += 1
while year < 2014 and year < self.date.year:
date = str(year) + debut_annee
while smic_key[k + 1] <= date:
k += 1
salref[date] = 150 * smic[smic_key[k]]
year += 1
self.P_longit.prive.RG.salref = salref
def main():
parser = argparse.ArgumentParser(description = __doc__)
parser.add_argument('-v', '--verbose', action = 'store_true', default = False, help = "increase output verbosity")
args = parser.parse_args()
logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)
if not os.path.exists(income_taxes_test_cases_dir):
os.makedirs(income_taxes_test_cases_dir)
if os.path.exists(tests_dir):
for filename in os.listdir(tests_dir):
if filename.endswith(filename):
os.remove(os.path.join(tests_dir, filename))
else:
os.makedirs(tests_dir)
if os.path.exists(variables_name_file_path):
with open(variables_name_file_path) as variables_name_file:
name_by_year_by_code = yaml.load(variables_name_file)
else:
name_by_year_by_code = {}
name_by_year_by_code_changed = False
for json_filename in sorted(os.listdir(json_dir)):
if not json_filename.endswith('.json'):
continue
log.info(u"Converting file {}...".format(json_filename))
with open(os.path.join(json_dir, json_filename)) as json_file:
data = conv.check(input_to_json_data)(json_file.read())
scenario = data['scenario']
tax_calculator_inputs = transform_scenario_to_tax_calculator_inputs(scenario)
tax_year = scenario.period.start.year + 1
if tax_year <= 2011:
# Tax calculator is no more available for years before 2011.
tax_calculator_outputs = collections.OrderedDict()
tax_calculator_outputs_infos = data['resultat_officiel']
for code, infos in tax_calculator_outputs_infos.iteritems():
float_value = infos['value']
int_value = int(float_value)
tax_calculator_outputs[code] = int_value if float_value == int_value else float_value
name = infos['name'].strip().rstrip(u'*').rstrip()
name = u' '.join(name.split()) # Remove duplicate spaces.
if name not in (u'', u'?', u'nom inconnu'):
name_by_year = name_by_year_by_code.setdefault(code, {})
current_name = name_by_year.get(tax_year)
if current_name is not None and current_name != name \
and not name.lower().endswith(current_name.lower()):
log.warning(u'Ignoring rename of variable {} for year {} from:\n {}\nto:\n {}'.format(code,
tax_year, current_name, name))
elif current_name != name:
name_by_year[tax_year] = name
name_by_year_by_code_changed = True
else:
page = call_tax_calculator(tax_year, tax_calculator_inputs)
page_doc = etree.parse(cStringIO.StringIO(page), html_parser)
codes_without_name = set()
tax_calculator_outputs = collections.OrderedDict()
for element in page_doc.xpath('//input[@type="hidden"][@name]'):
code = element.get('name')
name = None
parent = element.getparent()
parent_tag = parent.tag.lower()
if parent_tag == 'table':
tr = parent[parent.index(element) - 1]
assert tr.tag.lower() == 'tr', tr
elif parent_tag == 'tr':
tr = parent
elif code == 'NAPCR':
name = u'Contributions sociales supplémentaires'
else:
codes_without_name.add(code)
continue
if name is None:
while True:
name = etree.tostring(tr[1], encoding = unicode, method = 'text').strip().rstrip(u'*').rstrip()
if name:
name = u' '.join(name.split()) # Remove duplicate spaces.
break
table = tr.getparent()
tr = table[table.index(tr) - 1]
codes_without_name.discard(code)
float_value = float(element.get('value').strip())
int_value = int(float_value)
tax_calculator_outputs[code] = int_value if float_value == int_value else float_value
name_by_year = name_by_year_by_code.setdefault(code, {})
current_name = name_by_year.get(tax_year)
if current_name is not None and current_name != name \
and not name.lower().endswith(current_name.lower()):
log.warning(u'Renaming variable {} for year {} from:\n {}\nto:\n {}'.format(code, tax_year,
current_name, name))
if current_name != name and (current_name is None or not name.lower().endswith(current_name.lower())):
name_by_year[tax_year] = name
name_by_year_by_code_changed = True
assert not codes_without_name, 'Output variables {} have no name in page:\n{}'.format(
sorted(codes_without_name), page.decode('iso-8859-1').encode('utf-8'))
# Create or update test for "calculateur impôt".
sorted_tax_calculator_inputs = collections.OrderedDict(sorted(tax_calculator_inputs.iteritems()))
income_taxes_test_case_file_path = os.path.join(income_taxes_test_cases_dir, '{}.yaml'.format(
hashlib.md5(json.dumps(sorted_tax_calculator_inputs)).hexdigest()))
if os.path.exists(income_taxes_test_case_file_path):
with open(income_taxes_test_case_file_path) as income_taxes_test_case_file:
income_taxes_test_case = yaml.load(income_taxes_test_case_file)
income_taxes_test_case['output_variables'][str(tax_year)] = collections.OrderedDict(sorted(
tax_calculator_outputs.iteritems()))
income_taxes_test_case['output_variables'] = collections.OrderedDict(sorted(
income_taxes_test_case['output_variables'].iteritems()))
else:
income_taxes_test_case = collections.OrderedDict((
('input_variables', sorted_tax_calculator_inputs),
('output_variables', collections.OrderedDict((
(str(tax_year), collections.OrderedDict(sorted(tax_calculator_outputs.iteritems()))),
))),
))
with open(income_taxes_test_case_file_path, 'w') as income_taxes_test_case_file:
yaml.dump(income_taxes_test_case, income_taxes_test_case_file, allow_unicode = True,
default_flow_style = False, indent = 2, width = 120)
# Create or update YAML file containing the names associated to each result code of "calculateur impôt".
if name_by_year_by_code_changed:
variables_name_data = collections.OrderedDict(
(code, collections.OrderedDict(sorted(name_by_year.iteritems())))
for code, name_by_year in sorted(name_by_year_by_code.iteritems())
)
with open(variables_name_file_path, 'w') as variables_name_file:
yaml.dump(variables_name_data, variables_name_file, allow_unicode = True, default_flow_style = False,
indent = 2, width = 120)
name_by_year_by_code_changed = False
# Create or update YAML file containing OpenFisca test.
main_input_variable_name = json_filename.split('-', 1)[0]
test = collections.OrderedDict((
('name', main_input_variable_name),
))
test.update(scenario.to_json())
test['period'] = scenario.period.start.year # Replace period string with an integer.
test_case = test.pop('test_case', None)
for entity_name_plural, entity_variables in test_case.iteritems():
test[entity_name_plural] = entity_variables
test['output_variables'] = collections.OrderedDict(sorted(
(variable_name, variable_value)
for variable_name, variable_value in (
(openfisca_variable_name_by_tax_calculator_code[code], value)
for code, value in tax_calculator_outputs.iteritems()
)
if variable_name is not None
))
tests_file_path = os.path.join(tests_dir, '{}.yaml'.format(main_input_variable_name))
if os.path.exists(tests_file_path):
with open(tests_file_path) as tests_file:
tests = yaml.load(tests_file)
tests.append(test)
tests.sort(key = lambda test: (test['name'], test['period']))
else:
tests = [test]
with open(tests_file_path, 'w') as tests_file:
yaml.dump(tests, tests_file, allow_unicode = True, default_flow_style = False, indent = 2, width = 120)
return 0
def test_parametric_reform(year):
legislation_tree = xml.etree.ElementTree.parse(TaxBenefitSystem.PARAM_FILE)
legislation_xml_json = conv.check(legislationsxml.xml_legislation_to_json)(
legislation_tree.getroot(),
state = conv.default_state
)
_, legislation_json = legislationsxml.transform_node_xml_json_to_json(legislation_xml_json)
legislation_json_src = legislation_json
# import json
# with open("/tmp/src.json", "w") as f:
# f.write(
# json.dumps(legislation_json_src, ensure_ascii = False, encoding = "utf8", indent = 2).encode("utf8"))
# print str((legislation_json_src['children']['ir']['children']['bareme']['slices'][0]['rate'],))
# print str((legislation_json_src['children']['ir']['children']['bareme']['slices'][1]['rate'],))
legislation_json_reform = copy.deepcopy(legislation_json_src)
legislation_json_reform['children']['ir']['children']['bareme']['slices'][0]['rate'][-1]['value'] = 1
# import json_delta
# json_delta: bug with ordered dicts
#difference = json_delta.diff(p, p_copy)
# from dictdiffer import diff, patch, swap, revert
# difference = diff(p, p_copy)
# print difference (ugly)
legislation_json_patch = jsonpatch.make_patch(legislation_json_src, legislation_json_reform)
print legislation_json_patch
reform = Reform(name = u"Imposition à 100% dès le premier euro et jusqu'à la fin de la 1ère tranche",
legislation_json_patch = legislation_json_patch)
simulation = tax_benefit_system.new_scenario().init_single_entity(
axes = [
dict(
count = 3,
name = 'sali',
max = 100000,
min = 0,
),
],
parent1 = dict(birth = datetime.date(year - 40, 1, 1)),
year = year,
).new_simulation(debug = True)
assert max(abs(simulation.calculate('impo') - [0, -7889.20019531, -23435.52929688])) < .0001
tax_benefit_system.apply_reform(reform = reform)
# with open("/tmp/reform.json", "w") as f:
# f.write(
# json.dumps(
# tax_benefit_system.legislation_json,
# ensure_ascii = False,
# encoding = "utf8",
# indent = 2,
# ).encode("utf8"))
simulation2 = tax_benefit_system.new_scenario().init_single_entity(
axes = [
dict(
count = 3,
name = 'sali',
max = 100000,
min = 0,
),
],
parent1 = dict(birth = datetime.date(year - 40, 1, 1)),
year = year,
).new_simulation(debug = True)
assert max(abs(simulation2.calculate('impo') - [0., -13900.20019531, -29446.52929688])) < .0001
def main():
parser = argparse.ArgumentParser(description=__doc__)
# parser.add_argument('variable', help = u'Name of the variable to calculate. Example: "revdisp"')
parser.add_argument(
'-c',
'--country-package',
default='openfisca_france',
help=
u'Name of the OpenFisca package to use for country-specific variables & formulas'
)
parser.add_argument('-v',
'--verbose',
action='store_true',
default=False,
help="increase output verbosity")
args = parser.parse_args()
logging.basicConfig(
level=logging.DEBUG if args.verbose else logging.WARNING,
stream=sys.stdout)
country_package = importlib.import_module(args.country_package)
TaxBenefitSystem = country_package.init_country()
tax_benefit_system = TaxBenefitSystem()
state = formulas_parsers_ast.State()
state.tax_benefit_system = tax_benefit_system
for column in tax_benefit_system.column_by_name.itervalues():
formula_class = column.formula_class
if formula_class is None:
# Input variable
continue
if issubclass(formula_class, formulas.AbstractEntityToEntity):
# EntityToPerson or PersonToEntity converters
continue
if issubclass(formula_class, formulas.AlternativeFormula):
formulas_class = formula_class.alternative_formulas_class
elif issubclass(formula_class, formulas.DatedFormula):
formulas_class = [
dated_formula_class['formula_class']
for dated_formula_class in formula_class.dated_formulas_class
]
elif issubclass(formula_class, formulas.SelectFormula):
formulas_class = [
select_formula_class for select_formula_class in
formula_class.formula_class_by_main_variable_name.itervalues()
]
else:
assert issubclass(formula_class,
formulas.SimpleFormula), formula_class
formulas_class = [formula_class]
for formula_class in formulas_class:
# source_lines, line_number = inspect.getsourcelines(formula_class)
state.column = column
print column.name
if column.name in (
'cotsoc_noncontrib',
'scelli',
'zone_apl',
):
# TODO
continue
formula_function_definition = conv.check(
state.FormulaFunctionModule.parse)(formula_class.function,
state)
conv.check(formula_function_definition.call_formula)(state)
# print function_visitor.input_variable_by_name
return 0
def main():
parser = argparse.ArgumentParser(description=__doc__)
# parser.add_argument('variable', help = u'Name of the variable to calculate. Example: "revdisp"')
parser.add_argument(
'-c',
'--country-package',
default='openfisca_france',
help=
u'Name of the OpenFisca package to use for country-specific variables & formulas'
)
parser.add_argument('-v',
'--verbose',
action='store_true',
default=False,
help="increase output verbosity")
parser.add_argument('-w',
'--write',
action='store_true',
default=False,
help="replace content of source files")
args = parser.parse_args()
logging.basicConfig(
level=logging.DEBUG if args.verbose else logging.WARNING,
stream=sys.stdout)
country_package = importlib.import_module(args.country_package)
TaxBenefitSystem = country_package.init_country()
tax_benefit_system = TaxBenefitSystem()
state = formulas_parsers_2to3.State(
driver=lib2to3.pgen2.driver.Driver(lib2to3.pygram.python_grammar,
convert=lib2to3.pytree.convert,
logger=log),
tax_benefit_system=tax_benefit_system,
)
source_lines_by_path = {}
for column in tax_benefit_system.column_by_name.itervalues():
column_formula_class = column.formula_class
if column_formula_class is None:
# Input variable
continue
if issubclass(column_formula_class, formulas.AbstractEntityToEntity):
# EntityToPerson or PersonToEntity converters
continue
# if issubclass(column_formula_class, formulas.AlternativeFormula):
# continue # TODO
elif issubclass(column_formula_class, formulas.DatedFormula):
pass
# elif issubclass(column_formula_class, formulas.SelectFormula):
# continue # TODO
else:
assert issubclass(column_formula_class,
formulas.SimpleFormula), column_formula_class
print column.name
state.column = column
formula_class_wrapper = conv.check(state.FormulaClassFileInput.parse)(
column_formula_class, state=state)
if issubclass(column_formula_class, formulas.DatedFormula):
function_functions = []
for name, value in formula_class_wrapper.value_by_name.iteritems():
if isinstance(
value,
state.Decorator) and value.name == u'dated_function':
function_function = value.decorated
assert isinstance(function_function, state.Function)
assert name.startswith(
'function_') or name == 'function', name
function_functions.append(function_function)
else:
assert issubclass(column_formula_class,
formulas.SimpleFormula), column_formula_class
function_functions = [
formula_class_wrapper.value_by_name['function']
]
get_output_period_function = formula_class_wrapper.get_value_by_name(
'get_output_period', default=None, state=state)
if get_output_period_function is None:
print 'Skipping variable without get_output_period method.'
continue
assert get_output_period_function.node.type == symbols.funcdef
suite = get_output_period_function.node.children[4]
assert suite.type == symbols.suite
period_text = unicode(suite).strip()
assert period_text.startswith('return ')
period_text = period_text.replace(u'return ', 'period = ', 1)
# Remove method get_output_period from Python source file.
main_source_lines, main_line_number = inspect.getsourcelines(
column_formula_class)
function_line_number = get_output_period_function.node.get_lineno()
function_lines_count = len(
unicode(get_output_period_function.node).strip().split(u'\n'))
function_first_line_number = main_line_number - 1 + function_line_number
function_after_line_number = function_first_line_number + function_lines_count
module = inspect.getmodule(column_formula_class)
source_file_path = module.__file__
if source_file_path.endswith('.pyc'):
source_file_path = source_file_path[:-1]
source_lines = source_lines_by_path.get(source_file_path)
if source_lines is None:
with codecs.open(source_file_path, "r",
encoding='utf-8') as source_file:
source_text = source_file.read()
source_lines_by_path[
source_file_path] = source_lines = source_text.split(u'\n')
source_lines[function_first_line_number -
1:function_after_line_number -
1] = [None] * function_lines_count
for function_function in function_functions:
assert function_function.node.type == symbols.funcdef
colon_node = function_function.node.children[3]
assert colon_node.type == tokens.COLON
comment = function_function.node.children[3].get_suffix()
if comment:
comment = comment.decode('utf-8').strip().lstrip('#').lstrip()
if comment:
comment = u' # ' + comment
variables_name = [
parameter_name
for parameter_name in function_function.positional_parameters
if parameter_name not in ('_defaultP', '_P', 'period', 'self')
]
variables_line = []
for variable_name in variables_name:
if variable_name.endswith('_holder'):
line = u"{} = simulation.compute('{}', period)".format(
variable_name, variable_name[:-len('_holder')])
else:
line = u"{} = simulation.calculate('{}', period)".format(
variable_name, variable_name)
variables_line.append(line)
variables_text = u' ' + u'\n '.join(
variables_line) if variables_line else u''
law_node_lines = []
if '_defaultP' in function_function.positional_parameters:
law_node_lines.append(
u' _defaultP = simulation.legislation_at(period.start, reference = True)\n'
)
if '_P' in function_function.positional_parameters:
law_node_lines.append(
u' _P = simulation.legislation_at(period.start)\n')
law_node_by_name = {
name: value
for name, value in
function_function.named_parameters.iteritems()
}
law_node_by_name = copy.deepcopy(
function_function.named_parameters)
for parameter_name, law_node in sorted(
law_node_by_name.iteritems()):
law_node_path = unicode(law_node).strip()
assert law_node_path.startswith(u'law')
law_node_path = law_node_path[len(u'law'):]
law_node_lines.append(
u' {} = simulation.legislation_at(period.start){}\n'
.format(parameter_name, law_node_path))
law_nodes_text = u''.join(law_node_lines)
# Replace "return {array}" statements with "return period, {array}".
if not function_function.returns:
print "Missing return statement in {}".format(
function_function.name)
for return_wrapper in function_function.returns:
# print "###{}###{}".format(return_wrapper.node, repr(return_wrapper.node))
return_children = return_wrapper.node.children
assert len(return_children) == 2
return_value = return_children[1]
del return_value.parent
return_children[1] = lib2to3.pytree.Node(
symbols.testlist,
[
lib2to3.pytree.Leaf(tokens.NAME, 'period'),
lib2to3.pytree.Leaf(tokens.COMMA, ','), return_value
],
prefix=' ',
)
# print "###{}###{}".format(return_wrapper.node, repr(return_wrapper.node))
suite = function_function.node.children[4]
assert suite.type == symbols.suite
body_index = None
doc_text = u''
for statement_index, statement in enumerate(suite.children):
if statement.type in (tokens.INDENT, tokens.NEWLINE):
continue
if statement.type in (symbols.funcdef, symbols.if_stmt):
body_index = statement_index
break
assert statement.type == symbols.simple_stmt, type_symbol(
statement.type)
if statement.children:
statement_child = statement.children[0]
if statement_child.type == tokens.STRING:
doc_text = unicode(statement).rstrip() + u'\n '
body_index = statement_index + 1
break
body_index = statement_index
break
assert body_index is not None
body_text = unicode(u''.join(
unicode(statement) for statement in itertools.islice(
suite.children, body_index, None))).strip()
function_text = u"""\
def {name}(self, simulation, period):{comment}
{doc}{period}
{variables}
{law_nodes}
{body}\
""".format(
body=body_text,
doc=doc_text,
comment=comment,
law_nodes=law_nodes_text,
name=function_function.name,
period=period_text,
variables=variables_text,
)
# Replace old method with new method in Python source file.
function_line_number = function_function.node.get_lineno()
function_lines_count = len(
unicode(function_function.node).strip().split(u'\n'))
function_first_line_number = main_line_number - 1 + function_line_number
function_after_line_number = function_first_line_number + function_lines_count
module = inspect.getmodule(column_formula_class)
source_file_path = module.__file__
if source_file_path.endswith('.pyc'):
source_file_path = source_file_path[:-1]
source_lines = source_lines_by_path.get(source_file_path)
if source_lines is None:
with codecs.open(source_file_path, "r",
encoding='utf-8') as source_file:
source_text = source_file.read()
source_lines_by_path[
source_file_path] = source_lines = source_text.split(
u'\n')
source_lines[function_first_line_number - 1:function_after_line_number - 1] = [function_text] \
+ [None] * (function_lines_count - 1)
if args.write:
for source_file_path, source_lines in source_lines_by_path.iteritems():
with codecs.open(source_file_path, "w",
encoding='utf-8') as source_file:
source_file.write(u'\n'.join(line for line in source_lines
if line is not None))
return 0
