def init_single_entity(self, axes = None, enfants = None, famille = None, parent1 = None, parent2 = None, period = None): if enfants is None: enfants = [] assert parent1 is not None famille = famille.copy() if famille is not None else {} individus = [] for index, individu in enumerate([parent1, parent2] + (enfants or [])): if individu is None: continue id = individu.get('id') if id is None: individu = individu.copy() individu['id'] = id = 'ind{}'.format(index) individus.append(individu) if index <= 1: famille.setdefault('parents', []).append(id) else: famille.setdefault('enfants', []).append(id) conv.check(self.make_json_or_python_to_attributes())(dict( axes = axes, period = period, test_case = dict( familles = [famille], individus = individus, ), )) return self
def test(): dir_path = os.path.join(os.path.dirname(__file__), 'formulas') for filename in sorted(os.listdir(dir_path)): if not filename.endswith('.yaml'): continue filename_core = os.path.splitext(filename)[0] with open(os.path.join(dir_path, filename)) as yaml_file: tests = yaml.load(yaml_file) tests, error = conv.pipe( conv.make_item_to_singleton(), conv.uniform_sequence( conv.noop, drop_none_items = True, ), )(tests) if error is not None: embedding_error = conv.embed_error(tests, u'errors', error) assert embedding_error is None, embedding_error conv.check((tests, error)) # Generate an error. for test in tests: test, error = scenarios.make_json_or_python_to_test(tax_benefit_system)(test) if error is not None: embedding_error = conv.embed_error(test, u'errors', error) assert embedding_error is None, embedding_error conv.check((test, error)) # Generate an error. if test.get(u'ignore', False): continue yield check, test.get('name') or filename_core, unicode(test['scenario'].period), test
def init_single_entity( self, autres=None, axes=None, conjoint=None, enfants=None, menage=None, period=None, personne_de_reference=None ): if enfants is None: enfants = [] if autres is None: autres = [] assert personne_de_reference is not None menage = menage.copy() if menage is not None else {} individus = [] for index, individu in enumerate([personne_de_reference, conjoint] + (enfants or []) + (autres or [])): if individu is None: continue id = individu.get("id") if id is None: individu = individu.copy() individu["id"] = id = "ind{}".format(index) individus.append(individu) if index == 0: menage["personne_de_reference"] = id elif index == 1: menage["conjoint"] = id else: menage.setdefault("enfants", []).append(id) conv.check(self.make_json_or_python_to_attributes())( dict(axes=axes, period=period, test_case=dict(menages=[menage], individus=individus)) ) return self
def init_single_entity(self, axes = None, enfants = None, foyer_fiscal = None, menage = None, parent1 = None, parent2 = None, period = None): if enfants is None: enfants = [] assert parent1 is not None foyer_fiscal = foyer_fiscal.copy() if foyer_fiscal is not None else {} individus = [] menage = menage.copy() if menage is not None else {} for index, individu in enumerate([parent1, parent2] + (enfants or [])): if individu is None: continue id = individu.get('id') if id is None: individu = individu.copy() individu['id'] = id = 'ind{}'.format(index) individus.append(individu) if index <= 1: foyer_fiscal.setdefault('declarants', []).append(id) if index == 0: menage['personne_de_reference'] = id else: menage['conjoint'] = id else: foyer_fiscal.setdefault('personnes_a_charge', []).append(id) menage.setdefault('enfants', []).append(id) conv.check(self.make_json_or_python_to_attributes())(dict( axes = axes, period = period, test_case = dict( foyers_fiscaux = [foyer_fiscal], individus = individus, menages = [menage], ), )) return self
def init_single_entity(self, axes=None, enfants=None, famille=None, parent1=None, parent2=None, period=None): if enfants is None: enfants = [] assert parent1 is not None famille = famille.copy() if famille is not None else {} individus = [] for index, individu in enumerate([parent1, parent2] + (enfants or [])): if individu is None: continue id = individu.get('id') if id is None: individu = individu.copy() individu['id'] = id = 'ind{}'.format(index) individus.append(individu) if index <= 1: famille.setdefault('parents', []).append(id) else: famille.setdefault('children', []).append(id) conv.check(self.make_json_or_python_to_attributes())(dict( axes=axes, period=period, test_case=dict( households=[famille], persons=individus, ), )) return self
def check_legislation_xml_file(year): legislation_tree = conv.check(legislationsxml.make_xml_legislation_info_list_to_xml_element(False))( TaxBenefitSystem.legislation_xml_info_list, state = conv.default_state) legislation_xml_json = conv.check(legislationsxml.xml_legislation_to_json)( legislation_tree, state = conv.default_state, ) legislation_xml_json, errors = legislationsxml.validate_legislation_xml_json(legislation_xml_json, state = conv.default_state) if errors is not None: errors = conv.embed_error(legislation_xml_json, 'errors', errors) if errors is None: raise ValueError(unicode(json.dumps(legislation_xml_json, ensure_ascii = False, indent = 2)).encode('utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)), unicode(json.dumps(legislation_xml_json, ensure_ascii = False, indent = 2)), ).encode('utf-8')) _, legislation_json = legislationsxml.transform_node_xml_json_to_json(legislation_xml_json) legislation_json, errors = legislations.validate_legislation_json(legislation_json, state = conv.default_state) if errors is not None: errors = conv.embed_error(legislation_json, 'errors', errors) if errors is None: raise ValueError(unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)).encode('utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)), unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)), ).encode('utf-8')) # Create tax_benefit system only now, to be able to debug XML validation errors in above code. tax_benefit_system = TaxBenefitSystem() if tax_benefit_system.preprocess_legislation is not None: legislation_json = tax_benefit_system.preprocess_legislation(legislation_json) legislation_json = legislations.generate_dated_legislation_json(legislation_json, year) legislation_json, errors = legislations.validate_dated_legislation_json(legislation_json, state = conv.default_state) if errors is not None: errors = conv.embed_error(legislation_json, 'errors', errors) if errors is None: raise ValueError(unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)).encode( 'utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)), unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)), ).encode('utf-8')) compact_legislation = legislations.compact_dated_node_json(legislation_json) assert compact_legislation is not None
def get_decomposition_json(tax_benefit_system, xml_file_path=None): if xml_file_path is None: xml_file_path = tax_benefit_system.decomposition_file_path decomposition_tree = ElementTree.parse(xml_file_path) decomposition_xml_json = conv.check( decompositionsxml.xml_decomposition_to_json)( decomposition_tree.getroot(), state=conv.State) decomposition_xml_json = conv.check( decompositionsxml.make_validate_node_xml_json(tax_benefit_system))( decomposition_xml_json, state=conv.State) decomposition_json = decompositionsxml.transform_node_xml_json_to_json( decomposition_xml_json) return decomposition_json
def extract_formula_function_infos(function, state = None): formula_function = conv.check(state.FormulaFunctionFileInput.parse)(function, state = state) definition_colon_node = formula_function.node.children[3] assert definition_colon_node.type == tokens.COLON comment = formula_function.node.children[3].get_suffix() if comment: comment = comment.decode('utf-8').strip().lstrip('#').lstrip() if comment: comment = u' # ' + comment parameters_text = u', '.join(itertools.chain( [u'self'], ( parameter_name for parameter_name in formula_function.positional_parameters if parameter_name != 'self' ), ( u'{} = {}'.format(parameter_name, parameter_value) for parameter_name, parameter_value in formula_function.named_parameters.iteritems() if parameter_name != 'self' ), )) suite_node = formula_function.node.children[4] assert suite_node.type == symbols.suite body_text = u' ' + '\n '.join(textwrap.dedent(unicode(suite_node).strip()).split('\n')) return dict( body = body_text, comment = comment, parameters = parameters_text, )
def test_legislation_xml_file(): legislation_tree = xml.etree.ElementTree.parse(model.PARAM_FILE) legislation_xml_json = conv.check(legislationsxml.xml_legislation_to_json)(legislation_tree.getroot(), state = conv.default_state) legislation_xml_json, errors = legislationsxml.validate_node_xml_json(legislation_xml_json, state = conv.default_state) if errors is not None: errors = conv.embed_error(legislation_xml_json, 'errors', errors) if errors is None: raise ValueError(unicode(json.dumps(legislation_xml_json, ensure_ascii = False, indent = 2)).encode('utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)), unicode(json.dumps(legislation_xml_json, ensure_ascii = False, indent = 2)), ).encode('utf-8')) _, legislation_json = legislationsxml.transform_node_xml_json_to_json(legislation_xml_json) legislation_json, errors = legislations.validate_node_json(legislation_json, state = conv.default_state) if errors is not None: errors = conv.embed_error(legislation_json, 'errors', errors) if errors is None: raise ValueError(unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)).encode('utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)), unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)), ).encode('utf-8')) dated_legislation_json = legislations.generate_dated_legislation_json(legislation_json, datetime.date(2006, 1, 1)) # raise ValueError(unicode(json.dumps(dated_legislation_json, ensure_ascii = False, indent = 2)).encode('utf-8')) compact_legislation = legislations.compact_dated_node_json(dated_legislation_json)
def check_decomposition_xml_file(file_path): decomposition_tree = xml.etree.ElementTree.parse(os.path.join(file_path)) decomposition_xml_json = conv.check(decompositionsxml.xml_decomposition_to_json)(decomposition_tree.getroot(), state = conv.default_state) decomposition_xml_json, errors = decompositionsxml.make_validate_node_xml_json(base.tax_benefit_system)( decomposition_xml_json, state = conv.default_state) if errors is not None: errors = conv.embed_error(decomposition_xml_json, 'errors', errors) if errors is None: raise ValueError(unicode(json.dumps(decomposition_xml_json, ensure_ascii = False, indent = 2)).encode('utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)), unicode(json.dumps(decomposition_xml_json, ensure_ascii = False, indent = 2)), ).encode('utf-8')) decomposition_json = decompositionsxml.transform_node_xml_json_to_json(decomposition_xml_json) decomposition_json, errors = decompositions.make_validate_node_json(base.tax_benefit_system)( decomposition_json, state = conv.default_state) if errors is not None: errors = conv.embed_error(decomposition_json, 'errors', errors) if errors is None: raise ValueError(unicode(json.dumps(decomposition_json, ensure_ascii = False, indent = 2)).encode('utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)), unicode(json.dumps(decomposition_json, ensure_ascii = False, indent = 2)), ).encode('utf-8'))
def init_single_entity(self, axes=None, enfants=None, famille=None, foyer_fiscal=None, menage=None, parent1=None, parent2=None, period=None): if enfants is None: enfants = [] assert parent1 is not None famille = famille.copy() if famille is not None else {} foyer_fiscal = foyer_fiscal.copy() if foyer_fiscal is not None else {} individus = [] menage = menage.copy() if menage is not None else {} for index, individu in enumerate([parent1, parent2] + (enfants or [])): if individu is None: continue id = individu.get('id') if id is None: individu = individu.copy() individu['id'] = id = 'ind{}'.format(index) individus.append(individu) if index <= 1: famille.setdefault('parents', []).append(id) foyer_fiscal.setdefault('declarants', []).append(id) if index == 0: menage['personne_de_reference'] = id else: menage['conjoint'] = id else: famille.setdefault('enfants', []).append(id) foyer_fiscal.setdefault('personnes_a_charge', []).append(id) menage.setdefault('enfants', []).append(id) conv.check(self.make_json_or_python_to_attributes())(dict( axes=axes, period=period, test_case=dict( familles=[famille], foyers_fiscaux=[foyer_fiscal], individus=individus, menages=[menage], ), )) return self
def test(current_options_by_dir = None, force = False, name_filter = None): if current_options_by_dir is None: current_options_by_dir = options_by_dir for dir, options in sorted(current_options_by_dir.iteritems()): if not force and options.get('ignore', False): log.info(u'Ignoring directory: {}'.format(dir)) continue if not os.path.isdir(dir): log.warning(u'Skipping missing directory: {}'.format(dir)) continue if isinstance(name_filter, str): name_filter = name_filter.decode('utf-8') for filename in sorted(os.listdir(dir)): if not filename.endswith('.yaml'): continue filename_core = os.path.splitext(filename)[0] with open(os.path.join(dir, filename)) as yaml_file: tests = yaml.load(yaml_file) tests, error = conv.pipe( conv.make_item_to_singleton(), conv.uniform_sequence( conv.noop, drop_none_items = True, ), )(tests) if error is not None: embedding_error = conv.embed_error(tests, u'errors', error) assert embedding_error is None, embedding_error conv.check((tests, error)) # Generate an error. for test in tests: test, error = scenarios.make_json_or_python_to_test(get_tax_benefit_system(options.get('reform')), default_absolute_error_margin = options['default_absolute_error_margin'])(test) if error is not None: embedding_error = conv.embed_error(test, u'errors', error) assert embedding_error is None, embedding_error conv.check((test, error)) # Generate an error. if not force and test.get(u'ignore', False): continue if name_filter is not None and name_filter not in filename_core \ and name_filter not in (test.get('name', u'')) \ and name_filter not in (test.get('keywords', [])): continue checker = check_any_period if options['accept_other_period'] else check yield checker, test.get('name') or filename_core, unicode(test['scenario'].period), test, force
def check(test_id, test_name, scenario_data, output_variables): scenario = conv.check(tax_benefit_system.Scenario.make_json_to_instance( tax_benefit_system = tax_benefit_system, ))(scenario_data) scenario.suggest() simulation = scenario.new_simulation(debug = True) if output_variables is not None: for variable_name, expected_value in output_variables.iteritems(): assert_near2(simulation.calculate(variable_name, accept_other_period = True), expected_value, error_margin = 0.007, message = "{}: ".format(variable_name))
def init_single_entity(self, autres=None, axes=None, conjoint=None, enfants=None, menage=None, period=None, personne_de_reference=None): if enfants is None: enfants = [] if autres is None: autres = [] assert personne_de_reference is not None menage = menage.copy() if menage is not None else {} individus = [] for index, individu in enumerate([personne_de_reference, conjoint] + (enfants or []) + (autres or [])): if individu is None: continue id = individu.get('id') if id is None: individu = individu.copy() individu['id'] = id = 'ind{}'.format(index) individus.append(individu) if index == 0: menage['personne_de_reference'] = id elif index == 1: menage['conjoint'] = id else: menage.setdefault('enfants', []).append(id) conv.check(self.make_json_or_python_to_attributes())(dict( axes=axes, period=period, test_case=dict( menages=[menage], individus=individus, ), )) return self
def check_legislation_xml_file(year): legislation_tree = xml.etree.ElementTree.parse(base.TaxBenefitSystem.legislation_xml_file_path) legislation_xml_json = conv.check(legislationsxml.xml_legislation_to_json)(legislation_tree.getroot(), state = conv.default_state) legislation_xml_json, errors = legislationsxml.validate_legislation_xml_json(legislation_xml_json, state = conv.default_state) if errors is not None: errors = conv.embed_error(legislation_xml_json, 'errors', errors) if errors is None: raise ValueError(unicode(json.dumps(legislation_xml_json, ensure_ascii = False, indent = 2)).encode('utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)), unicode(json.dumps(legislation_xml_json, ensure_ascii = False, indent = 2)), ).encode('utf-8')) _, legislation_json = legislationsxml.transform_node_xml_json_to_json(legislation_xml_json) legislation_json, errors = legislations.validate_legislation_json(legislation_json, state = conv.default_state) if errors is not None: errors = conv.embed_error(legislation_json, 'errors', errors) if errors is None: raise ValueError(unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)).encode('utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)), unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)), ).encode('utf-8')) legislation_json = legislations.generate_dated_legislation_json(legislation_json, year) legislation_json, errors = legislations.validate_dated_legislation_json(legislation_json, state = conv.default_state) if errors is not None: errors = conv.embed_error(legislation_json, 'errors', errors) if errors is None: raise ValueError(unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)).encode( 'utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)), unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)), ).encode('utf-8')) compact_legislation = legislations.compact_dated_node_json(legislation_json) # Create tax_benefit system only now, to be able to debug XML validation errors in above code. if base.tax_benefit_system.preprocess_compact_legislation is not None: base.tax_benefit_system.preprocess_compact_legislation(compact_legislation)
def test_legislation_xml_file(): legislation_tree = xml.etree.ElementTree.parse(model.PARAM_FILE) legislation_xml_json = conv.check(legislationsxml.xml_legislation_to_json)(legislation_tree.getroot(), state = conv.default_state) legislation_xml_json, errors = legislationsxml.validate_legislation_xml_json(legislation_xml_json, state = conv.default_state) if errors is not None: errors = conv.embed_error(legislation_xml_json, 'errors', errors) if errors is None: raise ValueError(unicode(json.dumps(legislation_xml_json, ensure_ascii = False, indent = 2)).encode('utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)), unicode(json.dumps(legislation_xml_json, ensure_ascii = False, indent = 2)), ).encode('utf-8')) _, legislation_json = legislationsxml.transform_node_xml_json_to_json(legislation_xml_json) legislation_json, errors = legislations.validate_legislation_json(legislation_json, state = conv.default_state) if errors is not None: errors = conv.embed_error(legislation_json, 'errors', errors) if errors is None: raise ValueError(unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)).encode('utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)), unicode(json.dumps(legislation_json, ensure_ascii = False, indent = 2)), ).encode('utf-8')) dated_legislation_json = legislations.generate_dated_legislation_json(legislation_json, datetime.date(2006, 1, 1)) dated_legislation_json, errors = legislations.validate_dated_legislation_json(dated_legislation_json, state = conv.default_state) if errors is not None: errors = conv.embed_error(dated_legislation_json, 'errors', errors) if errors is None: raise ValueError(unicode(json.dumps(dated_legislation_json, ensure_ascii = False, indent = 2)).encode( 'utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode(json.dumps(errors, ensure_ascii = False, indent = 2, sort_keys = True)), unicode(json.dumps(dated_legislation_json, ensure_ascii = False, indent = 2)), ).encode('utf-8')) compact_legislation = legislations.compact_dated_node_json(dated_legislation_json)
def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('-v', '--verbose', action='store_true', default=False, help="increase output verbosity") args = parser.parse_args() logging.basicConfig( level=logging.DEBUG if args.verbose else logging.WARNING, stream=sys.stdout) if not os.path.exists(income_taxes_test_cases_dir): os.makedirs(income_taxes_test_cases_dir) if os.path.exists(tests_dir): for filename in os.listdir(tests_dir): if filename.endswith(filename): os.remove(os.path.join(tests_dir, filename)) else: os.makedirs(tests_dir) if os.path.exists(variables_name_file_path): with open(variables_name_file_path) as variables_name_file: name_by_year_by_code = yaml.load(variables_name_file) else: name_by_year_by_code = {} name_by_year_by_code_changed = False for json_filename in sorted(os.listdir(json_dir)): if not json_filename.endswith('.json'): continue log.info(u"Converting file {}...".format(json_filename)) with open(os.path.join(json_dir, json_filename)) as json_file: data = conv.check(input_to_json_data)(json_file.read()) scenario = data['scenario'] tax_calculator_inputs = transform_scenario_to_tax_calculator_inputs( scenario) tax_year = scenario.period.start.year + 1 if tax_year <= 2011: # Tax calculator is no more available for years before 2011. tax_calculator_outputs = collections.OrderedDict() tax_calculator_outputs_infos = data['resultat_officiel'] for code, infos in tax_calculator_outputs_infos.iteritems(): float_value = infos['value'] int_value = int(float_value) tax_calculator_outputs[ code] = int_value if float_value == int_value else float_value name = infos['name'].strip().rstrip(u'*').rstrip() name = u' '.join(name.split()) # Remove duplicate spaces. if name not in (u'', u'?', u'nom inconnu'): name_by_year = name_by_year_by_code.setdefault(code, {}) current_name = name_by_year.get(tax_year) if current_name is not None and current_name != name \ and not name.lower().endswith(current_name.lower()): log.warning( u'Ignoring rename of variable {} for year {} from:\n {}\nto:\n {}' .format(code, tax_year, current_name, name)) elif current_name != name: name_by_year[tax_year] = name name_by_year_by_code_changed = True else: page = call_tax_calculator(tax_year, tax_calculator_inputs) page_doc = etree.parse(cStringIO.StringIO(page), html_parser) codes_without_name = set() tax_calculator_outputs = collections.OrderedDict() for element in page_doc.xpath('//input[@type="hidden"][@name]'): code = element.get('name') name = None parent = element.getparent() parent_tag = parent.tag.lower() if parent_tag == 'table': tr = parent[parent.index(element) - 1] assert tr.tag.lower() == 'tr', tr elif parent_tag == 'tr': tr = parent elif code == 'NAPCR': name = u'Contributions sociales supplémentaires' else: codes_without_name.add(code) continue if name is None: while True: name = etree.tostring( tr[1], encoding=unicode, method='text').strip().rstrip(u'*').rstrip() if name: name = u' '.join( name.split()) # Remove duplicate spaces. break table = tr.getparent() tr = table[table.index(tr) - 1] codes_without_name.discard(code) float_value = float(element.get('value').strip()) int_value = int(float_value) tax_calculator_outputs[ code] = int_value if float_value == int_value else float_value name_by_year = name_by_year_by_code.setdefault(code, {}) current_name = name_by_year.get(tax_year) if current_name is not None and current_name != name \ and not name.lower().endswith(current_name.lower()): log.warning( u'Renaming variable {} for year {} from:\n {}\nto:\n {}' .format(code, tax_year, current_name, name)) if current_name != name and ( current_name is None or not name.lower().endswith(current_name.lower())): name_by_year[tax_year] = name name_by_year_by_code_changed = True assert not codes_without_name, 'Output variables {} have no name in page:\n{}'.format( sorted(codes_without_name), page.decode('iso-8859-1').encode('utf-8')) # Create or update test for "calculateur impôt". sorted_tax_calculator_inputs = collections.OrderedDict( sorted(tax_calculator_inputs.iteritems())) income_taxes_test_case_file_path = os.path.join( income_taxes_test_cases_dir, '{}.yaml'.format( hashlib.md5( json.dumps(sorted_tax_calculator_inputs)).hexdigest())) if os.path.exists(income_taxes_test_case_file_path): with open(income_taxes_test_case_file_path ) as income_taxes_test_case_file: income_taxes_test_case = yaml.load(income_taxes_test_case_file) income_taxes_test_case['output_variables'][str( tax_year)] = collections.OrderedDict( sorted(tax_calculator_outputs.iteritems())) income_taxes_test_case[ 'output_variables'] = collections.OrderedDict( sorted(income_taxes_test_case['output_variables']. iteritems())) else: income_taxes_test_case = collections.OrderedDict(( ('input_variables', sorted_tax_calculator_inputs), ('output_variables', collections.OrderedDict( ((str(tax_year), collections.OrderedDict( sorted(tax_calculator_outputs.iteritems()))), ))), )) with open(income_taxes_test_case_file_path, 'w') as income_taxes_test_case_file: yaml.dump(income_taxes_test_case, income_taxes_test_case_file, allow_unicode=True, default_flow_style=False, indent=2, width=120) # Create or update YAML file containing the names associated to each result code of "calculateur impôt". if name_by_year_by_code_changed: variables_name_data = collections.OrderedDict( (code, collections.OrderedDict(sorted(name_by_year.iteritems()))) for code, name_by_year in sorted( name_by_year_by_code.iteritems())) with open(variables_name_file_path, 'w') as variables_name_file: yaml.dump(variables_name_data, variables_name_file, allow_unicode=True, default_flow_style=False, indent=2, width=120) name_by_year_by_code_changed = False # Create or update YAML file containing OpenFisca test. main_input_variable_name = json_filename.split('-', 1)[0] test = collections.OrderedDict((('name', main_input_variable_name), )) test.update(scenario.to_json()) test[ 'period'] = scenario.period.start.year # Replace period string with an integer. test_case = test.pop('test_case', None) for entity_name_plural, entity_variables in test_case.iteritems(): test[entity_name_plural] = entity_variables test['output_variables'] = collections.OrderedDict( sorted((variable_name, variable_value) for variable_name, variable_value in ( (openfisca_variable_name_by_tax_calculator_code[code], value) for code, value in tax_calculator_outputs.iteritems()) if variable_name is not None)) tests_file_path = os.path.join( tests_dir, '{}.yaml'.format(main_input_variable_name)) if os.path.exists(tests_file_path): with open(tests_file_path) as tests_file: tests = yaml.load(tests_file) tests.append(test) tests.sort(key=lambda test: (test['name'], test['period'])) else: tests = [test] with open(tests_file_path, 'w') as tests_file: yaml.dump(tests, tests_file, allow_unicode=True, default_flow_style=False, indent=2, width=120) return 0
def init_from_test_case(self, period, test_case): conv.check(self.make_json_or_python_to_attributes())(dict( period=period, test_case=test_case)) return self
def init_from_attributes(self, repair=False, **attributes): conv.check( self.make_json_or_python_to_attributes(repair=repair))(attributes) return self
def __init__(self, dateleg, data): # TODO: use all attrbutes except data in a PensionParam class # example: # duration = data.last_year - data.first_year # self.param = PensionParam.builder(dateleg, data.info_ind, duration) # or, by anticipation: # self.param = PensionParam.builder(dateleg, data.info_ind, duration, method) # where method give how to shift legislation from on year to an other, constant_year, constant_sequence, etc? self.date = DateTil(dateleg) self.P = None self.P_longit = None # def load_param(self): # ''' should run after having a data ''' assert data is not None path_pension = os.path.dirname(os.path.abspath(__file__)) param_file = os.path.join(path_pension, 'France', 'param.xml') ''' It's a simplification of an (old) openfisca program ''' legislation_tree = ElementTree.parse(param_file) legislation_xml_json = conv.check(legislationsxml.xml_legislation_to_json)(legislation_tree.getroot(), state = conv.default_state) #legislation_xml_json, _ = legislationsxml.validate_node_xml_json(legislation_xml_json, # state = conv.default_state) _, legislation_json = legislationsxml.transform_node_xml_json_to_json(legislation_xml_json) dated_legislation_json = legislations.generate_dated_legislation_json(legislation_json, self.date.datetime) compact_legislation = legislations.compact_dated_node_json(dated_legislation_json, data.info_ind) #here is where data is needed self.P = compact_legislation long_dated_legislation_json = legislations.generate_long_legislation_json(legislation_json, self.date.datetime) compact_legislation_long = legislations.compact_long_dated_node_json(long_dated_legislation_json) self.P_longit = compact_legislation_long #Travail sur Salref # # Salref = salaire trimestriel de référence minimum pour le régime général # # Note: il existe deux opinions différentes acceptables mais non compatible sur la construction du salref # - la première c'est de considérer que le salaire de référence de chaque année est donné directement par la # legisaltion et que même si c'est écrit comme étant égal à d'autres paramètres de la législation, c'est un # élément indépendant avec son propre code # - la seconde, consiste à voir l'étape de détermination du salaire de référence comme une étape de la # législation; le fait d'indexer sur tel ou tel indice n'est pas anodin. # # Les deux sont valables, ici, on s'appuie sur la première, on ne fait que recalculer le niveau du salref # à partir de sa définition. On considère en particulier que l'indexation sur le SMIC a lieu en 1972 et # on ne peut pas, ailleur qu'ici faire comme si ce changement dépendant de l'année de la base de donnée # ce ne serait toutefois pas difficile à faire, il suffit de l'inscrire vraiment comme une étape de calcul # # En résumé, on ne fait ici qu'écrire un salref qu'on aurait pu relevé dans la législation # Note :Toute la série chronologique est exprimé en euros # Article R351-9 du code de la sécurité sociale # de 1949 à 1972 -> AVTS, après jusqu'en 2014, 200 fois le smic horaire de la première année, ensuite 150 fois. param_long = self.P_longit smic = param_long.common.smic avts = param_long.common.avts.montant if compare_destinie: smic = dict((key, val / (52*35)) for key, val in param_long.common.smic_proj.iteritems()) smic_key = sorted(smic.keys()) avts_key = sorted(avts.keys()) debut_annee = '-01-01' salref = dict() k = -1 year = 1949 # supposer and year < self.date.year while year < 1972 and year < self.date.year: date = str(year) + debut_annee while avts_key[k+1] <= date: k +=1 salref[date] = avts[avts_key[k]]/4 if compare_destinie == True: salref[date] = 1 year += 1 k = -1 while year < 2014 and year < self.date.year: date = str(year) + debut_annee while smic_key[k+1] <= date: k += 1 salref[date] = 200*smic[smic_key[k]] year += 1 while year < 2014 and year < self.date.year: date = str(year) + debut_annee while smic_key[k+1] <= date: k += 1 salref[date] = 150*smic[smic_key[k]] year += 1 self.P_longit.prive.RG.salref = salref
def check_legislation_xml_file(year): legislation_tree = conv.check( legislationsxml.make_xml_legislation_info_list_to_xml_element(False))( TaxBenefitSystem.legislation_xml_info_list, state=conv.default_state) legislation_xml_json = conv.check(legislationsxml.xml_legislation_to_json)( legislation_tree, state=conv.default_state, ) legislation_xml_json, errors = legislationsxml.validate_legislation_xml_json( legislation_xml_json, state=conv.default_state) if errors is not None: errors = conv.embed_error(legislation_xml_json, 'errors', errors) if errors is None: raise ValueError( unicode( json.dumps(legislation_xml_json, ensure_ascii=False, indent=2)).encode('utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode( json.dumps(errors, ensure_ascii=False, indent=2, sort_keys=True)), unicode( json.dumps(legislation_xml_json, ensure_ascii=False, indent=2)), ).encode('utf-8')) _, legislation_json = legislationsxml.transform_node_xml_json_to_json( legislation_xml_json) legislation_json, errors = legislations.validate_legislation_json( legislation_json, state=conv.default_state) if errors is not None: errors = conv.embed_error(legislation_json, 'errors', errors) if errors is None: raise ValueError( unicode( json.dumps(legislation_json, ensure_ascii=False, indent=2)).encode('utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode( json.dumps(errors, ensure_ascii=False, indent=2, sort_keys=True)), unicode(json.dumps(legislation_json, ensure_ascii=False, indent=2)), ).encode('utf-8')) # Create tax_benefit system only now, to be able to debug XML validation errors in above code. tax_benefit_system = TaxBenefitSystem() if tax_benefit_system.preprocess_legislation is not None: legislation_json = tax_benefit_system.preprocess_legislation( legislation_json) legislation_json = legislations.generate_dated_legislation_json( legislation_json, year) legislation_json, errors = legislations.validate_dated_legislation_json( legislation_json, state=conv.default_state) if errors is not None: errors = conv.embed_error(legislation_json, 'errors', errors) if errors is None: raise ValueError( unicode( json.dumps(legislation_json, ensure_ascii=False, indent=2)).encode('utf-8')) raise ValueError(u'{0} for: {1}'.format( unicode( json.dumps(errors, ensure_ascii=False, indent=2, sort_keys=True)), unicode(json.dumps(legislation_json, ensure_ascii=False, indent=2)), ).encode('utf-8')) compact_legislation = legislations.compact_dated_node_json( legislation_json) assert compact_legislation is not None
def __init__(self, dateleg, data): # TODO: use all attrbutes except data in a PensionParam class # example: # duration = data.last_year - data.first_year # self.param = PensionParam.builder(dateleg, data.info_ind, duration) # or, by anticipation: # self.param = PensionParam.builder(dateleg, data.info_ind, duration, method) # where method give how to shift legislation from on year to an other, constant_year, constant_sequence, etc? self.date = DateTil(dateleg) self.P = None self.P_longit = None # def load_param(self): # ''' should run after having a data ''' assert data is not None path_pension = os.path.dirname(os.path.abspath(__file__)) param_file = os.path.join(path_pension, 'France', 'param.xml') ''' It's a simplification of an (old) openfisca program ''' legislation_tree = ElementTree.parse(param_file) legislation_xml_json = conv.check( legislationsxml.xml_legislation_to_json)( legislation_tree.getroot(), state=conv.default_state) #legislation_xml_json, _ = legislationsxml.validate_node_xml_json(legislation_xml_json, # state = conv.default_state) _, legislation_json = legislationsxml.transform_node_xml_json_to_json( legislation_xml_json) dated_legislation_json = legislations.generate_dated_legislation_json( legislation_json, self.date.datetime) compact_legislation = legislations.compact_dated_node_json( dated_legislation_json, data.info_ind) #here is where data is needed self.P = compact_legislation long_dated_legislation_json = legislations.generate_long_legislation_json( legislation_json, self.date.datetime) compact_legislation_long = legislations.compact_long_dated_node_json( long_dated_legislation_json) self.P_longit = compact_legislation_long #Travail sur Salref # # Salref = salaire trimestriel de référence minimum pour le régime général # # Note: il existe deux opinions différentes acceptables mais non compatible sur la construction du salref # - la première c'est de considérer que le salaire de référence de chaque année est donné directement par la # legisaltion et que même si c'est écrit comme étant égal à d'autres paramètres de la législation, c'est un # élément indépendant avec son propre code # - la seconde, consiste à voir l'étape de détermination du salaire de référence comme une étape de la # législation; le fait d'indexer sur tel ou tel indice n'est pas anodin. # # Les deux sont valables, ici, on s'appuie sur la première, on ne fait que recalculer le niveau du salref # à partir de sa définition. On considère en particulier que l'indexation sur le SMIC a lieu en 1972 et # on ne peut pas, ailleur qu'ici faire comme si ce changement dépendant de l'année de la base de donnée # ce ne serait toutefois pas difficile à faire, il suffit de l'inscrire vraiment comme une étape de calcul # # En résumé, on ne fait ici qu'écrire un salref qu'on aurait pu relevé dans la législation # Note :Toute la série chronologique est exprimé en euros # Article R351-9 du code de la sécurité sociale # de 1949 à 1972 -> AVTS, après jusqu'en 2014, 200 fois le smic horaire de la première année, ensuite 150 fois. param_long = self.P_longit smic = param_long.common.smic avts = param_long.common.avts.montant if compare_destinie: smic = dict( (key, val / (52 * 35)) for key, val in param_long.common.smic_proj.iteritems()) smic_key = sorted(smic.keys()) avts_key = sorted(avts.keys()) debut_annee = '-01-01' salref = dict() k = -1 year = 1949 # supposer and year < self.date.year while year < 1972 and year < self.date.year: date = str(year) + debut_annee while avts_key[k + 1] <= date: k += 1 salref[date] = avts[avts_key[k]] / 4 if compare_destinie == True: salref[date] = 1 year += 1 k = -1 while year < 2014 and year < self.date.year: date = str(year) + debut_annee while smic_key[k + 1] <= date: k += 1 salref[date] = 200 * smic[smic_key[k]] year += 1 while year < 2014 and year < self.date.year: date = str(year) + debut_annee while smic_key[k + 1] <= date: k += 1 salref[date] = 150 * smic[smic_key[k]] year += 1 self.P_longit.prive.RG.salref = salref
def main(): parser = argparse.ArgumentParser(description = __doc__) parser.add_argument('-v', '--verbose', action = 'store_true', default = False, help = "increase output verbosity") args = parser.parse_args() logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout) if not os.path.exists(income_taxes_test_cases_dir): os.makedirs(income_taxes_test_cases_dir) if os.path.exists(tests_dir): for filename in os.listdir(tests_dir): if filename.endswith(filename): os.remove(os.path.join(tests_dir, filename)) else: os.makedirs(tests_dir) if os.path.exists(variables_name_file_path): with open(variables_name_file_path) as variables_name_file: name_by_year_by_code = yaml.load(variables_name_file) else: name_by_year_by_code = {} name_by_year_by_code_changed = False for json_filename in sorted(os.listdir(json_dir)): if not json_filename.endswith('.json'): continue log.info(u"Converting file {}...".format(json_filename)) with open(os.path.join(json_dir, json_filename)) as json_file: data = conv.check(input_to_json_data)(json_file.read()) scenario = data['scenario'] tax_calculator_inputs = transform_scenario_to_tax_calculator_inputs(scenario) tax_year = scenario.period.start.year + 1 if tax_year <= 2011: # Tax calculator is no more available for years before 2011. tax_calculator_outputs = collections.OrderedDict() tax_calculator_outputs_infos = data['resultat_officiel'] for code, infos in tax_calculator_outputs_infos.iteritems(): float_value = infos['value'] int_value = int(float_value) tax_calculator_outputs[code] = int_value if float_value == int_value else float_value name = infos['name'].strip().rstrip(u'*').rstrip() name = u' '.join(name.split()) # Remove duplicate spaces. if name not in (u'', u'?', u'nom inconnu'): name_by_year = name_by_year_by_code.setdefault(code, {}) current_name = name_by_year.get(tax_year) if current_name is not None and current_name != name \ and not name.lower().endswith(current_name.lower()): log.warning(u'Ignoring rename of variable {} for year {} from:\n {}\nto:\n {}'.format(code, tax_year, current_name, name)) elif current_name != name: name_by_year[tax_year] = name name_by_year_by_code_changed = True else: page = call_tax_calculator(tax_year, tax_calculator_inputs) page_doc = etree.parse(cStringIO.StringIO(page), html_parser) codes_without_name = set() tax_calculator_outputs = collections.OrderedDict() for element in page_doc.xpath('//input[@type="hidden"][@name]'): code = element.get('name') name = None parent = element.getparent() parent_tag = parent.tag.lower() if parent_tag == 'table': tr = parent[parent.index(element) - 1] assert tr.tag.lower() == 'tr', tr elif parent_tag == 'tr': tr = parent elif code == 'NAPCR': name = u'Contributions sociales supplémentaires' else: codes_without_name.add(code) continue if name is None: while True: name = etree.tostring(tr[1], encoding = unicode, method = 'text').strip().rstrip(u'*').rstrip() if name: name = u' '.join(name.split()) # Remove duplicate spaces. break table = tr.getparent() tr = table[table.index(tr) - 1] codes_without_name.discard(code) float_value = float(element.get('value').strip()) int_value = int(float_value) tax_calculator_outputs[code] = int_value if float_value == int_value else float_value name_by_year = name_by_year_by_code.setdefault(code, {}) current_name = name_by_year.get(tax_year) if current_name is not None and current_name != name \ and not name.lower().endswith(current_name.lower()): log.warning(u'Renaming variable {} for year {} from:\n {}\nto:\n {}'.format(code, tax_year, current_name, name)) if current_name != name and (current_name is None or not name.lower().endswith(current_name.lower())): name_by_year[tax_year] = name name_by_year_by_code_changed = True assert not codes_without_name, 'Output variables {} have no name in page:\n{}'.format( sorted(codes_without_name), page.decode('iso-8859-1').encode('utf-8')) # Create or update test for "calculateur impôt". sorted_tax_calculator_inputs = collections.OrderedDict(sorted(tax_calculator_inputs.iteritems())) income_taxes_test_case_file_path = os.path.join(income_taxes_test_cases_dir, '{}.yaml'.format( hashlib.md5(json.dumps(sorted_tax_calculator_inputs)).hexdigest())) if os.path.exists(income_taxes_test_case_file_path): with open(income_taxes_test_case_file_path) as income_taxes_test_case_file: income_taxes_test_case = yaml.load(income_taxes_test_case_file) income_taxes_test_case['output_variables'][str(tax_year)] = collections.OrderedDict(sorted( tax_calculator_outputs.iteritems())) income_taxes_test_case['output_variables'] = collections.OrderedDict(sorted( income_taxes_test_case['output_variables'].iteritems())) else: income_taxes_test_case = collections.OrderedDict(( ('input_variables', sorted_tax_calculator_inputs), ('output_variables', collections.OrderedDict(( (str(tax_year), collections.OrderedDict(sorted(tax_calculator_outputs.iteritems()))), ))), )) with open(income_taxes_test_case_file_path, 'w') as income_taxes_test_case_file: yaml.dump(income_taxes_test_case, income_taxes_test_case_file, allow_unicode = True, default_flow_style = False, indent = 2, width = 120) # Create or update YAML file containing the names associated to each result code of "calculateur impôt". if name_by_year_by_code_changed: variables_name_data = collections.OrderedDict( (code, collections.OrderedDict(sorted(name_by_year.iteritems()))) for code, name_by_year in sorted(name_by_year_by_code.iteritems()) ) with open(variables_name_file_path, 'w') as variables_name_file: yaml.dump(variables_name_data, variables_name_file, allow_unicode = True, default_flow_style = False, indent = 2, width = 120) name_by_year_by_code_changed = False # Create or update YAML file containing OpenFisca test. main_input_variable_name = json_filename.split('-', 1)[0] test = collections.OrderedDict(( ('name', main_input_variable_name), )) test.update(scenario.to_json()) test['period'] = scenario.period.start.year # Replace period string with an integer. test_case = test.pop('test_case', None) for entity_name_plural, entity_variables in test_case.iteritems(): test[entity_name_plural] = entity_variables test['output_variables'] = collections.OrderedDict(sorted( (variable_name, variable_value) for variable_name, variable_value in ( (openfisca_variable_name_by_tax_calculator_code[code], value) for code, value in tax_calculator_outputs.iteritems() ) if variable_name is not None )) tests_file_path = os.path.join(tests_dir, '{}.yaml'.format(main_input_variable_name)) if os.path.exists(tests_file_path): with open(tests_file_path) as tests_file: tests = yaml.load(tests_file) tests.append(test) tests.sort(key = lambda test: (test['name'], test['period'])) else: tests = [test] with open(tests_file_path, 'w') as tests_file: yaml.dump(tests, tests_file, allow_unicode = True, default_flow_style = False, indent = 2, width = 120) return 0
def test_parametric_reform(year): legislation_tree = xml.etree.ElementTree.parse(TaxBenefitSystem.PARAM_FILE) legislation_xml_json = conv.check(legislationsxml.xml_legislation_to_json)( legislation_tree.getroot(), state = conv.default_state ) _, legislation_json = legislationsxml.transform_node_xml_json_to_json(legislation_xml_json) legislation_json_src = legislation_json # import json # with open("/tmp/src.json", "w") as f: # f.write( # json.dumps(legislation_json_src, ensure_ascii = False, encoding = "utf8", indent = 2).encode("utf8")) # print str((legislation_json_src['children']['ir']['children']['bareme']['slices'][0]['rate'],)) # print str((legislation_json_src['children']['ir']['children']['bareme']['slices'][1]['rate'],)) legislation_json_reform = copy.deepcopy(legislation_json_src) legislation_json_reform['children']['ir']['children']['bareme']['slices'][0]['rate'][-1]['value'] = 1 # import json_delta # json_delta: bug with ordered dicts #difference = json_delta.diff(p, p_copy) # from dictdiffer import diff, patch, swap, revert # difference = diff(p, p_copy) # print difference (ugly) legislation_json_patch = jsonpatch.make_patch(legislation_json_src, legislation_json_reform) print legislation_json_patch reform = Reform(name = u"Imposition à 100% dès le premier euro et jusqu'à la fin de la 1ère tranche", legislation_json_patch = legislation_json_patch) simulation = tax_benefit_system.new_scenario().init_single_entity( axes = [ dict( count = 3, name = 'sali', max = 100000, min = 0, ), ], parent1 = dict(birth = datetime.date(year - 40, 1, 1)), year = year, ).new_simulation(debug = True) assert max(abs(simulation.calculate('impo') - [0, -7889.20019531, -23435.52929688])) < .0001 tax_benefit_system.apply_reform(reform = reform) # with open("/tmp/reform.json", "w") as f: # f.write( # json.dumps( # tax_benefit_system.legislation_json, # ensure_ascii = False, # encoding = "utf8", # indent = 2, # ).encode("utf8")) simulation2 = tax_benefit_system.new_scenario().init_single_entity( axes = [ dict( count = 3, name = 'sali', max = 100000, min = 0, ), ], parent1 = dict(birth = datetime.date(year - 40, 1, 1)), year = year, ).new_simulation(debug = True) assert max(abs(simulation2.calculate('impo') - [0., -13900.20019531, -29446.52929688])) < .0001
def main(): parser = argparse.ArgumentParser(description=__doc__) # parser.add_argument('variable', help = u'Name of the variable to calculate. Example: "revdisp"') parser.add_argument( '-c', '--country-package', default='openfisca_france', help= u'Name of the OpenFisca package to use for country-specific variables & formulas' ) parser.add_argument('-v', '--verbose', action='store_true', default=False, help="increase output verbosity") args = parser.parse_args() logging.basicConfig( level=logging.DEBUG if args.verbose else logging.WARNING, stream=sys.stdout) country_package = importlib.import_module(args.country_package) TaxBenefitSystem = country_package.init_country() tax_benefit_system = TaxBenefitSystem() state = formulas_parsers_ast.State() state.tax_benefit_system = tax_benefit_system for column in tax_benefit_system.column_by_name.itervalues(): formula_class = column.formula_class if formula_class is None: # Input variable continue if issubclass(formula_class, formulas.AbstractEntityToEntity): # EntityToPerson or PersonToEntity converters continue if issubclass(formula_class, formulas.AlternativeFormula): formulas_class = formula_class.alternative_formulas_class elif issubclass(formula_class, formulas.DatedFormula): formulas_class = [ dated_formula_class['formula_class'] for dated_formula_class in formula_class.dated_formulas_class ] elif issubclass(formula_class, formulas.SelectFormula): formulas_class = [ select_formula_class for select_formula_class in formula_class.formula_class_by_main_variable_name.itervalues() ] else: assert issubclass(formula_class, formulas.SimpleFormula), formula_class formulas_class = [formula_class] for formula_class in formulas_class: # source_lines, line_number = inspect.getsourcelines(formula_class) state.column = column print column.name if column.name in ( 'cotsoc_noncontrib', 'scelli', 'zone_apl', ): # TODO continue formula_function_definition = conv.check( state.FormulaFunctionModule.parse)(formula_class.function, state) conv.check(formula_function_definition.call_formula)(state) # print function_visitor.input_variable_by_name return 0
def main(): parser = argparse.ArgumentParser(description=__doc__) # parser.add_argument('variable', help = u'Name of the variable to calculate. Example: "revdisp"') parser.add_argument( '-c', '--country-package', default='openfisca_france', help= u'Name of the OpenFisca package to use for country-specific variables & formulas' ) parser.add_argument('-v', '--verbose', action='store_true', default=False, help="increase output verbosity") parser.add_argument('-w', '--write', action='store_true', default=False, help="replace content of source files") args = parser.parse_args() logging.basicConfig( level=logging.DEBUG if args.verbose else logging.WARNING, stream=sys.stdout) country_package = importlib.import_module(args.country_package) TaxBenefitSystem = country_package.init_country() tax_benefit_system = TaxBenefitSystem() state = formulas_parsers_2to3.State( driver=lib2to3.pgen2.driver.Driver(lib2to3.pygram.python_grammar, convert=lib2to3.pytree.convert, logger=log), tax_benefit_system=tax_benefit_system, ) source_lines_by_path = {} for column in tax_benefit_system.column_by_name.itervalues(): column_formula_class = column.formula_class if column_formula_class is None: # Input variable continue if issubclass(column_formula_class, formulas.AbstractEntityToEntity): # EntityToPerson or PersonToEntity converters continue # if issubclass(column_formula_class, formulas.AlternativeFormula): # continue # TODO elif issubclass(column_formula_class, formulas.DatedFormula): pass # elif issubclass(column_formula_class, formulas.SelectFormula): # continue # TODO else: assert issubclass(column_formula_class, formulas.SimpleFormula), column_formula_class print column.name state.column = column formula_class_wrapper = conv.check(state.FormulaClassFileInput.parse)( column_formula_class, state=state) if issubclass(column_formula_class, formulas.DatedFormula): function_functions = [] for name, value in formula_class_wrapper.value_by_name.iteritems(): if isinstance( value, state.Decorator) and value.name == u'dated_function': function_function = value.decorated assert isinstance(function_function, state.Function) assert name.startswith( 'function_') or name == 'function', name function_functions.append(function_function) else: assert issubclass(column_formula_class, formulas.SimpleFormula), column_formula_class function_functions = [ formula_class_wrapper.value_by_name['function'] ] get_output_period_function = formula_class_wrapper.get_value_by_name( 'get_output_period', default=None, state=state) if get_output_period_function is None: print 'Skipping variable without get_output_period method.' continue assert get_output_period_function.node.type == symbols.funcdef suite = get_output_period_function.node.children[4] assert suite.type == symbols.suite period_text = unicode(suite).strip() assert period_text.startswith('return ') period_text = period_text.replace(u'return ', 'period = ', 1) # Remove method get_output_period from Python source file. main_source_lines, main_line_number = inspect.getsourcelines( column_formula_class) function_line_number = get_output_period_function.node.get_lineno() function_lines_count = len( unicode(get_output_period_function.node).strip().split(u'\n')) function_first_line_number = main_line_number - 1 + function_line_number function_after_line_number = function_first_line_number + function_lines_count module = inspect.getmodule(column_formula_class) source_file_path = module.__file__ if source_file_path.endswith('.pyc'): source_file_path = source_file_path[:-1] source_lines = source_lines_by_path.get(source_file_path) if source_lines is None: with codecs.open(source_file_path, "r", encoding='utf-8') as source_file: source_text = source_file.read() source_lines_by_path[ source_file_path] = source_lines = source_text.split(u'\n') source_lines[function_first_line_number - 1:function_after_line_number - 1] = [None] * function_lines_count for function_function in function_functions: assert function_function.node.type == symbols.funcdef colon_node = function_function.node.children[3] assert colon_node.type == tokens.COLON comment = function_function.node.children[3].get_suffix() if comment: comment = comment.decode('utf-8').strip().lstrip('#').lstrip() if comment: comment = u' # ' + comment variables_name = [ parameter_name for parameter_name in function_function.positional_parameters if parameter_name not in ('_defaultP', '_P', 'period', 'self') ] variables_line = [] for variable_name in variables_name: if variable_name.endswith('_holder'): line = u"{} = simulation.compute('{}', period)".format( variable_name, variable_name[:-len('_holder')]) else: line = u"{} = simulation.calculate('{}', period)".format( variable_name, variable_name) variables_line.append(line) variables_text = u' ' + u'\n '.join( variables_line) if variables_line else u'' law_node_lines = [] if '_defaultP' in function_function.positional_parameters: law_node_lines.append( u' _defaultP = simulation.legislation_at(period.start, reference = True)\n' ) if '_P' in function_function.positional_parameters: law_node_lines.append( u' _P = simulation.legislation_at(period.start)\n') law_node_by_name = { name: value for name, value in function_function.named_parameters.iteritems() } law_node_by_name = copy.deepcopy( function_function.named_parameters) for parameter_name, law_node in sorted( law_node_by_name.iteritems()): law_node_path = unicode(law_node).strip() assert law_node_path.startswith(u'law') law_node_path = law_node_path[len(u'law'):] law_node_lines.append( u' {} = simulation.legislation_at(period.start){}\n' .format(parameter_name, law_node_path)) law_nodes_text = u''.join(law_node_lines) # Replace "return {array}" statements with "return period, {array}". if not function_function.returns: print "Missing return statement in {}".format( function_function.name) for return_wrapper in function_function.returns: # print "###{}###{}".format(return_wrapper.node, repr(return_wrapper.node)) return_children = return_wrapper.node.children assert len(return_children) == 2 return_value = return_children[1] del return_value.parent return_children[1] = lib2to3.pytree.Node( symbols.testlist, [ lib2to3.pytree.Leaf(tokens.NAME, 'period'), lib2to3.pytree.Leaf(tokens.COMMA, ','), return_value ], prefix=' ', ) # print "###{}###{}".format(return_wrapper.node, repr(return_wrapper.node)) suite = function_function.node.children[4] assert suite.type == symbols.suite body_index = None doc_text = u'' for statement_index, statement in enumerate(suite.children): if statement.type in (tokens.INDENT, tokens.NEWLINE): continue if statement.type in (symbols.funcdef, symbols.if_stmt): body_index = statement_index break assert statement.type == symbols.simple_stmt, type_symbol( statement.type) if statement.children: statement_child = statement.children[0] if statement_child.type == tokens.STRING: doc_text = unicode(statement).rstrip() + u'\n ' body_index = statement_index + 1 break body_index = statement_index break assert body_index is not None body_text = unicode(u''.join( unicode(statement) for statement in itertools.islice( suite.children, body_index, None))).strip() function_text = u"""\ def {name}(self, simulation, period):{comment} {doc}{period} {variables} {law_nodes} {body}\ """.format( body=body_text, doc=doc_text, comment=comment, law_nodes=law_nodes_text, name=function_function.name, period=period_text, variables=variables_text, ) # Replace old method with new method in Python source file. function_line_number = function_function.node.get_lineno() function_lines_count = len( unicode(function_function.node).strip().split(u'\n')) function_first_line_number = main_line_number - 1 + function_line_number function_after_line_number = function_first_line_number + function_lines_count module = inspect.getmodule(column_formula_class) source_file_path = module.__file__ if source_file_path.endswith('.pyc'): source_file_path = source_file_path[:-1] source_lines = source_lines_by_path.get(source_file_path) if source_lines is None: with codecs.open(source_file_path, "r", encoding='utf-8') as source_file: source_text = source_file.read() source_lines_by_path[ source_file_path] = source_lines = source_text.split( u'\n') source_lines[function_first_line_number - 1:function_after_line_number - 1] = [function_text] \ + [None] * (function_lines_count - 1) if args.write: for source_file_path, source_lines in source_lines_by_path.iteritems(): with codecs.open(source_file_path, "w", encoding='utf-8') as source_file: source_file.write(u'\n'.join(line for line in source_lines if line is not None)) return 0