def main(): parser = argparse.ArgumentParser(description = __doc__) parser.add_argument('-c', '--country-package', default = 'openfisca_france', help = u'name of the OpenFisca package to use for country-specific variables & formulas') parser.add_argument('-n', '--name', default = None, help = u'name of the formula to extract variables from (default: all)') parser.add_argument('-v', '--verbose', action = 'store_true', default = False, help = "increase output verbosity") args = parser.parse_args() logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout) country_package = importlib.import_module(args.country_package) tax_benefit_system = country_package.CountryTaxBenefitSystem() extractor = input_variables_extractors.setup(tax_benefit_system) if args.name is None: for column in tax_benefit_system.column_by_name.itervalues(): print column.name input_variables, parameters = extractor.get_input_variables_and_parameters(column) if input_variables is not None: print u' Input variables:', u', '.join(sorted(input_variables)) if parameters: print u' Parameters:', u', '.join(sorted(parameters)) else: column = tax_benefit_system.column_by_name[args.name] print column.name input_variables, parameters = extractor.get_input_variables_and_parameters(column) if input_variables is not None: print u' Input variables:', u', '.join(sorted(input_variables)) if parameters: print u' Parameters:', u', '.join(sorted(parameters)) return 0
def get_all_parameters(self, column_list): global x print([column.name for column in column_list]) x = x + 1 if x == 20: boum variables = self.variables tax_benefit_system = self.survey_scenario.simulation.tax_benefit_system extractor = input_variables_extractors.setup(tax_benefit_system) if len(column_list) == 0: return [] else: column_name = column_list[0].name print(column_name) if extractor.get_input_variables(variables[column_name]) is None: return column_list else: first_column = [column_list[0]] input_columns = self.get_all_parameters([ variables[clean(parameter)] for parameter in list(extractor.get_input_variables(variables[column_name])) ]) other_columns = list( set(self.get_all_parameters(column_list[1:])) - set(first_column + input_columns) ) print('input_variables: ', [column.name for column in input_columns]) print('new_variables: ', [column.name for column in other_columns]) new_column_list = first_column + input_columns + other_columns print('final list: ', [column.name for column in new_column_list]) return new_column_list
def main(): parser = argparse.ArgumentParser(description = __doc__) parser.add_argument('-c', '--country-package', default = 'openfisca_france', help = u'name of the OpenFisca package to use for country-specific variables & formulas') parser.add_argument('-n', '--name', default = None, help = u'name of the formula to extract variables from (default: all)') parser.add_argument('-v', '--verbose', action = 'store_true', default = False, help = "increase output verbosity") args = parser.parse_args() logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout) country_package = importlib.import_module(args.country_package) TaxBenefitSystem = country_package.init_country() tax_benefit_system = TaxBenefitSystem() extractor = input_variables_extractors.setup(tax_benefit_system) if args.name is None: for column in tax_benefit_system.column_by_name.itervalues(): print column.name input_variables, parameters = extractor.get_input_variables_and_parameters(column) if input_variables is not None: print u' Input variables:', u', '.join(sorted(input_variables)) if parameters: print u' Parameters:', u', '.join(sorted(parameters)) else: column = tax_benefit_system.column_by_name[args.name] print column.name input_variables, parameters = extractor.get_input_variables_and_parameters(column) if input_variables is not None: print u' Input variables:', u', '.join(sorted(input_variables)) if parameters: print u' Parameters:', u', '.join(sorted(parameters)) return 0
def get_variable_names_by_parameter_name(tax_benefit_system): parser = input_variables_extractors.setup(tax_benefit_system) get_input_variables_and_parameters = suppress_stdout( parser.get_input_variables_and_parameters) variable_names_by_parameter_name = collections.defaultdict(list) for column in tax_benefit_system.column_by_name.values(): _, parameter_names = get_input_variables_and_parameters(column) if parameter_names is not None: for parameter_name in parameter_names: variable_names_by_parameter_name[parameter_name].append( column.name) return variable_names_by_parameter_name
def load_environment(global_conf, app_conf): """Configure the application environment.""" conf.update(strings.deep_decode(global_conf)) conf.update(strings.deep_decode(app_conf)) conf.update(conv.check(conv.struct( { 'app_conf': conv.set_value(app_conf), 'app_dir': conv.set_value(app_dir), 'country_package': conv.pipe( conv.make_input_to_slug(separator = u'_'), conv.test_in(( u'openfisca_france', u'openfisca_tunisia', u'openfisca_tunisia_pension', )), conv.not_none, ), 'debug': conv.pipe(conv.guess_bool, conv.default(False)), 'global_conf': conv.set_value(global_conf), 'i18n_dir': conv.default(os.path.join(app_dir, 'i18n')), 'load_alert': conv.pipe(conv.guess_bool, conv.default(False)), 'log_level': conv.pipe( conv.default('WARNING'), conv.function(lambda log_level: getattr(logging, log_level.upper())), ), 'package_name': conv.default('openfisca-web-api'), 'realm': conv.default(u'OpenFisca Web API'), 'reforms': conv.ini_str_to_list, # Another validation is done below. 'extensions': conv.ini_str_to_list, }, default = 'drop', ))(conf)) # Configure logging. logging.basicConfig(level = conf['log_level'], stream = sys.stderr) errorware = conf.setdefault('errorware', {}) errorware['debug'] = conf['debug'] if not errorware['debug']: errorware['error_email'] = conf['email_to'] errorware['error_log'] = conf.get('error_log', None) errorware['error_message'] = conf.get('error_message', 'An internal server error occurred') errorware['error_subject_prefix'] = conf.get('error_subject_prefix', 'OpenFisca Web API Error: ') errorware['from_address'] = conf['from_address'] errorware['smtp_server'] = conf.get('smtp_server', 'localhost') errorware['show_exceptions_in_wsgi_errors'] = conf.get('show_exceptions_in_wsgi_errors', True) # Initialize tax-benefit system. country_package = importlib.import_module(conf['country_package']) tax_benefit_system = country_package.CountryTaxBenefitSystem() extensions = conf['extensions'] if extensions is not None: for extension in extensions: tax_benefit_system.load_extension(extension) class Scenario(tax_benefit_system.Scenario): instance_and_error_couple_cache = {} if conf['debug'] else weakref.WeakValueDictionary() # class attribute @classmethod def make_json_to_cached_or_new_instance(cls, ctx, repair, tax_benefit_system): def json_to_cached_or_new_instance(value, state = None): key = (unicode(ctx.lang), unicode(value), repair, tax_benefit_system) instance_and_error_couple = cls.instance_and_error_couple_cache.get(key) if instance_and_error_couple is None: instance_and_error_couple = cls.make_json_to_instance(repair, tax_benefit_system)( value, state = state or conv.default_state) # Note: Call to ValueAndError() is needed below, otherwise it raises TypeError: cannot create # weak reference to 'tuple' object. cls.instance_and_error_couple_cache[key] = ValueAndError(instance_and_error_couple) return instance_and_error_couple return json_to_cached_or_new_instance tax_benefit_system.Scenario = Scenario model.tax_benefit_system = tax_benefit_system log.debug(u'Pre-fill tax and benefit system cache.') tax_benefit_system.prefill_cache() log.debug(u'Initialize reforms.') reforms = conv.check( conv.uniform_sequence( conv.module_and_function_str_to_function, ) )(conf['reforms']) model.reforms = {} model.reformed_tbs = {} if reforms is not None: for reform in reforms: reformed_tbs = reform(tax_benefit_system) key = reformed_tbs.key full_key = reformed_tbs.full_key model.reforms[key] = reform model.reformed_tbs[full_key] = reformed_tbs log.debug(u'Cache default decomposition.') if tax_benefit_system.decomposition_file_path is not None: # Ignore the returned value, because we just want to pre-compute the cache. model.get_cached_or_new_decomposition_json(tax_benefit_system) log.debug(u'Initialize lib2to3-based input variables extractor.') if input_variables_extractors is not None: model.input_variables_extractor = input_variables_extractors.setup(tax_benefit_system) global country_package_dir_path # Using pkg_resources.get_distribution(conf["country_package"]).location # returns a wrong path in virtualenvs (<venv>/lib versus <venv>/local/lib). country_package_dir_path = country_package.__path__[0] global api_package_version api_package_version = pkg_resources.get_distribution('openfisca_web_api').version global country_package_version country_package_version = pkg_resources.get_distribution(conf["country_package"]).version log.debug(u'Cache legislation JSON with references to original XML.') legislation_json = tax_benefit_system.get_legislation(with_source_file_infos=True) parameters_json = [] walk_legislation_json( legislation_json, descriptions = [], parameters_json = parameters_json, path_fragments = [], ) model.parameters_json_cache = parameters_json if not conf['debug']: # Do this after tax_benefit_system.get_legislation(with_source_file_infos=True). log.debug(u'Compute and cache compact legislation for each first day of month since at least 2 legal years.') today = periods.instant(datetime.date.today()) first_day_of_year = today.offset('first-of', 'year') instant = first_day_of_year.offset(-2, 'year') two_years_later = first_day_of_year.offset(2, 'year') while instant < two_years_later: tax_benefit_system.get_compact_legislation(instant) instant = instant.offset(1, 'month') # Initialize multiprocessing and load_alert if conf['load_alert']: global cpu_count cpu_count = multiprocessing.cpu_count()
def load_environment(global_conf, app_conf): """Configure the application environment.""" conf.update(strings.deep_decode(global_conf)) conf.update(strings.deep_decode(app_conf)) conf.update(conv.check(conv.struct( { 'app_conf': conv.set_value(app_conf), 'app_dir': conv.set_value(app_dir), 'country_package': conv.pipe( conv.make_input_to_slug(separator = u'_'), conv.test_in(( u'openfisca_france', u'openfisca_tunisia', u'openfisca_tunisia_pension', )), conv.not_none, ), 'debug': conv.pipe(conv.guess_bool, conv.default(False)), 'global_conf': conv.set_value(global_conf), 'i18n_dir': conv.default(os.path.join(app_dir, 'i18n')), 'load_alert': conv.pipe(conv.guess_bool, conv.default(False)), 'log_level': conv.pipe( conv.default('WARNING'), conv.function(lambda log_level: getattr(logging, log_level.upper())), ), 'package_name': conv.default('openfisca-web-api'), 'realm': conv.default(u'OpenFisca Web API'), 'reforms': conv.ini_str_to_list, # Another validation is done below. }, default = 'drop', ))(conf)) # Configure logging. logging.basicConfig(level = conf['log_level'], stream = sys.stderr) errorware = conf.setdefault('errorware', {}) errorware['debug'] = conf['debug'] if not errorware['debug']: errorware['error_email'] = conf['email_to'] errorware['error_log'] = conf.get('error_log', None) errorware['error_message'] = conf.get('error_message', 'An internal server error occurred') errorware['error_subject_prefix'] = conf.get('error_subject_prefix', 'OpenFisca Web API Error: ') errorware['from_address'] = conf['from_address'] errorware['smtp_server'] = conf.get('smtp_server', 'localhost') # Initialize tax-benefit system. country_package = importlib.import_module(conf['country_package']) CountryTaxBenefitSystem = country_package.init_country() class Scenario(CountryTaxBenefitSystem.Scenario): instance_and_error_couple_cache = {} if conf['debug'] else weakref.WeakValueDictionary() # class attribute @classmethod def make_json_to_cached_or_new_instance(cls, ctx, repair, tax_benefit_system): def json_to_cached_or_new_instance(value, state = None): key = (unicode(ctx.lang), unicode(value), repair, tax_benefit_system) instance_and_error_couple = cls.instance_and_error_couple_cache.get(key) if instance_and_error_couple is None: instance_and_error_couple = cls.make_json_to_instance(repair, tax_benefit_system)( value, state = state or conv.default_state) # Note: Call to ValueAndError() is needed below, otherwise it raises TypeError: cannot create # weak reference to 'tuple' object. cls.instance_and_error_couple_cache[key] = ValueAndError(instance_and_error_couple) return instance_and_error_couple return json_to_cached_or_new_instance class TaxBenefitSystem(CountryTaxBenefitSystem): pass TaxBenefitSystem.Scenario = Scenario model.TaxBenefitSystem = TaxBenefitSystem model.tax_benefit_system = tax_benefit_system = TaxBenefitSystem() tax_benefit_system.prefill_cache() # Initialize reforms build_reform_functions = conv.check( conv.uniform_sequence( conv.module_and_function_str_to_function, ) )(conf['reforms']) if build_reform_functions is not None: api_reforms = [ build_reform(tax_benefit_system) for build_reform in build_reform_functions ] api_reforms = conv.check( conv.uniform_sequence(conv.test_isinstance(reforms.AbstractReform)) )(api_reforms) model.build_reform_function_by_key = { reform.key: build_reform for build_reform, reform in zip(build_reform_functions, api_reforms) } model.reform_by_full_key = { reform.full_key: reform for reform in api_reforms } # Cache default decomposition. if hasattr(tax_benefit_system, 'DEFAULT_DECOMP_FILE'): model.get_cached_or_new_decomposition_json(tax_benefit_system) # Compute and cache compact legislation for each first day of month since at least 2 legal years. today = periods.instant(datetime.date.today()) first_day_of_year = today.offset('first-of', 'year') instant = first_day_of_year.offset(-2, 'year') two_years_later = first_day_of_year.offset(2, 'year') while instant < two_years_later: tax_benefit_system.get_compact_legislation(instant) instant = instant.offset(1, 'month') # Initialize lib2to3-based input variables extractor. if input_variables_extractors is not None: model.input_variables_extractor = input_variables_extractors.setup(tax_benefit_system) global country_package_dir_path country_package_dir_path = pkg_resources.get_distribution(conf['country_package']).location # Store Git last commit SHA global git_head_sha git_head_sha = get_git_head_sha() global country_package_git_head_sha country_package_git_head_sha = get_git_head_sha(cwd = country_package.__path__[0]) # Cache legislation JSON with references to original XML legislation_json_with_references_to_xml = tax_benefit_system.get_legislation_json(with_source_file_infos = True) parameters_json = [] walk_legislation_json( legislation_json_with_references_to_xml, descriptions = [], parameters_json = parameters_json, path_fragments = [], ) model.parameters_json_cache = parameters_json # Initialize multiprocessing and load_alert if conf['load_alert']: global cpu_count cpu_count = multiprocessing.cpu_count()
def load_environment(global_conf, app_conf): """Configure the application environment.""" conf = openfisca_web_api.conf # Empty dictionary conf.update(strings.deep_decode(global_conf)) conf.update(strings.deep_decode(app_conf)) conf.update(conv.check(conv.struct( { 'app_conf': conv.set_value(app_conf), 'app_dir': conv.set_value(app_dir), 'country_package': conv.pipe( conv.make_input_to_slug(separator = u'_'), conv.test_in(( u'openfisca_france', u'openfisca_tunisia', u'openfisca_tunisia_pension', )), conv.not_none, ), 'debug': conv.pipe(conv.guess_bool, conv.default(False)), 'global_conf': conv.set_value(global_conf), 'i18n_dir': conv.default(os.path.join(app_dir, 'i18n')), 'load_alert': conv.pipe(conv.guess_bool, conv.default(False)), 'log_level': conv.pipe( conv.default('WARNING'), conv.function(lambda log_level: getattr(logging, log_level.upper())), ), 'package_name': conv.default('openfisca-web-api'), 'realm': conv.default(u'OpenFisca Web API'), 'reforms': conv.ini_items_list_to_ordered_dict, # Another validation is done below. }, default = 'drop', ))(conf)) # Configure logging. logging.basicConfig(level = conf['log_level'], stream = sys.stderr) errorware = conf.setdefault('errorware', {}) errorware['debug'] = conf['debug'] if not errorware['debug']: errorware['error_email'] = conf['email_to'] errorware['error_log'] = conf.get('error_log', None) errorware['error_message'] = conf.get('error_message', 'An internal server error occurred') errorware['error_subject_prefix'] = conf.get('error_subject_prefix', 'OpenFisca Web API Error: ') errorware['from_address'] = conf['from_address'] errorware['smtp_server'] = conf.get('smtp_server', 'localhost') # Initialize tax-benefit system. country_package = importlib.import_module(conf['country_package']) CountryTaxBenefitSystem = country_package.init_country() class Scenario(CountryTaxBenefitSystem.Scenario): instance_and_error_couple_by_json_str_cache = weakref.WeakValueDictionary() # class attribute @classmethod def cached_or_new(cls): return conv.check(cls.json_to_cached_or_new_instance)(None) @classmethod def make_json_to_cached_or_new_instance(cls, repair, tax_benefit_system): def json_to_cached_or_new_instance(value, state = None): json_str = json.dumps(value, separators = (',', ':')) if value is not None else None instance_and_error_couple = cls.instance_and_error_couple_by_json_str_cache.get(json_str) if instance_and_error_couple is None: instance_and_error_couple = cls.make_json_to_instance(repair, tax_benefit_system)( value, state = state or conv.default_state) # Note: Call to ValueAndError() is needed below, otherwise it raises TypeError: cannot create # weak reference to 'tuple' object. cls.instance_and_error_couple_by_json_str_cache[json_str] = ValueAndError( instance_and_error_couple) return instance_and_error_couple return json_to_cached_or_new_instance class TaxBenefitSystem(CountryTaxBenefitSystem): pass TaxBenefitSystem.Scenario = Scenario model.TaxBenefitSystem = TaxBenefitSystem model.tax_benefit_system = tax_benefit_system = TaxBenefitSystem() tax_benefit_system.prefill_cache() # Cache default decomposition. model.get_cached_or_new_decomposition_json(tax_benefit_system) # Compute and cache compact legislation for each first day of month since at least 2 legal years. today = periods.instant(datetime.date.today()) first_day_of_year = today.offset('first-of', 'year') instant = first_day_of_year.offset(-2, 'year') two_years_later = first_day_of_year.offset(2, 'year') while instant < two_years_later: tax_benefit_system.get_compact_legislation(instant) instant = instant.offset(1, 'month') # Initialize lib2to3-based input variables extractor. if input_variables_extractors is not None: model.input_variables_extractor = input_variables_extractors.setup(tax_benefit_system) # Store Git last commit SHA global last_commit_sha last_commit_sha = get_git_last_commit_sha() # Load reform modules and store build_reform functions. model.build_reform_function_by_key = build_reform_function_by_key = conv.check( conv.uniform_mapping( conv.noop, conv.module_function_str_to_function, )(conf['reforms']) ) # Check that each reform builds and cache instances. Must not be used with composed reforms. model.reform_by_key = conv.check( conv.uniform_mapping( conv.noop, conv.pipe( conv.function(lambda build_reform: build_reform(tax_benefit_system)), conv.test_isinstance(reforms.AbstractReform), ), )(build_reform_function_by_key) )
def load_environment(global_conf, app_conf): """Configure the application environment.""" conf.update(strings.deep_decode(global_conf)) conf.update(strings.deep_decode(app_conf)) conf.update( conv.check( conv.struct( { 'app_conf': conv.set_value(app_conf), 'app_dir': conv.set_value(app_dir), 'country_package': conv.pipe( conv.make_input_to_slug(separator=u'_'), conv.test_in(( u'openfisca_france', u'openfisca_tunisia', u'openfisca_tunisia_pension', )), conv.not_none, ), 'debug': conv.pipe(conv.guess_bool, conv.default(False)), 'global_conf': conv.set_value(global_conf), 'i18n_dir': conv.default(os.path.join(app_dir, 'i18n')), 'load_alert': conv.pipe(conv.guess_bool, conv.default(False)), 'log_level': conv.pipe( conv.default('WARNING'), conv.function(lambda log_level: getattr( logging, log_level.upper())), ), 'package_name': conv.default('openfisca-web-api'), 'realm': conv.default(u'OpenFisca Web API'), 'reforms': conv.ini_str_to_list, # Another validation is done below. }, default='drop', ))(conf)) # Configure logging. logging.basicConfig(level=conf['log_level'], stream=sys.stderr) errorware = conf.setdefault('errorware', {}) errorware['debug'] = conf['debug'] if not errorware['debug']: errorware['error_email'] = conf['email_to'] errorware['error_log'] = conf.get('error_log', None) errorware['error_message'] = conf.get( 'error_message', 'An internal server error occurred') errorware['error_subject_prefix'] = conf.get( 'error_subject_prefix', 'OpenFisca Web API Error: ') errorware['from_address'] = conf['from_address'] errorware['smtp_server'] = conf.get('smtp_server', 'localhost') # Initialize tax-benefit system. country_package = importlib.import_module(conf['country_package']) CountryTaxBenefitSystem = country_package.init_country() class Scenario(CountryTaxBenefitSystem.Scenario): instance_and_error_couple_cache = {} if conf[ 'debug'] else weakref.WeakValueDictionary() # class attribute @classmethod def make_json_to_cached_or_new_instance(cls, ctx, repair, tax_benefit_system): def json_to_cached_or_new_instance(value, state=None): key = (unicode(ctx.lang), unicode(value), repair, tax_benefit_system) instance_and_error_couple = cls.instance_and_error_couple_cache.get( key) if instance_and_error_couple is None: instance_and_error_couple = cls.make_json_to_instance( repair, tax_benefit_system)(value, state=state or conv.default_state) # Note: Call to ValueAndError() is needed below, otherwise it raises TypeError: cannot create # weak reference to 'tuple' object. cls.instance_and_error_couple_cache[key] = ValueAndError( instance_and_error_couple) return instance_and_error_couple return json_to_cached_or_new_instance class TaxBenefitSystem(CountryTaxBenefitSystem): pass TaxBenefitSystem.Scenario = Scenario model.TaxBenefitSystem = TaxBenefitSystem model.tax_benefit_system = tax_benefit_system = TaxBenefitSystem() tax_benefit_system.prefill_cache() # Initialize reforms build_reform_functions = conv.check( conv.uniform_sequence(conv.module_and_function_str_to_function, ))( conf['reforms']) if build_reform_functions is not None: api_reforms = [ build_reform(tax_benefit_system) for build_reform in build_reform_functions ] api_reforms = conv.check( conv.uniform_sequence(conv.test_isinstance( reforms.AbstractReform)))(api_reforms) model.build_reform_function_by_key = { reform.key: build_reform for build_reform, reform in zip(build_reform_functions, api_reforms) } model.reform_by_full_key = { reform.full_key: reform for reform in api_reforms } # Cache default decomposition. if hasattr(tax_benefit_system, 'DEFAULT_DECOMP_FILE'): model.get_cached_or_new_decomposition_json(tax_benefit_system) # Compute and cache compact legislation for each first day of month since at least 2 legal years. today = periods.instant(datetime.date.today()) first_day_of_year = today.offset('first-of', 'year') instant = first_day_of_year.offset(-2, 'year') two_years_later = first_day_of_year.offset(2, 'year') while instant < two_years_later: tax_benefit_system.get_compact_legislation(instant) instant = instant.offset(1, 'month') # Initialize lib2to3-based input variables extractor. if input_variables_extractors is not None: model.input_variables_extractor = input_variables_extractors.setup( tax_benefit_system) global country_package_dir_path country_package_dir_path = pkg_resources.get_distribution( conf['country_package']).location # Store Git last commit SHA global git_head_sha git_head_sha = get_git_head_sha() global country_package_git_head_sha country_package_git_head_sha = get_git_head_sha( cwd=country_package.__path__[0]) # Cache legislation JSON with references to original XML legislation_json_with_references_to_xml = tax_benefit_system.get_legislation_json( with_source_file_infos=True) parameters_json = [] walk_legislation_json( legislation_json_with_references_to_xml, descriptions=[], parameters_json=parameters_json, path_fragments=[], ) model.parameters_json_cache = parameters_json # Initialize multiprocessing and load_alert if conf['load_alert']: global cpu_count cpu_count = multiprocessing.cpu_count()