def store_input_data_frame(data_frame = None, collection = None, survey = None, table = None): assert data_frame is not None assert collection is not None assert survey is not None try: openfisca_survey_collection = SurveyCollection.load(collection = collection) except Exception as e: openfisca_survey_collection = SurveyCollection(name = collection) log.debug("In collection {} the following survey are present: {}".format(collection, openfisca_survey_collection.surveys)) output_data_directory = openfisca_survey_collection.config.get('data', 'output_directory') if table is None: table = "input" # survey_name = survey hdf5_file_path = os.path.join(os.path.dirname(output_data_directory), "{}.h5".format(survey_name)) available_survey_names = [survey_.name for survey_ in openfisca_survey_collection.surveys] if survey_name in available_survey_names: survey = openfisca_survey_collection.get_survey(survey_name) else: survey = Survey(name = survey_name, hdf5_file_path = hdf5_file_path) survey.insert_table(name = table, data_frame = data_frame) openfisca_survey_collection.surveys.append(survey) collections_directory = openfisca_survey_collection.config.get('collections', 'collections_directory') json_file_path = os.path.join(collections_directory, '{}.json'.format(collection)) log.debug("In collection {} the following surveyx are present: {}".format(collection, openfisca_survey_collection.surveys)) openfisca_survey_collection.dump(json_file_path = json_file_path)
def test_survey(): name = 'fake' data_dir = os.path.join( pkg_resources.get_distribution('openfisca-survey-manager').location, 'openfisca_survey_manager', 'tests', 'data_files', ) survey_collection = SurveyCollection( name = name, config_files_directory = data_dir, json_file_path = os.path.join(data_dir, 'fake.json') ) saved_fake_survey_hdf5_file_path = os.path.join(data_dir, 'fake.hdf5') saved_fake_survey_file_path = os.path.join(data_dir, 'help.sas7bdat') survey = Survey( hdf5_file_path = saved_fake_survey_hdf5_file_path, name = 'fake_survey', sas_files = [saved_fake_survey_file_path], survey_collection = survey_collection, ) survey.insert_table(name = 'help') survey.fill_hdf(source_format = 'sas') print survey.tables
def add_survey_to_collection(survey_name=None, survey_collection=None, sas_files=[], stata_files=[]): assert survey_collection is not None overwrite = True label = survey_name for test_survey in survey_collection.surveys: if test_survey.name == survey_name: survey = survey_collection.get_survey(survey_name) if overwrite: survey = Survey( name=survey_name, label=label, sas_files=sas_files, stata_files=stata_files, survey_collection=survey_collection, ) else: survey = survey_collection.get(survey_name) survey.label = label survey.informations.update({ "sas_files": sas_files, "stata_files": stata_files, }) survey_collection.surveys = [ kept_survey for kept_survey in survey_collection.surveys if kept_survey.name != survey_name ] survey_collection.surveys.append(survey)
def add_survey_to_collection(survey_name = None, survey_collection = None, sas_files = [], stata_files = []): assert survey_collection is not None overwrite = True label = survey_name for test_survey in survey_collection.surveys: if test_survey.name == survey_name: survey = survey_collection.get_survey(survey_name) if overwrite: survey = Survey( name = survey_name, label = label, sas_files = sas_files, stata_files = stata_files, survey_collection = survey_collection, ) else: survey = survey_collection.get(survey_name) survey.label = label survey.informations.update({ "sas_files": sas_files, "stata_files": stata_files, }) survey_collection.surveys = [ kept_survey for kept_survey in survey_collection.surveys if kept_survey.name != survey_name ] survey_collection.surveys.append(survey)
def load(cls, json_file_path = None, collection = None, config_files_directory = default_config_files_directory): assert os.path.exists(config_files_directory) if json_file_path is None: assert collection is not None config = Config(config_files_directory = config_files_directory) try: json_file_path = config.get("collections", collection) except Exception as error: log.debug("Looking for congi file in {}".format(config_files_directory)) log.error(error) raise with open(json_file_path, 'r') as _file: self_json = json.load(_file) name = self_json.get('name') self = cls(name = name) self.config = config with open(json_file_path, 'r') as _file: self_json = json.load(_file) self.json_file_path = json_file_path self.label = self_json.get('label') self.name = self_json.get('name') surveys = self_json.get('surveys') for survey_name, survey_json in surveys.items(): survey = Survey(name = survey_name) self.surveys.append(survey.create_from_json(survey_json)) return self
def run_all(year = None, filename = "test", check = False): assert year is not None pre_processing.create_indivim_menage_en_mois(year = year) pre_processing.create_enfants_a_naitre(year = year) # imputation_loyer.imputation_loyer(year = year) fip.create_fip(year = year) famille.famille(year = year) foyer.sif(year = year) foyer.foyer_all(year = year) rebuild.create_totals(year = year) rebuild.create_final(year = year) invalides.invalide(year = year) data_frame = final.final(year = year, check = check) # Saving the data_frame openfisca_survey_collection = SurveyCollection(name = "openfisca", config_files_directory = config_files_directory) output_data_directory = openfisca_survey_collection.config.get('data', 'output_directory') survey_name = "openfisca_data_{}".format(year) table = "input" hdf5_file_path = os.path.join(os.path.dirname(output_data_directory), "{}.h5".format(survey_name)) survey = Survey( name = survey_name, hdf5_file_path = hdf5_file_path, ) survey.insert_table(name = table, data_frame = data_frame) openfisca_survey_collection.surveys.append(survey) collections_directory = openfisca_survey_collection.config.get('collections', 'collections_directory') json_file_path = os.path.join(collections_directory, 'openfisca.json') openfisca_survey_collection.dump(json_file_path = json_file_path)
def load(cls, json_file_path=None, collection=None, config_files_directory=default_config_files_directory): assert os.path.exists(config_files_directory) if json_file_path is None: assert collection is not None config = Config(config_files_directory=config_files_directory) try: json_file_path = config.get("collections", collection) except Exception as error: log.debug("Looking for congi file in {}".format( config_files_directory)) log.error(error) raise with open(json_file_path, 'r') as _file: self_json = json.load(_file) name = self_json.get('name') self = cls(name=name) self.config = config with open(json_file_path, 'r') as _file: self_json = json.load(_file) self.json_file_path = json_file_path self.label = self_json.get('label') self.name = self_json.get('name') surveys = self_json.get('surveys') for survey_name, survey_json in surveys.items(): survey = Survey(name=survey_name) self.surveys.append(survey.create_from_json(survey_json)) return self
def build_empty_logement_survey_collection(years=None): if years is None: log.error("A list of years to process is needed") logement_survey_collection = SurveyCollection(name="logement") logement_survey_collection.set_config_files_directory( config_files_directory) input_data_directory = logement_survey_collection.config.get( 'data', 'input_directory') output_data_directory = logement_survey_collection.config.get( 'data', 'output_directory') for year in years: surveys = logement_survey_collection.surveys survey_name = 'logement_{}'.format(year) hdf5_file_path = os.path.join(os.path.dirname(output_data_directory), "{}{}".format(survey_name, ".h5")) survey = Survey(name=survey_name, hdf5_file_path=hdf5_file_path) surveys[survey_name] = survey yr = str(year)[2:] yr1 = str(year + 1)[2:] if yr == "03": lgt_men = "menage" lgt_logt = None renameidlgt = dict(ident='ident') elif yr in ["06", "07", "08", "09"]: # TODO: clean this lgt_men = "menage1" lgt_lgt = "logement" renameidlgt = dict(idlog='ident') logement_tables = { "adresse": "adresse", "lgt_menage": lgt_men, "lgt_logt": lgt_lgt, } RData_directory = os.path.join(os.path.dirname(input_data_directory), 'R', 'logement', str(year)) SasData_directory = os.path.join(os.path.dirname(input_data_directory), 'enqlog2006/enq_06') for name, Rdata_table in logement_tables.iteritems(): Rdata_file = os.path.join(RData_directory, "{}.Rdata".format(Rdata_table)) sas_file = os.path.join(SasData_directory, "{}.sas7bdat".format(Rdata_table)) survey.insert_table(name=name, year=year, Rdata_file=Rdata_file, Rdata_table=Rdata_table, sas_file=sas_file) return logement_survey_collection
def build_empty_logement_survey_collection(years= None): if years is None: log.error("A list of years to process is needed") logement_survey_collection = SurveyCollection(name = "logement") logement_survey_collection.set_config_files_directory(config_files_directory) input_data_directory = logement_survey_collection.config.get('data', 'input_directory') output_data_directory = logement_survey_collection.config.get('data', 'output_directory') for year in years: surveys = logement_survey_collection.surveys survey_name = 'logement_{}'.format(year) hdf5_file_path = os.path.join( os.path.dirname(output_data_directory), "{}{}".format(survey_name, ".h5") ) survey = Survey( name = survey_name, hdf5_file_path = hdf5_file_path ) surveys[survey_name] = survey yr = str(year)[2:] yr1 = str(year+1)[2:] if yr == "03": lgt_men = "menage" lgt_logt = None renameidlgt = dict(ident='ident') elif yr in ["06" ,"07", "08", "09"]: # TODO: clean this lgt_men = "menage1" lgt_lgt = "logement" renameidlgt = dict(idlog='ident') logement_tables = { "adresse" : "adresse", "lgt_menage" : lgt_men, "lgt_logt" : lgt_lgt, } RData_directory = os.path.join(os.path.dirname(input_data_directory),'R','logement', str(year)) SasData_directory = os.path.join(os.path.dirname(input_data_directory),'enqlog2006/enq_06') for name, Rdata_table in logement_tables.iteritems(): Rdata_file = os.path.join(RData_directory, "{}.Rdata".format(Rdata_table)) sas_file = os.path.join(SasData_directory, "{}.sas7bdat".format(Rdata_table)) survey.insert_table(name = name, year = year, Rdata_file = Rdata_file, Rdata_table = Rdata_table, sas_file = sas_file ) return logement_survey_collection
def run_all(year_calage = 2007, year_data_list = [1995, 2000, 2005, 2011]): # Quelle base de données choisir pour le calage ? year_data = find_nearest_inferior(year_data_list, year_calage) # 4 étape parallèles d'homogénéisation des données sources : # Gestion des dépenses de consommation: build_depenses_homogenisees(year = year_data) build_imputation_loyers_proprietaires(year = year_data) build_depenses_calees(year_calage, year_data) build_menage_consumption_by_categorie_fiscale(year_calage, year_data) categorie_fiscale_data_frame = temporary_store["menage_consumption_by_categorie_fiscale_{}".format(year_calage)] depenses_calees_by_grosposte = temporary_store["depenses_calees_by_grosposte_{}".format(year_calage)] # Gestion des véhicules: build_homogeneisation_vehicules(year = year_data) vehicule = temporary_store['automobile_{}'.format(year_data)] # Gestion des variables socio démographiques: build_homogeneisation_caracteristiques_sociales(year = year_data) menage = temporary_store['donnes_socio_demog_{}'.format(year_data)] # Gestion des variables revenues: build_homogeneisation_revenus_menages(year = year_data) revenus = temporary_store["revenus_{}".format(year_data)] # DataFrame résultant de ces 4 étapes data_frame = pandas.concat( [revenus, vehicule, categorie_fiscale_data_frame, menage, depenses_calees_by_grosposte], axis = 1) data_frame.index.name = "ident_men" data_frame.reset_index(inplace = True) # Remove duplicated colums causing bug with HDFStore # according to https://github.com/pydata/pandas/issues/6240 # using solution form stackoverflow # http://stackoverflow.com/questions/16938441/how-to-remove-duplicate-columns-from-a-dataframe-using-python-pandas data_frame = data_frame.T.groupby(level = 0).first().T # Saving the data_frame openfisca_survey_collection = SurveyCollection.load( collection = 'openfisca_indirect_taxation', config_files_directory = config_files_directory) output_data_directory = openfisca_survey_collection.config.get('data', 'output_directory') survey_name = "openfisca_indirect_taxation_data_{}".format(year_calage) table = "input" hdf5_file_path = os.path.join(os.path.dirname(output_data_directory), "{}.h5".format(survey_name)) survey = Survey( name = survey_name, hdf5_file_path = hdf5_file_path, ) survey.insert_table(name = table, data_frame = data_frame) openfisca_survey_collection.surveys.append(survey) openfisca_survey_collection.dump()
def set_table_in_survey(input_dataframe, entity, period, collection, survey_name, survey_label = None, table_label = None, table_name = None): period = periods.period(period) if table_name is None: table_name = entity + '_' + str(period) if table_label is None: table_label = "Input data for entity {} at period {}".format(entity, period) try: survey_collection = SurveyCollection.load(collection = collection) except configparser.NoOptionError: survey_collection = SurveyCollection(name = collection) except configparser.NoSectionError: # For tests data_dir = os.path.join( pkg_resources.get_distribution('openfisca-survey-manager').location, 'openfisca_survey_manager', 'tests', 'data_files', ) survey_collection = SurveyCollection( name = collection, config_files_directory = data_dir, ) try: survey = survey_collection.get_survey(survey_name) except AssertionError: survey = Survey( name = survey_name, label = survey_label or None, survey_collection = survey_collection, ) if survey.hdf5_file_path is None: config = survey.survey_collection.config directory_path = config.get("data", "output_directory") if not os.path.isdir(directory_path): log.warn("{} who should be the HDF5 data directory does not exist: we create the directory".format( directory_path)) os.makedirs(directory_path) survey.hdf5_file_path = os.path.join(directory_path, survey.name + '.h5') assert survey.hdf5_file_path is not None survey.insert_table(label = table_label, name = table_name, dataframe = input_dataframe) survey_collection.surveys = [ kept_survey for kept_survey in survey_collection.surveys if kept_survey.name != survey_name ] survey_collection.surveys.append(survey) collections_directory = survey_collection.config.get('collections', 'collections_directory') assert os.path.isdir(collections_directory), """{} who should be the collections' directory does not exist. Fix the option collections_directory in the collections section of your config file.""".format(collections_directory) collection_json_path = os.path.join(collections_directory, "{}.json".format(collection)) survey_collection.dump(json_file_path = collection_json_path)
def set_table_in_survey(input_dataframe, entity, period, collection, survey_name, survey_label = None, table_label = None, table_name = None): period = periods.period(period) if table_name is None: table_name = entity + '_' + str(period) if table_label is None: table_label = "Input data for entity {} at period {}".format(entity, period) try: survey_collection = SurveyCollection.load(collection = collection) except configparser.NoOptionError: survey_collection = SurveyCollection(name = collection) except configparser.NoSectionError: # For tests data_dir = os.path.join( pkg_resources.get_distribution('openfisca-survey-manager').location, 'openfisca_survey_manager', 'tests', 'data_files', ) survey_collection = SurveyCollection( name = collection, config_files_directory = data_dir, ) try: survey = survey_collection.get_survey(survey_name) except AssertionError: survey = Survey( name = survey_name, label = survey_label or None, survey_collection = survey_collection, ) if survey.hdf5_file_path is None: config = survey.survey_collection.config directory_path = config.get("data", "output_directory") if not os.path.isdir(directory_path): log.warn("{} who should be the HDF5 data directory does not exist: we create the directory".format( directory_path)) os.makedirs(directory_path) survey.hdf5_file_path = os.path.join(directory_path, survey.name + '.h5') assert survey.hdf5_file_path is not None survey.insert_table(label = table_label, name = table_name, dataframe = input_dataframe) survey_collection.surveys = [ kept_survey for kept_survey in survey_collection.surveys if kept_survey.name != survey_name ] survey_collection.surveys.append(survey) collections_directory = survey_collection.config.get('collections', 'collections_directory') assert os.path.isdir(collections_directory), """{} who should be the collections' directory does not exist. Fix the option collections_directory in the collections section of your config file.""".format(collections_directory) collection_json_path = os.path.join(collections_directory, "{}.json".format(collection)) survey_collection.dump(json_file_path = collection_json_path)
def test_survey_dump_load(): erfs_survey_collection = build_empty_erfs_survey_collection(years=[2006]) erfs_survey = erfs_survey_collection.surveys['erfs_2006'] saved_fake_survey_file_path = os.path.join(current_dir, 'saved_fake_survey') erfs_survey.dump(saved_fake_survey_file_path) erfs_survey_bis = Survey.load(saved_fake_survey_file_path) assert erfs_survey.to_json() == erfs_survey_bis.to_json()
def test_survey_dump_load(): erfs_survey_collection = build_empty_erfs_survey_collection(years=[2006]) erfs_survey = erfs_survey_collection.surveys['erfs_2006'] saved_fake_survey_file_path = os.path.join(current_dir, 'saved_fake_survey') erfs_survey.dump(saved_fake_survey_file_path) erfs_survey_bis = Survey.load(saved_fake_survey_file_path) assert erfs_survey.to_json() == erfs_survey_bis.to_json()
def add_survey_to_collection(survey_name = None, survey_collection = None, sas_files = [], stata_files = [], question = False): assert survey_collection is not None overwrite = True if question: label = click.prompt('Enter a description for the survey {}'.format(survey_name), default = survey_name) else: label = survey_name for test_survey in survey_collection.surveys: if test_survey.name == survey_name: if question: click.echo('The following information is available for survey {}'.format(survey_name)) survey = survey_collection.get_survey(survey_name) if question: click.echo(survey) overwrite = click.confirm( 'Overwrite previous survey {} informations ?'.format(survey_name), default = True) else: overwrite = True if question: same_survey = click.confirm('Are all the files part of the same survey ?', default = True) else: same_survey = True if same_survey: if overwrite: survey = Survey( name = survey_name, label = label, sas_files = sas_files, stata_files = stata_files, survey_collection = survey_collection, ) else: survey = survey_collection.get(survey_name) survey.label = label survey.informations.update({ "sas_files": sas_files, "stata_files": stata_files, }) survey_collection.surveys = [ kept_survey for kept_survey in survey_collection.surveys if kept_survey.name != survey_name ] survey_collection.surveys.append(survey)
def store_input_data_frame(data_frame = None, collection = None, survey = None): assert data_frame is not None assert collection is not None assert survey is not None openfisca_survey_collection = SurveyCollection(name = collection, config_files_directory = config_files_directory) output_data_directory = openfisca_survey_collection.config.get('data', 'output_directory') survey_name = survey table = "input" hdf5_file_path = os.path.join(os.path.dirname(output_data_directory), "{}.h5".format(survey_name)) survey = Survey( name = survey_name, hdf5_file_path = hdf5_file_path, ) survey.insert_table(name = table, data_frame = data_frame) openfisca_survey_collection.surveys.append(survey) collections_directory = openfisca_survey_collection.config.get('collections', 'collections_directory') json_file_path = os.path.join(collections_directory, 'openfisca_erfs_fpr.json') openfisca_survey_collection.dump(json_file_path = json_file_path)
def build_empty_recensement_survey_collection(years= None): if years is None: log.error("A list of years to process is needed") recensement_survey_collection = SurveyCollection(name = "recensement") recensement_survey_collection.set_config_files_directory(config_files_directory) input_data_directory = recensement_survey_collection.config.get('data', 'input_directory') output_data_directory = recensement_survey_collection.config.get('data', 'output_directory') for year in years: tables = [ # "LOGMENT_ECH_{}".format(year), # TODO: produces a strange error "individu_ech_{}".format(year), "MENAG_ECH_{}".format(year), ] survey_tables = dict() for year in years: for table in tables: survey_tables[table] = { "spss_file": os.path.join( os.path.dirname(input_data_directory), "recensement", str(year), "{}.sav".format(table), ), "year": year, } survey_name = u"recensement_{}".format(year) hdf5_file_path = os.path.join( os.path.dirname(output_data_directory), u"{}{}".format(survey_name, u".h5") ) print hdf5_file_path survey = Survey( name = survey_name, hdf5_file_path = hdf5_file_path ) for table, table_kwargs in survey_tables.iteritems(): survey.insert_table(name = table, **table_kwargs) surveys = recensement_survey_collection.surveys surveys[survey_name] = survey return recensement_survey_collection
def build_empty_budget_consommation_survey_collection(years= None): if years is None: log.error("A list of years to process is needed") budget_consommation_survey_collection = SurveyCollection(name = "budget_consommation") budget_consommation_survey_collection.set_config_files_directory(config_files_directory) input_data_directory = budget_consommation_survey_collection.config.get('data', 'input_directory') output_data_directory = budget_consommation_survey_collection.config.get('data', 'output_directory') for year in years: tables = ["budg0{}".format(i) for i in range(1, 10)] + ["budg{}".format(i) for i in range(10, 21)] tables.remove("budg03") survey_tables = dict() for year in years: for table in tables: survey_tables[table] = { "stata_file": os.path.join( os.path.dirname(input_data_directory), "budget_consommation", str(year), "stata", "{}.dta".format(table), ), "year": year, } survey_name = u"budget_consommation_{}".format(year) hdf5_file_path = os.path.join( os.path.dirname(output_data_directory), u"{}{}".format(survey_name, u".h5") ) print hdf5_file_path survey = Survey( name = survey_name, hdf5_file_path = hdf5_file_path ) for table, table_kwargs in survey_tables.iteritems(): survey.insert_table(name = table, **table_kwargs) surveys = budget_consommation_survey_collection.surveys surveys[survey_name] = survey return budget_consommation_survey_collection
def build(year=None, check=False): assert year is not None pre_processing.create_indivim_menagem(year=year) pre_processing.create_enfants_a_naitre(year=year) # try: # imputation_loyer.imputation_loyer(year = year) # except Exception, e: # log.info('Do not impute loyer because of the following error: \n {}'.format(e)) # pass fip.create_fip(year=year) famille.famille(year=year) foyer.sif(year=year) foyer.foyer_all(year=year) rebuild.create_totals_first_pass(year=year) rebuild.create_totals_second_pass(year=year) rebuild.create_final(year=year) invalides.invalide(year=year) final.final(year=year, check=check) temporary_store = get_store(file_name='erfs') data_frame = temporary_store['input_{}'.format(year)] # Saving the data_frame openfisca_survey_collection = SurveyCollection( name="openfisca", config_files_directory=config_files_directory) output_data_directory = openfisca_survey_collection.config.get( 'data', 'output_directory') survey_name = "openfisca_data_{}".format(year) table = "input" hdf5_file_path = os.path.join(os.path.dirname(output_data_directory), "{}.h5".format(survey_name)) survey = Survey( name=survey_name, hdf5_file_path=hdf5_file_path, ) survey.insert_table(name=table, data_frame=data_frame) openfisca_survey_collection.surveys.append(survey) collections_directory = openfisca_survey_collection.config.get( 'collections', 'collections_directory') json_file_path = os.path.join(collections_directory, 'openfisca.json') openfisca_survey_collection.dump(json_file_path=json_file_path)
def build_empty_piketty_survey_collection(years=None): if years is None: log.error("A list of years to process is needed") survey_collection = SurveyCollection(name="piketty") survey_collection.set_config_files_directory(config_files_directory) input_data_directory = survey_collection.config.get( 'data', 'input_directory') output_data_directory = survey_collection.config.get( 'data', 'output_directory') tables = [ "indiv_conj", "indiv_ded", "indiv_demo", "indiv_logt", "indiv_rev" ] piketty_tables = dict() for year in [2006]: for table in tables: piketty_tables[table] = { "stata_file": os.path.join( os.path.dirname(input_data_directory), "revolution_fiscale", "Fichiers", "original", "{}_{}.dta".format(table, year), ), "year": year, } survey_name = u"piketty_{}".format(year) hdf5_file_path = os.path.join(os.path.dirname(output_data_directory), u"{}{}".format(survey_name, u".h5")) survey = Survey(name=survey_name, hdf5_file_path=hdf5_file_path) for table, table_kwargs in piketty_tables.iteritems(): survey.insert_table(name=table, **table_kwargs) surveys = survey_collection.surveys surveys[survey_name] = survey return survey_collection
def dump_data_frame(data_frame, year): from openfisca_france_data.build_openfisca_survey_data import utils utils.print_id(data_frame) # utils.control(data_frame, verbose = True) utils.check_structure(data_frame) survey_collection = SurveyCollection(name = "eipp") survey_collection.set_config_files_directory() output_data_directory = survey_collection.config.get('data', 'output_directory') survey_name = "eipp_data_{}".format(year) table = "input" hdf5_file_path = os.path.join( os.path.dirname(output_data_directory), "{}{}".format(survey_name, ".h5"), ) survey = Survey( name = survey_name, hdf5_file_path = hdf5_file_path, ) survey.insert_table(name = table) survey.fill_hdf(table, data_frame) survey_collection.surveys[survey_name] = survey survey_collection.dump(collection = "eipp")
def test_survey(): name = 'fake' data_dir = os.path.join( pkg_resources.get_distribution('openfisca-survey-manager').location, 'openfisca_survey_manager', 'tests', 'data_files', ) survey_collection = SurveyCollection(name=name, config_files_directory=data_dir, json_file_path=os.path.join( data_dir, 'fake.json')) saved_fake_survey_hdf5_file_path = os.path.join(data_dir, 'fake.hdf5') saved_fake_survey_file_path = os.path.join(data_dir, 'help.sas7bdat') survey = Survey( hdf5_file_path=saved_fake_survey_hdf5_file_path, name='fake_survey', sas_files=[saved_fake_survey_file_path], survey_collection=survey_collection, ) survey.insert_table(name='help') survey.fill_hdf(source_format='sas') print survey.tables
def build_empty_piketty_survey_collection(years= None): if years is None: log.error("A list of years to process is needed") survey_collection = SurveyCollection(name = "piketty") survey_collection.set_config_files_directory(config_files_directory) input_data_directory = survey_collection.config.get('data', 'input_directory') output_data_directory = survey_collection.config.get('data', 'output_directory') tables = ["indiv_conj", "indiv_ded", "indiv_demo", "indiv_logt", "indiv_rev"] piketty_tables = dict() for year in [2006]: for table in tables: piketty_tables[table] = { "stata_file": os.path.join( os.path.dirname(input_data_directory), "revolution_fiscale", "Fichiers", "original", "{}_{}.dta".format(table, year), ), "year": year, } survey_name = u"piketty_{}".format(year) hdf5_file_path = os.path.join( os.path.dirname(output_data_directory), u"{}{}".format(survey_name, u".h5") ) survey = Survey( name = survey_name, hdf5_file_path = hdf5_file_path ) for table, table_kwargs in piketty_tables.iteritems(): survey.insert_table(name = table, **table_kwargs) surveys = survey_collection.surveys surveys[survey_name] = survey return survey_collection
def dump_simulation_results_data_frame(survey_scenario, collection = None): assert collection is not None year = survey_scenario.year data_frame_by_entity = get_calculated_data_frame_by_entity(survey_scenario) openfisca_survey_collection = SurveyCollection.load(collection = "openfisca") output_data_directory = openfisca_survey_collection.config.get('data', 'output_directory') survey_name = "openfisca_data_{}".format(year) for entity, data_frame in data_frame_by_entity.iteritems(): print entity table = entity hdf5_file_path = os.path.join( os.path.dirname(output_data_directory), "{}{}".format(survey_name, ".h5"), ) survey = Survey( name = survey_name, hdf5_file_path = hdf5_file_path, ) survey.insert_table(name = table) survey.fill_hdf(table, data_frame) openfisca_survey_collection.surveys[survey_name] = survey openfisca_survey_collection.dump(collection = "openfisca")
def store_input_data_frame(data_frame=None, collection=None, survey=None, table=None): assert data_frame is not None assert collection is not None assert survey is not None try: openfisca_survey_collection = SurveyCollection.load( collection=collection) except Exception as e: openfisca_survey_collection = SurveyCollection(name=collection) log.debug("In collection {} the following survey are present: {}".format( collection, openfisca_survey_collection.surveys)) output_data_directory = openfisca_survey_collection.config.get( 'data', 'output_directory') if table is None: table = "input" # survey_name = survey hdf5_file_path = os.path.join(os.path.dirname(output_data_directory), "{}.h5".format(survey_name)) available_survey_names = [ survey_.name for survey_ in openfisca_survey_collection.surveys ] if survey_name in available_survey_names: survey = openfisca_survey_collection.get_survey(survey_name) else: survey = Survey(name=survey_name, hdf5_file_path=hdf5_file_path) survey.insert_table(name=table, data_frame=data_frame) openfisca_survey_collection.surveys.append(survey) collections_directory = openfisca_survey_collection.config.get( 'collections', 'collections_directory') json_file_path = os.path.join(collections_directory, '{}.json'.format(collection)) log.debug("In collection {} the following surveyx are present: {}".format( collection, openfisca_survey_collection.surveys)) openfisca_survey_collection.dump(json_file_path=json_file_path)
def build_empty_eipp_survey_collection(years= None): if years is None: log.error("A list of years to process is needed") base_eipp_survey_collection = SurveyCollection(name = "eipp") base_eipp_survey_collection.set_config_files_directory(config_files_directory) input_data_directory = base_eipp_survey_collection.config.get('data', 'input_directory') output_data_directory = base_eipp_survey_collection.config.get('data', 'output_directory') tables = ["base"] eipp_tables = dict() for year in years: for table in tables: eipp_tables[table] = { "stata_file": os.path.join( os.path.dirname(input_data_directory), "fichiers_eipp", "{}_{}.dta".format(table, year), ), "year": year, } survey_name = u"eipp_{}".format(year) hdf5_file_path = os.path.join( os.path.dirname(output_data_directory), u"{}{}".format(survey_name, u".h5") ) survey = Survey( name = survey_name, hdf5_file_path = hdf5_file_path ) for table, table_kwargs in eipp_tables.iteritems(): survey.insert_table(name = table, **table_kwargs) surveys = base_eipp_survey_collection.surveys surveys[survey_name] = survey return base_eipp_survey_collection
def run_all(year = None, check = False): assert year is not None pre_processing.create_indivim_menagem(year = year) pre_processing.create_enfants_a_naitre(year = year) # try: # imputation_loyer.imputation_loyer(year = year) # except Exception, e: # log.info('Do not impute loyer because of the following error: \n {}'.format(e)) # pass fip.create_fip(year = year) famille.famille(year = year) foyer.sif(year = year) foyer.foyer_all(year = year) rebuild.create_totals_first_pass(year = year) rebuild.create_totals_second_pass(year = year) rebuild.create_final(year = year) invalides.invalide(year = year) final.final(year = year, check = check) temporary_store = get_store(file_name = 'erfs') data_frame = temporary_store['input_{}'.format(year)] # Saving the data_frame openfisca_survey_collection = SurveyCollection(name = "openfisca", config_files_directory = config_files_directory) output_data_directory = openfisca_survey_collection.config.get('data', 'output_directory') survey_name = "openfisca_data_{}".format(year) table = "input" hdf5_file_path = os.path.join(os.path.dirname(output_data_directory), "{}.h5".format(survey_name)) survey = Survey( name = survey_name, hdf5_file_path = hdf5_file_path, ) survey.insert_table(name = table, data_frame = data_frame) openfisca_survey_collection.surveys.append(survey) collections_directory = openfisca_survey_collection.config.get('collections', 'collections_directory') json_file_path = os.path.join(collections_directory, 'openfisca.json') openfisca_survey_collection.dump(json_file_path = json_file_path)
def dump_simulation_results_data_frame(survey_scenario, collection = None): assert collection is not None year = survey_scenario.year data_frame_by_entity = get_calculated_data_frame_by_entity(survey_scenario) openfisca_survey_collection = SurveyCollection.load(collection = "openfisca") output_data_directory = openfisca_survey_collection.config.get('data', 'output_directory') survey_name = "openfisca_data_{}".format(year) for entity, data_frame in data_frame_by_entity.iteritems(): print entity table = entity hdf5_file_path = os.path.join( os.path.dirname(output_data_directory), "{}{}".format(survey_name, ".h5"), ) survey = Survey( name = survey_name, hdf5_file_path = hdf5_file_path, ) survey.insert_table(name = table) survey.fill_hdf(table, data_frame) openfisca_survey_collection.surveys[survey_name] = survey openfisca_survey_collection.dump(collection = "openfisca")
data_frame = data_frame.T.groupby(level = 0).first().T log.info('Saving the openfisca indirect taxation input dataframe') try: openfisca_survey_collection = SurveyCollection.load( collection = 'openfisca_indirect_taxation', config_files_directory = config_files_directory) except: openfisca_survey_collection = SurveyCollection( name = 'openfisca_indirect_taxation', config_files_directory = config_files_directory) output_data_directory = openfisca_survey_collection.config.get('data', 'output_directory') survey_name = "openfisca_indirect_taxation_data_{}".format(year_calage) table = "input" hdf5_file_path = os.path.join(output_data_directory, "{}.h5".format(survey_name)) survey = Survey( name = survey_name, hdf5_file_path = hdf5_file_path, ) survey.insert_table(name = table, data_frame = data_frame) openfisca_survey_collection.surveys.append(survey) openfisca_survey_collection.dump() def run(years_calage): import time year_data_list = [1995, 2000, 2005, 2011] for year_calage in years_calage: start = time.time() run_all(year_calage, year_data_list) log.info("Finished {}".format(time.time() - start)) if __name__ == '__main__':