def store_input_data_frame(data_frame = None, collection = None, survey = None, table = None):
assert data_frame is not None
assert collection is not None
assert survey is not None
try:
openfisca_survey_collection = SurveyCollection.load(collection = collection)
except Exception as e:
openfisca_survey_collection = SurveyCollection(name = collection)
log.debug("In collection {} the following survey are present: {}".format(collection, openfisca_survey_collection.surveys))
output_data_directory = openfisca_survey_collection.config.get('data', 'output_directory')
if table is None:
table = "input"
#
survey_name = survey
hdf5_file_path = os.path.join(os.path.dirname(output_data_directory), "{}.h5".format(survey_name))
available_survey_names = [survey_.name for survey_ in openfisca_survey_collection.surveys]
if survey_name in available_survey_names:
survey = openfisca_survey_collection.get_survey(survey_name)
else:
survey = Survey(name = survey_name, hdf5_file_path = hdf5_file_path)
survey.insert_table(name = table, data_frame = data_frame)
openfisca_survey_collection.surveys.append(survey)
collections_directory = openfisca_survey_collection.config.get('collections', 'collections_directory')
json_file_path = os.path.join(collections_directory, '{}.json'.format(collection))
log.debug("In collection {} the following surveyx are present: {}".format(collection, openfisca_survey_collection.surveys))
openfisca_survey_collection.dump(json_file_path = json_file_path)
def set_table_in_survey(input_dataframe, entity, period, collection, survey_name, survey_label = None,
table_label = None, table_name = None):
period = periods.period(period)
if table_name is None:
table_name = entity + '_' + str(period)
if table_label is None:
table_label = "Input data for entity {} at period {}".format(entity, period)
try:
survey_collection = SurveyCollection.load(collection = collection)
except configparser.NoOptionError:
survey_collection = SurveyCollection(name = collection)
except configparser.NoSectionError: # For tests
data_dir = os.path.join(
pkg_resources.get_distribution('openfisca-survey-manager').location,
'openfisca_survey_manager',
'tests',
'data_files',
)
survey_collection = SurveyCollection(
name = collection,
config_files_directory = data_dir,
)
try:
survey = survey_collection.get_survey(survey_name)
except AssertionError:
survey = Survey(
name = survey_name,
label = survey_label or None,
survey_collection = survey_collection,
)
if survey.hdf5_file_path is None:
config = survey.survey_collection.config
directory_path = config.get("data", "output_directory")
if not os.path.isdir(directory_path):
log.warn("{} who should be the HDF5 data directory does not exist: we create the directory".format(
directory_path))
os.makedirs(directory_path)
survey.hdf5_file_path = os.path.join(directory_path, survey.name + '.h5')
assert survey.hdf5_file_path is not None
survey.insert_table(label = table_label, name = table_name, dataframe = input_dataframe)
survey_collection.surveys = [
kept_survey for kept_survey in survey_collection.surveys if kept_survey.name != survey_name
]
survey_collection.surveys.append(survey)
collections_directory = survey_collection.config.get('collections', 'collections_directory')
assert os.path.isdir(collections_directory), """{} who should be the collections' directory does not exist.
Fix the option collections_directory in the collections section of your config file.""".format(collections_directory)
collection_json_path = os.path.join(collections_directory, "{}.json".format(collection))
survey_collection.dump(json_file_path = collection_json_path)
def set_table_in_survey(input_dataframe, entity, period, collection, survey_name, survey_label = None,
table_label = None, table_name = None):
period = periods.period(period)
if table_name is None:
table_name = entity + '_' + str(period)
if table_label is None:
table_label = "Input data for entity {} at period {}".format(entity, period)
try:
survey_collection = SurveyCollection.load(collection = collection)
except configparser.NoOptionError:
survey_collection = SurveyCollection(name = collection)
except configparser.NoSectionError: # For tests
data_dir = os.path.join(
pkg_resources.get_distribution('openfisca-survey-manager').location,
'openfisca_survey_manager',
'tests',
'data_files',
)
survey_collection = SurveyCollection(
name = collection,
config_files_directory = data_dir,
)
try:
survey = survey_collection.get_survey(survey_name)
except AssertionError:
survey = Survey(
name = survey_name,
label = survey_label or None,
survey_collection = survey_collection,
)
if survey.hdf5_file_path is None:
config = survey.survey_collection.config
directory_path = config.get("data", "output_directory")
if not os.path.isdir(directory_path):
log.warn("{} who should be the HDF5 data directory does not exist: we create the directory".format(
directory_path))
os.makedirs(directory_path)
survey.hdf5_file_path = os.path.join(directory_path, survey.name + '.h5')
assert survey.hdf5_file_path is not None
survey.insert_table(label = table_label, name = table_name, dataframe = input_dataframe)
survey_collection.surveys = [
kept_survey for kept_survey in survey_collection.surveys if kept_survey.name != survey_name
]
survey_collection.surveys.append(survey)
collections_directory = survey_collection.config.get('collections', 'collections_directory')
assert os.path.isdir(collections_directory), """{} who should be the collections' directory does not exist.
Fix the option collections_directory in the collections section of your config file.""".format(collections_directory)
collection_json_path = os.path.join(collections_directory, "{}.json".format(collection))
survey_collection.dump(json_file_path = collection_json_path)
def store_input_data_frame(data_frame=None,
collection=None,
survey=None,
table=None):
assert data_frame is not None
assert collection is not None
assert survey is not None
try:
openfisca_survey_collection = SurveyCollection.load(
collection=collection)
except Exception as e:
openfisca_survey_collection = SurveyCollection(name=collection)
log.debug("In collection {} the following survey are present: {}".format(
collection, openfisca_survey_collection.surveys))
output_data_directory = openfisca_survey_collection.config.get(
'data', 'output_directory')
if table is None:
table = "input"
#
survey_name = survey
hdf5_file_path = os.path.join(os.path.dirname(output_data_directory),
"{}.h5".format(survey_name))
available_survey_names = [
survey_.name for survey_ in openfisca_survey_collection.surveys
]
if survey_name in available_survey_names:
survey = openfisca_survey_collection.get_survey(survey_name)
else:
survey = Survey(name=survey_name, hdf5_file_path=hdf5_file_path)
survey.insert_table(name=table, data_frame=data_frame)
openfisca_survey_collection.surveys.append(survey)
collections_directory = openfisca_survey_collection.config.get(
'collections', 'collections_directory')
json_file_path = os.path.join(collections_directory,
'{}.json'.format(collection))
log.debug("In collection {} the following surveyx are present: {}".format(
collection, openfisca_survey_collection.surveys))
openfisca_survey_collection.dump(json_file_path=json_file_path)