def read_coefficients(self, source_dim, destination_dim):
try:
return self._coefficients[(source_dim, destination_dim)]
except KeyError:
msg = "Could not find coefficients for conversion from {}>{}"
raise SmifDataNotFoundError(msg.format(source_dim,
destination_dim))
def read_scenario_variant(self, scenario_name, variant_name):
try:
return self._scenarios[scenario_name]['variants'][variant_name]
except KeyError:
raise SmifDataNotFoundError(
"scenario '%s' variant '%s' not found" %
(scenario_name, variant_name))
def read_scenario(self, scenario_name):
try:
scenario = self._scenarios[scenario_name]
return _variant_dict_to_list(scenario)
except KeyError:
raise SmifDataNotFoundError("scenario '%s' not found" %
(scenario_name))
def __init__(self, base_folder):
super().__init__()
self.logger = getLogger(__name__)
# extension for DataArray/list-of-dict data - override in implementations
self.ext = ''
# extension for bare numpy.ndarray data - override in implementations
self.coef_ext = ''
self.base_folder = str(base_folder)
self.data_folder = str(os.path.join(self.base_folder, 'data'))
self.data_folders = {}
self.results_folder = str(os.path.join(self.base_folder, 'results'))
data_folders = [
'coefficients',
'strategies',
'initial_conditions',
'interventions',
'narratives',
'scenarios',
'strategies',
'parameters'
]
for folder in data_folders:
dirname = os.path.join(self.data_folder, folder)
# ensure each directory exists
if not os.path.exists(dirname):
msg = "Expected data folder at '{}' but it does not exist"
abs_path = os.path.abspath(dirname)
raise SmifDataNotFoundError(msg.format(abs_path))
self.data_folders[folder] = dirname
def read_state(self, modelrun_name, timestep, decision_iteration=None):
path = self._get_state_path(modelrun_name, timestep, decision_iteration)
try:
state = self._read_list_of_dicts(path)
except FileNotFoundError:
msg = "Decision state file not found for timestep {}, decision {}"
raise SmifDataNotFoundError(msg.format(timestep, decision_iteration))
return state
def read_coefficients(self, source_dim, destination_dim):
results_path = self._get_coefficients_path(source_dim, destination_dim)
try:
return self._read_ndarray(results_path)
except FileNotFoundError:
msg = "Could not find the coefficients file for %s to %s"
self.logger.warning(msg, source_dim, destination_dim)
raise SmifDataNotFoundError(msg.format(source_dim, destination_dim))
def _read_list_of_dicts(self, path):
"""Read file to list[dict]
"""
try:
return pandas.read_parquet(path, engine='pyarrow').to_dict('records')
except pa.lib.ArrowIOError as ex:
msg = "Unable to read file at {}"
raise SmifDataNotFoundError(msg.format(path)) from ex
def update_scenario(self, scenario_name, scenario):
if scenario_name in self._scenarios:
scenario = _variant_list_to_dict(scenario)
scenario = _skip_coords(scenario, ['provides'])
self._scenarios[scenario_name] = scenario
else:
raise SmifDataNotFoundError("scenario '%s' not found" %
(scenario_name))
def read_narrative(self, sos_model_name, narrative_name):
try:
narrative = [
x for x in self._read_narratives(sos_model_name)
if x['name'] == narrative_name
][0]
except IndexError:
msg = "Narrative '{}' not found in '{}'"
raise SmifDataNotFoundError(
msg.format(narrative_name, sos_model_name))
return narrative
def _read_narrative_variant(self, sos_model_name, narrative_name,
variant_name):
narrative = self.read_narrative(sos_model_name, narrative_name)
try:
variant = [
x for x in narrative['variants'] if x['name'] == variant_name
][0]
except IndexError:
msg = "Variant '{}' not found in '{}'"
raise SmifDataNotFoundError(
msg.format(variant_name, narrative_name))
return variant
def get_intervention(self, name):
"""Return an intervention dict
Returns
-------
dict
"""
try:
return self._register[name]
except KeyError:
msg = "Intervention '{}' is not found in the list of available interventions"
raise SmifDataNotFoundError(msg.format(name))
def _read_data_array(self, path, spec, timestep=None, timesteps=None):
"""Read DataArray from file
"""
try:
dataframe = pandas.read_csv(path)
except FileNotFoundError as ex:
msg = "Could not find data for {} at {}"
raise SmifDataNotFoundError(msg.format(spec.name, path)) from ex
dataframe, spec = DataStore.filter_on_timesteps(
dataframe, spec, path, timestep, timesteps)
data_array = DataStore.dataframe_to_data_array(dataframe, spec, path)
return data_array
def _read_data_array(self, path, spec, timestep=None, timesteps=None):
"""Read DataArray from file
"""
try:
dataframe = pandas.read_parquet(path, engine='pyarrow')
except (pa.lib.ArrowIOError, OSError) as ex:
msg = "Could not find data for {} at {}"
raise SmifDataNotFoundError(msg.format(spec.name, path)) from ex
dataframe, spec = DataStore.filter_on_timesteps(
dataframe, spec, path, timestep, timesteps)
data_array = DataStore.dataframe_to_data_array(dataframe, spec, path)
return data_array
def filter_on_timesteps(cls,
dataframe,
spec,
path,
timestep=None,
timesteps=None):
"""Filter dataframe by timestep
The 'timestep' dimension is treated as follows:
If a single timestep is specified, the spec MAY include 'timestep' as a dimension. If
so, the returned DataArray's spec will match the timestep requested. Otherwise, the
DataArray will not include timestep as a dimension.
If multiple timesteps are specified, the returned DataArray's spec will include a
'timestep' dimension to match the timesteps requested.
If timestep and timesteps are None, and the stored data has a timestep column, read all
available timesteps. The returned DataArray's spec 'timestep' dimension will match the
timesteps requested. If the stored data does not have a timestep column, ignore and
pass through unchanged.
"""
if timestep is not None:
dataframe = cls._check_timestep_column_exists(
dataframe, spec, path)
dataframe = dataframe[dataframe.timestep == timestep]
if 'timestep' in spec.dims:
spec = cls._set_spec_timesteps(spec, [timestep])
else:
dataframe = dataframe.drop('timestep', axis=1)
elif timesteps is not None:
dataframe = cls._check_timestep_column_exists(
dataframe, spec, path)
dataframe = dataframe[dataframe.timestep.isin(timesteps)]
spec = cls._set_spec_timesteps(spec, timesteps)
elif timestep is None and timesteps is None:
try:
dataframe = cls._check_timestep_column_exists(
dataframe, spec, path)
spec = cls._set_spec_timesteps(
spec, sorted(list(dataframe.timestep.unique())))
except SmifDataMismatchError:
pass
if dataframe.empty:
raise SmifDataNotFoundError(
"Data for '{}' not found for timestep {}".format(
spec.name, timestep))
return dataframe, spec
def read_results(self,
modelrun_name,
model_name,
output_spec,
timestep=None,
decision_iteration=None):
key = (modelrun_name, model_name, output_spec.name, timestep,
decision_iteration)
try:
results = self._results[key]
except KeyError:
raise SmifDataNotFoundError(
"Cannot find results for {}".format(key))
return DataArray(output_spec, results)
def read_results(self, modelrun_id, model_name, output_spec, timestep,
decision_iteration=None):
if timestep is None:
raise ValueError("You must pass a timestep argument")
results_path = self._get_results_path(
modelrun_id, model_name, output_spec.name,
timestep, decision_iteration
)
try:
return self._read_data_array(results_path, output_spec)
except FileNotFoundError:
key = str([modelrun_id, model_name, output_spec.name, timestep,
decision_iteration])
raise SmifDataNotFoundError("Could not find results for {}".format(key))
def _read_data_array(self,
lookup,
key,
spec,
timestep=None,
timesteps=None):
try:
data = lookup[key]
except KeyError:
raise SmifDataNotFoundError("Data for {} not found".format(
spec.name))
dataframe = data.as_df()
dataframe, spec = DataStore.filter_on_timesteps(
dataframe, spec, key, timestep, timesteps)
data_array = DataStore.dataframe_to_data_array(dataframe, spec, key)
return data_array
def _read_spatial_file(filepath) -> List[Dict]:
try:
with fiona.Env():
return _read_spatial_data(filepath)
except AttributeError:
# older fiona versions
with fiona.drivers():
return _read_spatial_data(filepath)
except NameError as ex:
msg = "Could not read spatial dimension definition '%s' " % (filepath)
msg += "Please install fiona to read geographic data files. Try running: \n"
msg += " pip install smif[spatial]\n"
msg += "or:\n"
msg += " conda install fiona shapely rtree\n"
raise SmifDataReadError(msg) from ex
except IOError as ex:
msg = "Could not read spatial dimension definition '%s' " % (filepath)
msg += "Please verify that the path is correct and "
msg += "that the file is present on this location."
raise SmifDataNotFoundError(msg) from ex
def read_sos_model(self, sos_model_name):
try:
return self._sos_models[sos_model_name]
except KeyError:
raise SmifDataNotFoundError("sos_model '%s' not found" %
(sos_model_name))
def get_intervention(self, value):
try:
return self._register[value]
except KeyError:
msg = ""
raise SmifDataNotFoundError(msg.format(value))
def _check_exist(config, name):
if name == 'does_not_exist':
raise SmifDataNotFoundError("%s '%s' not found" % (config, name))
def delete_scenario(self, scenario_name):
try:
del self._scenarios[scenario_name]
except KeyError:
raise SmifDataNotFoundError("scenario '%s' not found" %
(scenario_name))
def read_strategies(self, modelrun_name):
try:
return self._strategies[modelrun_name]
except KeyError:
raise SmifDataNotFoundError(
"strategies in modelrun '%s' not found" % (modelrun_name))
def read_model_run(self, model_run_name):
try:
return self._model_runs[model_run_name]
except KeyError:
raise SmifDataNotFoundError("sos_model_run '%s' not found" %
(model_run_name))
def update_model_run(self, model_run_name, model_run):
if model_run_name in self._model_runs:
self._model_runs[model_run_name] = model_run
else:
raise SmifDataNotFoundError("model_run '%s' not found" %
(model_run_name))
def read_model(self, model_name):
try:
return self._models[model_name]
except KeyError:
raise SmifDataNotFoundError("model '%s' not found" % (model_name))
def delete_sos_model(self, sos_model_name):
try:
del self._sos_models[sos_model_name]
except KeyError:
raise SmifDataNotFoundError("sos_model '%s' not found" %
(sos_model_name))
def update_sos_model(self, sos_model_name, sos_model):
if sos_model_name in self._sos_models:
self._sos_models[sos_model_name] = sos_model
else:
raise SmifDataNotFoundError("sos_model '%s' not found" %
(sos_model_name))
def read_interventions_index(self, model_name, index_name, ext):
if model_name not in self._index:
raise SmifDataNotFoundError("model '%s' not found" % (model_name))
if index_name not in self._index[model_name]:
raise SmifDataNotFoundError("index '%s' not found" % (index_name))
return self._index[model_name][index_name]
def delete_model_run(self, model_run_name):
try:
del self._model_runs[model_run_name]
except KeyError:
raise SmifDataNotFoundError("model_run '%s' not found" %
(model_run_name))