def convert_df(self, data: List[List[str]], variable: str, start_year: int,
end_year: int) -> pd.DataFrame:
"""Read the cplex lines into a pandas DataFrame"""
index = self.results_config[variable]["indices"]
columns = ["variable"] + index[:-1] + list(
range(start_year, end_year + 1, 1))
df = pd.DataFrame(data=data, columns=columns)
df, index = check_duplicate_index(df, columns, index)
df = df.drop(columns="variable")
LOGGER.debug(
f"Attempting to set index for {variable} with columns {index[:-1]}"
)
try:
df = df.set_index(index[:-1])
except NotImplementedError as ex:
LOGGER.error(f"Error setting index for {df.head()}")
raise NotImplementedError(ex)
df = df.melt(var_name="YEAR", value_name="VALUE", ignore_index=False)
df = df.reset_index()
df = check_datatypes(df, {
**self.input_config,
**self.results_config
}, variable)
df = df.set_index(index)
df = df[(df != 0).any(axis=1)]
return df
def extract_param(self, config, name, datafile_parser,
dict_of_dataframes) -> pd.DataFrame:
indices = config[name]["indices"].copy()
indices_dtypes = [config[index]["dtype"] for index in indices]
indices.append("VALUE")
indices_dtypes.append("float")
raw_data = datafile_parser[name].data
data = self._convert_amply_data_to_list(raw_data)
df = pd.DataFrame(data=data, columns=indices)
try:
return check_datatypes(df, config, name)
except ValueError as ex:
msg = "Validation error when checking datatype of {}: {}".format(
name, str(ex))
raise ValueError(msg)
def read(
self, filepath, **kwargs
) -> Tuple[Dict[str, pd.DataFrame], Dict[str, Any]]:
input_data = {}
default_values = self._read_default_values(self.input_config)
for parameter, details in self.input_config.items():
logger.info("Looking for %s", parameter)
config_details = self.input_config[parameter]
csv_path = os.path.join(filepath, parameter + ".csv")
try:
df = pd.read_csv(csv_path)
except pd.errors.EmptyDataError:
logger.error("No data found in file for %s", parameter)
expected_columns = config_details["indices"]
default_columns = expected_columns + ["VALUE"]
df = pd.DataFrame(columns=default_columns)
entity_type = self.input_config[parameter]["type"]
if entity_type == "param":
narrow = self._check_parameter(df, config_details["indices"], parameter)
if not narrow.empty:
narrow_checked = check_datatypes(
narrow, self.input_config, parameter
)
else:
narrow_checked = narrow
elif entity_type == "set":
narrow = self._check_set(df, config_details, parameter)
if not narrow.empty:
narrow_checked = check_set_datatype(
narrow, self.input_config, parameter
)
else:
narrow_checked = narrow
input_data[parameter] = narrow_checked
input_data = self._check_index(input_data)
return input_data, default_values
def convert_amply_to_dataframe(datafile_parser,
config) -> Dict[str, pd.DataFrame]:
"""Converts an amply parser to a dict of pandas dataframes
Arguments
---------
datafile_parser : Amply
config : Dict
Returns
-------
dict of pandas.DataFrame
"""
dict_of_dataframes = {}
for name in datafile_parser.symbols.keys():
logger.debug("Extracting data for %s", name)
if config[name]["type"] == "param":
indices = config[name]["indices"]
indices_dtypes = [config[index]["dtype"] for index in indices]
indices.append("VALUE")
indices_dtypes.append("float")
raw_data = datafile_parser[name].data
data = convert_amply_data_to_list(raw_data)
df = pd.DataFrame(data=data, columns=indices)
try:
dict_of_dataframes[name] = check_datatypes(df, config, name)
except ValueError as ex:
msg = "Validation error when checking datatype of {}: {}".format(
name, str(ex))
raise ValueError(msg)
elif config[name]["type"] == "set":
data = datafile_parser[name].data
logger.debug(data)
indices = ["VALUE"]
df = pd.DataFrame(data=data,
columns=indices,
dtype=config[name]["dtype"])
dict_of_dataframes[name] = check_set_datatype(df, config, name)
logger.debug("\n%s\n", dict_of_dataframes[name])
return dict_of_dataframes
def read(self, filepath: Union[str, TextIO],
**kwargs) -> Tuple[Dict[str, pd.DataFrame], Dict[str, Any]]:
config = self.input_config
default_values = self._read_default_values(config)
xl = pd.ExcelFile(filepath)
input_data = {}
for name in xl.sheet_names:
try:
mod_name = EXCEL_TO_CSV[name]
except KeyError:
mod_name = name
config_details = config[mod_name]
df = xl.parse(name)
entity_type = config[mod_name]["type"]
if entity_type == "param":
narrow = self._check_parameter(df, config_details, mod_name)
if not narrow.empty:
narrow_checked = check_datatypes(narrow, config, mod_name)
else:
narrow_checked = narrow
elif entity_type == "set":
narrow = self._check_set(df, config_details, mod_name)
if not narrow.empty:
narrow_checked = check_set_datatype(
narrow, config, mod_name)
else:
narrow_checked = narrow
input_data[mod_name] = narrow_checked
input_data = self._check_index(input_data)
return input_data, default_values
def convert_df(self, row_as_list: List, start_year: int,
end_year: int) -> Tuple[str, pd.DataFrame]:
"""Read the cplex line into a pandas DataFrame
"""
variable, dimensions, values = self.extract_variable_dimensions_values(
row_as_list)
index = self.results_config[variable]["indices"]
columns = index[:-1] + list(range(start_year, end_year + 1, 1))
df = pd.DataFrame(data=[list(dimensions) + values],
columns=columns).set_index(index[:-1])
df = df.melt(var_name="YEAR", value_name="VALUE", ignore_index=False)
df = df.reset_index()
df = check_datatypes(df, {
**self.input_config,
**self.results_config
}, variable)
df = df.set_index(index)
df = df[(df != 0).any(axis=1)]
return (variable, df)