def extract_set( self, datafile_parser, name, config, dict_of_dataframes ) -> pd.DataFrame: data = datafile_parser[name].data indices = ["VALUE"] df = pd.DataFrame(data=data, columns=indices, dtype=config[name]["dtype"]) return check_set_datatype(df, config, name)
def read( self, filepath, **kwargs ) -> Tuple[Dict[str, pd.DataFrame], Dict[str, Any]]: input_data = {} default_values = self._read_default_values(self.input_config) for parameter, details in self.input_config.items(): logger.info("Looking for %s", parameter) config_details = self.input_config[parameter] csv_path = os.path.join(filepath, parameter + ".csv") try: df = pd.read_csv(csv_path) except pd.errors.EmptyDataError: logger.error("No data found in file for %s", parameter) expected_columns = config_details["indices"] default_columns = expected_columns + ["VALUE"] df = pd.DataFrame(columns=default_columns) entity_type = self.input_config[parameter]["type"] if entity_type == "param": narrow = self._check_parameter(df, config_details["indices"], parameter) if not narrow.empty: narrow_checked = check_datatypes( narrow, self.input_config, parameter ) else: narrow_checked = narrow elif entity_type == "set": narrow = self._check_set(df, config_details, parameter) if not narrow.empty: narrow_checked = check_set_datatype( narrow, self.input_config, parameter ) else: narrow_checked = narrow input_data[parameter] = narrow_checked input_data = self._check_index(input_data) return input_data, default_values
def convert_amply_to_dataframe(datafile_parser, config) -> Dict[str, pd.DataFrame]: """Converts an amply parser to a dict of pandas dataframes Arguments --------- datafile_parser : Amply config : Dict Returns ------- dict of pandas.DataFrame """ dict_of_dataframes = {} for name in datafile_parser.symbols.keys(): logger.debug("Extracting data for %s", name) if config[name]["type"] == "param": indices = config[name]["indices"] indices_dtypes = [config[index]["dtype"] for index in indices] indices.append("VALUE") indices_dtypes.append("float") raw_data = datafile_parser[name].data data = convert_amply_data_to_list(raw_data) df = pd.DataFrame(data=data, columns=indices) try: dict_of_dataframes[name] = check_datatypes(df, config, name) except ValueError as ex: msg = "Validation error when checking datatype of {}: {}".format( name, str(ex)) raise ValueError(msg) elif config[name]["type"] == "set": data = datafile_parser[name].data logger.debug(data) indices = ["VALUE"] df = pd.DataFrame(data=data, columns=indices, dtype=config[name]["dtype"]) dict_of_dataframes[name] = check_set_datatype(df, config, name) logger.debug("\n%s\n", dict_of_dataframes[name]) return dict_of_dataframes
def read(self, filepath: Union[str, TextIO], **kwargs) -> Tuple[Dict[str, pd.DataFrame], Dict[str, Any]]: config = self.input_config default_values = self._read_default_values(config) xl = pd.ExcelFile(filepath) input_data = {} for name in xl.sheet_names: try: mod_name = EXCEL_TO_CSV[name] except KeyError: mod_name = name config_details = config[mod_name] df = xl.parse(name) entity_type = config[mod_name]["type"] if entity_type == "param": narrow = self._check_parameter(df, config_details, mod_name) if not narrow.empty: narrow_checked = check_datatypes(narrow, config, mod_name) else: narrow_checked = narrow elif entity_type == "set": narrow = self._check_set(df, config_details, mod_name) if not narrow.empty: narrow_checked = check_set_datatype( narrow, config, mod_name) else: narrow_checked = narrow input_data[mod_name] = narrow_checked input_data = self._check_index(input_data) return input_data, default_values