def deserialize(self, data: str) -> Any:
df = pd.read_csv(StringIO(data))
df = _pre_transform(
[df], self.num_cols, self.cat_cols, self.codes, self.mean, self.std, self.target_col, self.target_codes
)[0]
cat_vars = _to_cat_vars_numpy(df, self.cat_cols)
num_vars = _to_num_vars_numpy(df, self.num_cols)
cat_vars = np.stack(cat_vars, 1)
num_vars = np.stack(num_vars, 1)
return [{DefaultDataKeys.INPUT: [c, n]} for c, n in zip(cat_vars, num_vars)]
def serve_load_sample(self, data: str) -> Any:
parameters = self._parameters
df = read_csv(StringIO(data))
df = _pre_transform(
df,
parameters["numerical_fields"],
parameters["categorical_fields"],
parameters["codes"],
parameters["mean"],
parameters["std"],
)
cat_vars = _to_cat_vars_numpy(df, parameters["categorical_fields"])
num_vars = _to_num_vars_numpy(df, parameters["numerical_fields"])
cat_vars = np.stack(cat_vars, 1)
num_vars = np.stack(num_vars, 1)
return [{DataKeys.INPUT: [c, n]} for c, n in zip(cat_vars, num_vars)]
def deserialize(self, data: str) -> Any:
columns = data.split("\n")[0].split(',')
df = pd.DataFrame([
TabularDeserializer._convert_row(x.split(',')[1:])
for x in data.split('\n')[1:-1]
],
columns=columns)
df = _pre_transform([df], self.num_cols, self.cat_cols, self.codes,
self.mean, self.std, self.target_col,
self.target_codes)[0]
cat_vars = _to_cat_vars_numpy(df, self.cat_cols)
num_vars = _to_num_vars_numpy(df, self.num_cols)
cat_vars = np.stack(cat_vars, 1)
num_vars = np.stack(num_vars, 1)
return [{
DefaultDataKeys.INPUT: [c, n]
} for c, n in zip(cat_vars, num_vars)]
def preprocess(
self,
df: DataFrame,
categorical_fields: Optional[List[str]] = None,
numerical_fields: Optional[List[str]] = None,
parameters: Dict[str, Any] = None,
):
if self.training:
categorical_fields, numerical_fields = self._sanetize_fields(
categorical_fields, numerical_fields)
parameters = self.compute_parameters(df, numerical_fields,
categorical_fields)
elif parameters is None:
raise MisconfigurationException(
"Loading tabular data for evaluation or inference requires parameters from the train data. Either "
"construct the train data at the same time as evaluation and inference or provide the train "
"`datamodule.parameters` in the `parameters` argument.")
self.parameters = parameters
# impute and normalize data
df = _pre_transform(
df,
parameters["numerical_fields"],
parameters["categorical_fields"],
parameters["codes"],
parameters["mean"],
parameters["std"],
)
cat_vars = _to_cat_vars_numpy(df, parameters["categorical_fields"])
num_vars = _to_num_vars_numpy(df, parameters["numerical_fields"])
num_samples = len(df)
cat_vars = np.stack(cat_vars, 1) if len(cat_vars) else np.zeros(
(num_samples, 0), dtype=np.int64)
num_vars = np.stack(num_vars, 1) if len(num_vars) else np.zeros(
(num_samples, 0), dtype=np.float32)
return cat_vars, num_vars
def common_load_data(
self,
df: DataFrame,
dataset: Optional[Any] = None,
):
# impute_data
# compute train dataset stats
dfs = _pre_transform(
[df], self.num_cols, self.cat_cols, self.codes, self.mean, self.std, self.target_col, self.target_codes
)
df = dfs[0]
if dataset is not None:
dataset.num_samples = len(df)
cat_vars = _to_cat_vars_numpy(df, self.cat_cols)
num_vars = _to_num_vars_numpy(df, self.num_cols)
cat_vars = np.stack(cat_vars, 1) # if len(cat_vars) else np.zeros((len(self), 0))
num_vars = np.stack(num_vars, 1) # if len(num_vars) else np.zeros((len(self), 0))
return df, cat_vars, num_vars