def fit_on_spark(self,
train_df: DF,
evaluate_df: OPTIONAL_DF = None,
fs_directory: Optional[str] = None,
compression: Optional[str] = None,
num_steps=None,
profile=False,
reduce_results=True,
max_retries=3,
info=None):
super().fit_on_spark(train_df, evaluate_df)
train_df = self._check_and_convert(train_df)
if evaluate_df is not None:
evaluate_df = self._check_and_convert(evaluate_df)
train_ds = create_ml_dataset_from_spark(train_df, self._num_workers,
self._batch_size, fs_directory,
compression)
evaluate_ds = None
if evaluate_df is not None:
evaluate_ds = create_ml_dataset_from_spark(evaluate_df,
self._num_workers,
self._batch_size,
fs_directory,
compression)
return self.fit(train_ds, evaluate_ds, num_steps, profile,
reduce_results, max_retries, info)
def fit_on_spark(self,
train_df: DF,
evaluate_df: OPTIONAL_DF = None,
fs_directory: Optional[str] = None,
compression: Optional[str] = None) -> NoReturn:
super(TFEstimator, self).fit_on_spark(train_df, evaluate_df)
train_df = self._check_and_convert(train_df)
if evaluate_df is not None:
evaluate_df = self._check_and_convert(evaluate_df)
train_ds = create_ml_dataset_from_spark(train_df, self._num_workers,
self._batch_size, fs_directory,
compression)
evaluate_ds = None
if evaluate_df is not None:
evaluate_ds = create_ml_dataset_from_spark(evaluate_df,
self._num_workers,
self._batch_size,
fs_directory,
compression)
return self.fit(train_ds, evaluate_ds)