def test_early_stopping(self):
test_rounds = [i for i in range(10, 100)]
decrease_rounds = [np.random.randint(i) for i in test_rounds]
for test_round, decrease_round in zip(test_rounds, decrease_rounds):
eval_dicts = self.generate_fake_eval_metrics(
test_round,
decrease_round,
)
self.early_stopping_round = decrease_round - 1
if self.early_stopping_round <= 0:
continue
validation_strategy = ValidationStrategy(
self.role,
self.mode,
early_stopping_rounds=self.early_stopping_round,
use_first_metric_only=self.use_first_metric_only)
for idx, eval_res in enumerate(eval_dicts):
validation_strategy.performance_recorder.update(eval_res)
check_rs = validation_strategy.check_early_stopping()
if check_rs:
self.assertTrue((test_round - decrease_round +
self.early_stopping_round - 1) == idx)
print('test checking passed')
break
def test_homo_checking(self):
try:
validation_strategy = ValidationStrategy(self.role,
mode='h**o',
early_stopping_rounds=1)
except Exception as e:
# throwing an error is expected
print(e)
print('error detected {}, h**o checking passed'.format(e))
def init_validation_strategy(self, train_data=None, validate_data=None):
validation_strategy = ValidationStrategy(self.role, self.mode, self.validation_freqs,
self.early_stopping_rounds,
self.use_first_metric_only)
validation_strategy.set_train_data(train_data)
validation_strategy.set_validate_data(validate_data)
return validation_strategy
def init_validation_strategy(self, train_data=None, validate_data=None):
validation_strategy = ValidationStrategy(self.role,
consts.HETERO,
self.validation_freqs,
self.early_stopping_rounds,
self.use_first_metric_only,
arbiter_comm=False)
validation_strategy.set_train_data(train_data)
validation_strategy.set_validate_data(validate_data)
return validation_strategy
def init_validation_strategy(self, train_data=None, validate_data=None):
validation_strategy = ValidationStrategy(self.role, self.mode,
self.validation_freqs)
validation_strategy.set_train_data(train_data)
validation_strategy.set_validate_data(validate_data)
return validation_strategy