def test_pytorch_starter_load_then_train(self, mq_logger):
k = pytorchKernel.PS_torch(mq_logger)
# will also analyze data
k.run(end_stage=KernelRunningState.PREPARE_DATA_DONE)
# k._debug_continue_training = True
k.load_model_weight_continue_train()
def test_pytorch_FL_in_model_early_stop(self, mq_logger):
self._prepare_data(mq_logger)
kernel_load_back = pytorchKernel.PS_torch(mq_logger)
kernel_load_back.load_state_data_only(
KernelRunningState.PREPARE_DATA_DONE)
kernel_load_back.build_and_set_model()
kernel_load_back.train_model()
def test_pytorch_dataset_mean_std(self, mq_logger):
k = pytorchKernel.PS_torch(mq_logger)
# k.run(end_stage=KernelRunningState.PREPARE_DATA_DONE,
# dump_flag=True) # will also analyze data
k._debug_less_data = True
k.run(
end_stage=KernelRunningState.PREPARE_DATA_DONE, dump_flag=False
) # dump not working for torch
k.pre_train()
assert k.img_mean is not None
def test_pytorch_cv_train_more(self, mq_logger):
k = pytorchKernel.PS_torch(mq_logger)
# k._debug_less_data = True
k.run(end_stage=KernelRunningState.PREPARE_DATA_DONE)
k.build_and_set_model()
k.logger.debug("start train 5 epochs")
k.num_epochs = 5
k.train_model()
def test_pytorch_cv_data_prepare(self, mq_logger):
k = pytorchKernel.PS_torch(mq_logger)
k.run(end_stage=KernelRunningState.PREPARE_DATA_DONE)
# k2 = pytorchKernel.PS_torch(mq_logger)
# k2.run(end_stage=KernelRunningState.PREPARE_DATA_DONE)
l = len(k.data_loader)
l_dev = len(k.data_loader_dev)
ratio = l / l_dev
assert ratio > 3.5 # around 0.8/0.2
assert ratio < 4.5
def test_pytorch_cv_train_dev(self, mq_logger):
k = pytorchKernel.PS_torch(mq_logger)
k._debug_less_data = True
k.run(end_stage=KernelRunningState.PREPARE_DATA_DONE)
k.logger.debug("data done")
k.build_and_set_model()
k.num_epochs = 1
k.logger.debug("start train one epoch")
k.train_model()
k.logger.debug("end train one epoch")
assert True
def test_pytorch_data_aug(self, mq_logger):
self._prepare_data(mq_logger)
k = pytorchKernel.PS_torch(mq_logger)
k._debug_less_data = True
k.run(end_stage=KernelRunningState.PREPARE_DATA_DONE)
# k.load_state_data_only(KernelRunningState.PREPARE_DATA_DONE)
# k.run(end_stage=KernelRunningState.PREPARE_DATA_DONE,
# dump_flag=True) # will also analyze data
k.data_loader.dataset._test_(1)
k.data_loader.dataset._test_(2) # test should choose small idx, as
# batch might be small
# k.run() # dump not working for torch
assert k is not None
def test_pytorch_starter_load_then_submit(self, mq_logger):
kernel_load_back = pytorchKernel.PS_torch(mq_logger)
kernel_load_back.load_state_data_only(KernelRunningState.TRAINING_DONE)
kernel_load_back.load_model_weight()
kernel_load_back.run()
def _pytorch_starter_dump(self, mq_logger):
k = pytorchKernel.PS_torch(mq_logger)
k.run(
end_stage=KernelRunningState.PREPARE_DATA_DONE, dump_flag=True
) # will also analyze data