def create(config: Config,
dataset: Optional[Dataset] = None,
parent_job=None,
model=None):
"Create a new job."
from kge.job import TrainingJob, EvaluationJob, SearchJob
if dataset is None:
dataset = Dataset.create(config)
job_type = config.get("job.type")
if job_type == "train":
return TrainingJob.create(config,
dataset,
parent_job=parent_job,
model=model)
elif job_type == "search":
return SearchJob.create(config, dataset, parent_job=parent_job)
elif job_type == "eval":
return EvaluationJob.create(config,
dataset,
parent_job=parent_job,
model=model)
else:
raise ValueError("unknown job type")
def resume(self, checkpoint_file=None):
"""Load model state from last or specified checkpoint."""
# load model
from kge.job import TrainingJob
training_job = TrainingJob.create(self.config, self.dataset)
training_job.resume(checkpoint_file)
self.model = training_job.model
self.epoch = training_job.epoch
self.resumed_from_job_id = training_job.resumed_from_job_id
self.trace(event="job_resumed",
epoch=self.epoch,
checkpoint_file=checkpoint_file)
def create(config: Config,
dataset: Dataset,
parent_job: "Job" = None) -> "Job":
"""Creates a job for a given configuration."""
from kge.job import TrainingJob, EvaluationJob, SearchJob
if config.get("job.type") == "train":
job = TrainingJob.create(config, dataset, parent_job)
elif config.get("job.type") == "search":
job = SearchJob.create(config, dataset, parent_job)
elif config.get("job.type") == "eval":
job = EvaluationJob.create(config, dataset, parent_job)
else:
raise ValueError("unknown job type")
return job
def __init__(self, config: Config, dataset: Dataset, parent_job, model):
super().__init__(config, dataset, parent_job, model)
training_loss_eval_config = config.clone()
# TODO set train split to include validation data here
# once support is added
# Then reflect this change in the trace entries
self._train_job = TrainingJob.create(
config=training_loss_eval_config,
parent_job=self,
dataset=dataset,
model=model,
forward_only=True,
)
if self.__class__ == TrainingLossEvaluationJob:
for f in Job.job_created_hooks:
f(self)