def pull_job_info(self):
service_id = self._service_id
logger.info('Reading job info from meta store...')
with self._meta_store:
sub_train_job = self._meta_store.get_sub_train_job_by_advisor(
service_id)
if sub_train_job is None:
raise InvalidSubTrainJobError(
'No sub train job associated with advisor "{}"'.format(
service_id))
train_job = self._meta_store.get_train_job(
sub_train_job.train_job_id)
if train_job is None:
raise InvalidSubTrainJobError(
'No such train job with ID "{}"'.format(
sub_train_job.train_job_id))
model = self._meta_store.get_model(sub_train_job.model_id)
if model is None:
raise InvalidSubTrainJobError(
'No such model with ID "{}"'.format(sub_train_job.model_id))
logger.info(f'Using model "{model.name}"...')
logger.info(f'Using budget "{train_job.budget}"...')
trials = self._meta_store.get_trials_of_sub_train_job(
sub_train_job.id)
self.sub_train_job_id = sub_train_job.id
self.budget = train_job.budget
self.model_class = load_model_class(model.model_file_bytes,
model.model_class)
self._num_trials = len(trials)
self._model_id = model.id
def _pull_job_info(self):
service_id = self._service_id
logger.info('Reading job info from meta store...')
with self._meta_store:
worker = self._meta_store.get_inference_job_worker(service_id)
if worker is None:
raise InvalidWorkerError(
'No such worker "{}"'.format(service_id))
inference_job = self._meta_store.get_inference_job(
worker.inference_job_id)
if inference_job is None:
raise InvalidWorkerError(
'No such inference job with ID "{}"'.format(
worker.inference_job_id))
trial = self._meta_store.get_trial(worker.trial_id)
# check if there are trained model saved
if trial is None or trial.store_params_id is None:
# if there are no train job, then check if there is checkpoint uplaoded
if inference_job.model_id:
model = self._meta_store.get_model(inference_job.model_id)
logger.info(
f'Using checkpoint of the model "{model.name}"...')
self._proposal = Proposal.from_jsonable({
"trial_no": 1,
"knobs": {}
})
self._store_params_id = model.checkpoint_id
else:
# if there is no checkpoint id and no trained model saved
raise InvalidTrialError(
'No saved trial with ID "{}" and no checkpoint uploaded'
.format(worker.trial_id))
else:
# create inference with trained parameters first
logger.info(f'Using trial "{trial.id}"...')
model = self._meta_store.get_model(trial.model_id)
if model is None:
raise InvalidTrialError(
'No such model with ID "{}"'.format(trial.model_id))
logger.info(f'Using model "{model.name}"...')
self._proposal = Proposal.from_jsonable(trial.proposal)
self._store_params_id = trial.store_params_id
self._inference_job_id = inference_job.id
self._py_model_class = load_model_class(model.model_file_bytes,
model.model_class, None,
model.model_type,
model.model_file_name)
def _pull_job_info(self):
service_id = self._service_id
logger.info('Reading job info from meta store...')
with self._meta_store:
worker = self._meta_store.get_inference_job_worker(service_id)
if worker is None:
raise InvalidWorkerError(
'No such worker "{}"'.format(service_id))
inference_job = self._meta_store.get_inference_job(
worker.inference_job_id)
if inference_job is None:
raise InvalidWorkerError(
'No such inference job with ID "{}"'.format(
worker.inference_job_id))
if inference_job.model_id:
model = self._meta_store.get_model(inference_job.model_id)
logger.info(f'Using checkpoint of the model "{model.name}"...')
self._proposal = Proposal.from_jsonable({
"trial_no": 1,
"knobs": {}
})
self._store_params_id = model.checkpoint_id
else:
trial = self._meta_store.get_trial(worker.trial_id)
if trial is None or trial.store_params_id is None: # Must have model saved
raise InvalidTrialError(
'No saved trial with ID "{}"'.format(worker.trial_id))
logger.info(f'Using trial "{trial.id}"...')
model = self._meta_store.get_model(trial.model_id)
if model is None:
raise InvalidTrialError(
'No such model with ID "{}"'.format(trial.model_id))
logger.info(f'Using model "{model.name}"...')
self._proposal = Proposal.from_jsonable(trial.proposal)
self._store_params_id = trial.store_params_id
self._inference_job_id = inference_job.id
self._py_model_class = load_model_class(model.model_file_bytes,
model.model_class)
def pull_job_info(self):
service_id = self._service_id
logger.info('Reading job info from meta store...')
with self._meta_store:
worker = self._meta_store.get_train_job_worker(service_id)
if worker is None:
raise InvalidWorkerError(
'No such worker "{}"'.format(service_id))
sub_train_job = self._meta_store.get_sub_train_job(
worker.sub_train_job_id)
if sub_train_job is None:
raise InvalidWorkerError(
'No such sub train job associated with advisor "{}"'.format(
service_id))
train_job = self._meta_store.get_train_job(
sub_train_job.train_job_id)
if train_job is None:
raise InvalidWorkerError(
'No such train job with ID "{}"'.format(
sub_train_job.train_job_id))
model = self._meta_store.get_model(sub_train_job.model_id)
if model is None:
raise InvalidWorkerError('No such model with ID "{}"'.format(
sub_train_job.model_id))
logger.info(f'Using model "{model.name}"...')
(self.train_dataset_path, self.val_dataset_path, self.annotation_dataset_path) = self._load_datasets(train_job)
logger.info('Dataset_loaded')
self.train_args = train_job.train_args
self.sub_train_job_id = sub_train_job.id
logger.info('Loading model class for model...')
self.model_class = load_model_class(model.model_file_bytes,
model.model_class, None,
model.model_type,
model.model_file_name)
logger.info('Model Loaded')