def start_dockerizer(build_job):
# Update job status to show that its started
build_job.set_status(JobLifeCycle.SCHEDULED)
spawner = DockerizerSpawner(
project_name=build_job.project.unique_name,
project_uuid=build_job.project.uuid.hex,
job_name=build_job.unique_name,
job_uuid=build_job.uuid.hex,
k8s_config=settings.K8S_CONFIG,
namespace=settings.K8S_NAMESPACE,
in_cluster=True)
try:
results = spawner.start_dockerizer(resources=build_job.resources,
node_selectors=build_job.node_selectors)
auditor.record(event_type=BUILD_JOB_STARTED,
instance=build_job,
target='project')
except ApiException as e:
_logger.warning('Could not start build job, please check your polyaxon spec %s', e)
build_job.set_status(
JobLifeCycle.FAILED,
message='Could not start build job, encountered a Kubernetes ApiException.')
return False
except Exception as e:
_logger.warning('Could not start build job, please check your polyaxon spec %s', e)
build_job.set_status(
JobLifeCycle.FAILED,
message='Could not start build job encountered an {} exception.'.format(
e.__class__.__name__
))
return False
build_job.definition = get_job_definition(results)
build_job.save()
return True
def start_dockerizer(build_job):
# Update job status to show that its started
build_job.set_status(JobLifeCycle.SCHEDULED)
spawner = DockerizerSpawner(
project_name=build_job.project.unique_name,
project_uuid=build_job.project.uuid.hex,
job_name=build_job.unique_name,
job_uuid=build_job.uuid.hex,
k8s_config=settings.K8S_CONFIG,
namespace=settings.K8S_NAMESPACE,
in_cluster=True)
error = {}
try:
results = spawner.start_dockerizer(resources=build_job.resources,
node_selector=build_job.node_selector,
affinity=build_job.affinity,
tolerations=build_job.tolerations)
auditor.record(event_type=BUILD_JOB_STARTED,
instance=build_job)
build_job.definition = get_job_definition(results)
build_job.save(update_fields=['definition'])
return True
except ApiException:
_logger.error('Could not start build job, please check your polyaxon spec',
exc_info=True)
error = {
'raised': True,
'traceback': traceback.format_exc(),
'message': 'Could not start build job, encountered a Kubernetes ApiException.'
}
except VolumeNotFoundError as e:
_logger.error('Could not start build job, please check your volume definitions.',
exc_info=True)
error = {
'raised': True,
'traceback': traceback.format_exc(),
'message': 'Could not start build job, encountered a volume definition problem. %s' % e
}
except Exception as e:
_logger.error('Could not start build job, please check your polyaxon spec.',
exc_info=True)
error = {
'raised': True,
'traceback': traceback.format_exc(),
'message': 'Could not start build job encountered an {} exception.'.format(
e.__class__.__name__
)
}
finally:
if error.get('raised'):
build_job.set_status(
JobLifeCycle.FAILED,
message=error.get('message'),
traceback=error.get('traceback'))
def stop_dockerizer(project_name, project_uuid, build_job_name,
build_job_uuid):
spawner = DockerizerSpawner(project_name=project_name,
project_uuid=project_uuid,
job_name=build_job_name,
job_uuid=build_job_uuid,
k8s_config=conf.get('K8S_CONFIG'),
namespace=conf.get('K8S_NAMESPACE'),
in_cluster=True)
return spawner.stop_dockerizer()
def stop_dockerizer(project_name, project_uuid, build_job_name,
build_job_uuid):
spawner = DockerizerSpawner(project_name=project_name,
project_uuid=project_uuid,
job_name=build_job_name,
job_uuid=build_job_uuid,
k8s_config=settings.K8S_CONFIG,
namespace=settings.K8S_NAMESPACE,
in_cluster=True)
spawner.stop_dockerizer()
def stop_dockerizer(build_job, update_status=False):
spawner = DockerizerSpawner(project_name=build_job.project.unique_name,
project_uuid=build_job.project.uuid.hex,
job_name=build_job.unique_name,
job_uuid=build_job.uuid.hex,
k8s_config=settings.K8S_CONFIG,
namespace=settings.K8S_NAMESPACE,
in_cluster=True)
spawner.stop_dockerizer()
if update_status:
# Update experiment status to show that its stopped
build_job.set_status(status=JobLifeCycle.STOPPED,
message='BuildJob was stopped')
