def start_notebook(notebook, image): spawner = NotebookSpawner(project_name=notebook.project.unique_name, project_uuid=notebook.project.uuid.hex, job_name=notebook.unique_name, job_uuid=notebook.uuid.hex, k8s_config=settings.K8S_CONFIG, namespace=settings.K8S_NAMESPACE, in_cluster=True) try: results = spawner.start_notebook( image=image, resources=notebook.resources, node_selectors=notebook.node_selectors) except ApiException as e: logger.warning( 'Could not start notebook, please check your polyaxon spec %s', e) notebook.set_status( JobLifeCycle.FAILED, message= 'Could not start notebook, encountered a Kubernetes ApiException.') return except Exception as e: logger.warning( 'Could not start notebook, please check your polyaxon spec %s', e) notebook.set_status( JobLifeCycle.FAILED, message='Could not start notebook encountered an {} exception.'. format(e.__class__.__name__)) return notebook.definition = get_job_definition(results) notebook.save()
def get_notebook_token(notebook): spawner = NotebookSpawner(project_name=notebook.project.unique_name, project_uuid=notebook.project.uuid.hex, job_name=notebook.unique_name, job_uuid=notebook.uuid.hex, k8s_config=conf.get('K8S_CONFIG'), namespace=conf.get('K8S_NAMESPACE'), in_cluster=True) return spawner.get_notebook_token()
def get_notebook_url(notebook): spawner = NotebookSpawner(project_name=notebook.project.unique_name, project_uuid=notebook.project.uuid.hex, job_name=notebook.unique_name, job_uuid=notebook.uuid.hex, k8s_config=settings.K8S_CONFIG, namespace=settings.K8S_NAMESPACE, in_cluster=True) return spawner.get_notebook_url()
def get_notebook_token(notebook): spawner = NotebookSpawner( project_name=notebook.project.unique_name, project_uuid=notebook.project.uuid.hex, job_name=notebook.unique_name, job_uuid=notebook.uuid.hex, k8s_config=settings.K8S_CONFIG, namespace=settings.K8S_NAMESPACE, in_cluster=True) return spawner.get_notebook_token()
def stop_notebook(project_name, project_uuid, notebook_job_name, notebook_job_uuid): spawner = NotebookSpawner(project_name=project_name, project_uuid=project_uuid, job_name=notebook_job_name, job_uuid=notebook_job_uuid, k8s_config=conf.get('K8S_CONFIG'), namespace=conf.get('K8S_NAMESPACE'), in_cluster=True) return spawner.stop_notebook()
def stop_notebook(project_name, project_uuid, notebook_job_name, notebook_job_uuid): spawner = NotebookSpawner(project_name=project_name, project_uuid=project_uuid, job_name=notebook_job_name, job_uuid=notebook_job_uuid, k8s_config=settings.K8S_CONFIG, namespace=settings.K8S_NAMESPACE, in_cluster=True) return spawner.stop_notebook()
def stop_notebook(notebook, update_status=False): spawner = NotebookSpawner(project_name=notebook.project.unique_name, project_uuid=notebook.project.uuid.hex, job_name=notebook.unique_name, job_uuid=notebook.uuid.hex, k8s_config=settings.K8S_CONFIG, namespace=settings.K8S_NAMESPACE, in_cluster=True) spawner.stop_notebook() if update_status: # Update experiment status to show that its stopped notebook.set_status(status=JobLifeCycle.STOPPED, message='Notebook was stopped')
def stop_notebook(notebook, update_status=False): spawner = NotebookSpawner( project_name=notebook.project.unique_name, project_uuid=notebook.project.uuid.hex, job_name=notebook.unique_name, job_uuid=notebook.uuid.hex, k8s_config=settings.K8S_CONFIG, namespace=settings.K8S_NAMESPACE, in_cluster=True) spawner.stop_notebook() if update_status: # Update experiment status to show that its stopped notebook.set_status(status=JobLifeCycle.STOPPED, message='Notebook was stopped')
def start_notebook(notebook): # Update job status to show that its started notebook.set_status(JobLifeCycle.SCHEDULED) try: image_name, image_tag = get_image_info(build_job=notebook.build_job) except ValueError as e: _logger.warning('Could not start the notebook, %s', e) notebook.set_status(JobLifeCycle.FAILED, message='External git repo was note found.') return job_docker_image = '{}:{}'.format(image_name, image_tag) _logger.info('Start notebook with built image `%s`', job_docker_image) spawner = NotebookSpawner(project_name=notebook.project.unique_name, project_uuid=notebook.project.uuid.hex, job_name=notebook.unique_name, job_uuid=notebook.uuid.hex, k8s_config=settings.K8S_CONFIG, namespace=settings.K8S_NAMESPACE, in_cluster=True) try: results = spawner.start_notebook( image=job_docker_image, resources=notebook.resources, node_selectors=notebook.node_selectors) except ApiException as e: _logger.warning( 'Could not start notebook, please check your polyaxon spec %s', e) notebook.set_status( JobLifeCycle.FAILED, message= 'Could not start notebook, encountered a Kubernetes ApiException.') return except Exception as e: _logger.warning( 'Could not start notebook, please check your polyaxon spec %s', e) notebook.set_status( JobLifeCycle.FAILED, message='Could not start notebook encountered an {} exception.'. format(e.__class__.__name__)) return notebook.definition = get_job_definition(results) notebook.save()
def start_notebook(notebook): # Update job status to show that its started notebook.set_status(JobLifeCycle.SCHEDULED) try: image_name, image_tag = get_image_info(build_job=notebook.build_job) except ValueError as e: _logger.warning('Could not start the notebook, %s', e) notebook.set_status(JobLifeCycle.FAILED, message='External git repo was note found.') return job_docker_image = '{}:{}'.format(image_name, image_tag) _logger.info('Start notebook with built image `%s`', job_docker_image) spawner = NotebookSpawner( project_name=notebook.project.unique_name, project_uuid=notebook.project.uuid.hex, job_name=notebook.unique_name, job_uuid=notebook.uuid.hex, k8s_config=settings.K8S_CONFIG, namespace=settings.K8S_NAMESPACE, in_cluster=True) try: results = spawner.start_notebook(image=job_docker_image, resources=notebook.resources, node_selectors=notebook.node_selectors) except ApiException as e: _logger.warning('Could not start notebook, please check your polyaxon spec %s', e) notebook.set_status( JobLifeCycle.FAILED, message='Could not start notebook, encountered a Kubernetes ApiException.') return except Exception as e: _logger.warning('Could not start notebook, please check your polyaxon spec %s', e) notebook.set_status( JobLifeCycle.FAILED, message='Could not start notebook encountered an {} exception.'.format( e.__class__.__name__ )) return notebook.definition = get_job_definition(results) notebook.save()
def start_notebook(notebook): # Update job status to show that its started notebook.set_status(JobLifeCycle.SCHEDULED) try: image_name, image_tag = get_image_info(build_job=notebook.build_job) except (ValueError, AttributeError): _logger.error('Could not start the notebook.', exc_info=True) notebook.set_status(JobLifeCycle.FAILED, message='Image info was not found.') return job_docker_image = '{}:{}'.format(image_name, image_tag) _logger.info('Start notebook with built image `%s`', job_docker_image) spawner = NotebookSpawner(project_name=notebook.project.unique_name, project_uuid=notebook.project.uuid.hex, job_name=notebook.unique_name, job_uuid=notebook.uuid.hex, k8s_config=conf.get('K8S_CONFIG'), namespace=conf.get('K8S_NAMESPACE'), job_docker_image=job_docker_image, in_cluster=True) error = {} try: mount_code_in_notebooks = conf.get('MOUNT_CODE_IN_NOTEBOOKS') results = spawner.start_notebook( persistence_outputs=notebook.persistence_outputs, persistence_data=notebook.persistence_data, outputs_refs_jobs=notebook.outputs_refs_jobs, outputs_refs_experiments=notebook.outputs_refs_experiments, resources=notebook.resources, secret_refs=notebook.secret_refs, configmap_refs=notebook.configmap_refs, node_selector=notebook.node_selector, affinity=notebook.affinity, tolerations=notebook.tolerations, backend=notebook.backend, mount_code_in_notebooks=mount_code_in_notebooks) notebook.definition = get_job_definition(results) notebook.save(update_fields=['definition']) return except ApiException: _logger.error( 'Could not start notebook, please check your polyaxon spec.', exc_info=True) error = { 'raised': True, 'traceback': traceback.format_exc(), 'message': 'Could not start the job, encountered a Kubernetes ApiException.', } except VolumeNotFoundError as e: _logger.error( 'Could not start the notebook, please check your volume definitions', exc_info=True) error = { 'raised': True, 'traceback': traceback.format_exc(), 'message': 'Could not start the job, encountered a volume definition problem. %s' % e, } except Exception as e: _logger.error( 'Could not start notebook, please check your polyaxon spec.', exc_info=True) error = { 'raised': True, 'traceback': traceback.format_exc(), 'message': 'Could not start notebook encountered an {} exception.'.format( e.__class__.__name__) } finally: if error.get('raised'): notebook.set_status(JobLifeCycle.FAILED, message=error.get('message'), traceback=error.get('traceback'))
def start_notebook(notebook): # Update job status to show that its started notebook.set_status(JobLifeCycle.SCHEDULED) try: image_name, image_tag = get_image_info(build_job=notebook.build_job) except (ValueError, AttributeError): _logger.error('Could not start the notebook.', exc_info=True) notebook.set_status(JobLifeCycle.FAILED, message='Image info was not found.') return job_docker_image = '{}:{}'.format(image_name, image_tag) _logger.info('Start notebook with built image `%s`', job_docker_image) spawner = NotebookSpawner(project_name=notebook.project.unique_name, project_uuid=notebook.project.uuid.hex, job_name=notebook.unique_name, job_uuid=notebook.uuid.hex, k8s_config=settings.K8S_CONFIG, namespace=settings.K8S_NAMESPACE, in_cluster=True) try: allow_commits = False if settings.REPOS_CLAIM_NAME or notebook.node_selector: allow_commits = True results = spawner.start_notebook( image=job_docker_image, persistence_outputs=notebook.persistence_outputs, persistence_data=notebook.persistence_data, outputs_refs_jobs=notebook.outputs_refs_jobs, outputs_refs_experiments=notebook.outputs_refs_experiments, resources=notebook.resources, node_selector=notebook.node_selector, affinity=notebook.affinity, tolerations=notebook.tolerations, allow_commits=allow_commits) except ApiException: _logger.error( 'Could not start notebook, please check your polyaxon spec.', exc_info=True) notebook.set_status( JobLifeCycle.FAILED, message= 'Could not start notebook, encountered a Kubernetes ApiException.') return except VolumeNotFoundError as e: _logger.error( 'Could not start the notebook, please check your volume definitions', exc_info=True) notebook.set_status(JobLifeCycle.FAILED, message='Could not start the notebook, ' 'encountered a volume definition problem. %s' % e) return False except Exception as e: _logger.error( 'Could not start notebook, please check your polyaxon spec.', exc_info=True) notebook.set_status( JobLifeCycle.FAILED, message='Could not start notebook encountered an {} exception.'. format(e.__class__.__name__)) return notebook.definition = get_job_definition(results) notebook.save()