def auto_generated_pipeline(date_of_processing='02_06_2021',
dir_mask_specie='Ponca_DV',
dir_specie='Ponca_DV_loc',
dir_years='forest_jEquihua_mar',
file_mask_specie='poncamask.tif',
file_specie='poncadav2',
specie='pan_onca',
vol_shared_volume='hostpath-pvc'):
_kale_pvolumes_dict = OrderedDict()
_kale_volume_step_names = []
_kale_volume_name_parameters = []
_kale_annotations = {}
_kale_volume = _kfp_dsl.PipelineVolume(pvc=vol_shared_volume)
_kale_pvolumes_dict['/shared_volume'] = _kale_volume
_kale_volume_step_names.sort()
_kale_volume_name_parameters.sort()
_kale_downloadfroms3_task = _kale_downloadfroms3_op()\
.add_pvolumes(_kale_pvolumes_dict)\
.after()
_kale_downloadfroms3_task.container.working_dir = "//shared_volume/kube_sipecam_playground/hsi/notebooks"
_kale_downloadfroms3_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_output_artifacts.update({'downloadfroms3': '/downloadfroms3.html'})
_kale_downloadfroms3_task.output_artifact_paths.update(
_kale_output_artifacts)
_kale_downloadfroms3_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_downloadfroms3_task.dependent_names +
_kale_volume_step_names)
_kale_downloadfroms3_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_downloadfroms3_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
_kale_readdatainput_task = _kale_readdatainput_op(dir_mask_specie, dir_specie, file_mask_specie, file_specie)\
.add_pvolumes(_kale_pvolumes_dict)\
.after(_kale_downloadfroms3_task)
_kale_readdatainput_task.container.working_dir = "//shared_volume/kube_sipecam_playground/hsi/notebooks"
_kale_readdatainput_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_output_artifacts.update({'readdatainput': '/readdatainput.html'})
_kale_readdatainput_task.output_artifact_paths.update(
_kale_output_artifacts)
_kale_readdatainput_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_readdatainput_task.dependent_names +
_kale_volume_step_names)
_kale_readdatainput_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_readdatainput_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
_kale_reproject_task = _kale_reproject_op()\
.add_pvolumes(_kale_pvolumes_dict)\
.after(_kale_readdatainput_task)
_kale_reproject_task.container.working_dir = "//shared_volume/kube_sipecam_playground/hsi/notebooks"
_kale_reproject_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_output_artifacts.update({'reproject': '/reproject.html'})
_kale_reproject_task.output_artifact_paths.update(_kale_output_artifacts)
_kale_reproject_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_reproject_task.dependent_names +
_kale_volume_step_names)
_kale_reproject_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_reproject_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
_kale_createtestdata_task = _kale_createtestdata_op(dir_years)\
.add_pvolumes(_kale_pvolumes_dict)\
.after(_kale_reproject_task)
_kale_createtestdata_task.container.working_dir = "//shared_volume/kube_sipecam_playground/hsi/notebooks"
_kale_createtestdata_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_output_artifacts.update({'createtestdata': '/createtestdata.html'})
_kale_createtestdata_task.output_artifact_paths.update(
_kale_output_artifacts)
_kale_createtestdata_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_createtestdata_task.dependent_names +
_kale_volume_step_names)
_kale_createtestdata_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_createtestdata_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
_kale_maskandextract_task = _kale_maskandextract_op()\
.add_pvolumes(_kale_pvolumes_dict)\
.after(_kale_createtestdata_task)
_kale_maskandextract_task.container.working_dir = "//shared_volume/kube_sipecam_playground/hsi/notebooks"
_kale_maskandextract_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_output_artifacts.update({'maskandextract': '/maskandextract.html'})
_kale_maskandextract_task.output_artifact_paths.update(
_kale_output_artifacts)
_kale_maskandextract_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_maskandextract_task.dependent_names +
_kale_volume_step_names)
_kale_maskandextract_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_maskandextract_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
_kale_bestmodel_task = _kale_bestmodel_op()\
.add_pvolumes(_kale_pvolumes_dict)\
.after(_kale_maskandextract_task)
_kale_bestmodel_task.container.working_dir = "//shared_volume/kube_sipecam_playground/hsi/notebooks"
_kale_bestmodel_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_output_artifacts.update({'bestmodel': '/bestmodel.html'})
_kale_bestmodel_task.output_artifact_paths.update(_kale_output_artifacts)
_kale_bestmodel_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_bestmodel_task.dependent_names +
_kale_volume_step_names)
_kale_bestmodel_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_bestmodel_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
_kale_temporalprojection_task = _kale_temporalprojection_op(date_of_processing, specie)\
.add_pvolumes(_kale_pvolumes_dict)\
.after(_kale_bestmodel_task)
_kale_temporalprojection_task.container.working_dir = "//shared_volume/kube_sipecam_playground/hsi/notebooks"
_kale_temporalprojection_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_output_artifacts.update(
{'temporalprojection': '/temporalprojection.html'})
_kale_temporalprojection_task.output_artifact_paths.update(
_kale_output_artifacts)
_kale_temporalprojection_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_temporalprojection_task.dependent_names +
_kale_volume_step_names)
_kale_temporalprojection_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_temporalprojection_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
_kale_uploadtos3_task = _kale_uploadtos3_op(date_of_processing)\
.add_pvolumes(_kale_pvolumes_dict)\
.after(_kale_temporalprojection_task)
_kale_uploadtos3_task.container.working_dir = "//shared_volume/kube_sipecam_playground/hsi/notebooks"
_kale_uploadtos3_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_output_artifacts.update({'uploadtos3': '/uploadtos3.html'})
_kale_uploadtos3_task.output_artifact_paths.update(_kale_output_artifacts)
_kale_uploadtos3_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_uploadtos3_task.dependent_names +
_kale_volume_step_names)
_kale_uploadtos3_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_uploadtos3_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
def auto_generated_pipeline(DEPLOY_NAMESPACE='admin',
EXPLAINER_MODEL_PATH='sklearn/income/explainer',
INCOME_MODEL_PATH='sklearn/income/model',
MINIO_ACCESS_KEY='minio',
MINIO_HOST='minio-service.kubeflow:9000',
MINIO_MODEL_BUCKET='seldon',
MINIO_SECRET_KEY='minio123',
OUTLIER_MODEL_PATH='sklearn/income/outlier'):
pvolumes_dict = OrderedDict()
volume_step_names = []
volume_name_parameters = []
marshal_vop = dsl.VolumeOp(name="kale-marshal-volume",
resource_name="kale-marshal-pvc",
modes=dsl.VOLUME_MODE_RWM,
size="1Gi")
volume_step_names.append(marshal_vop.name)
volume_name_parameters.append(marshal_vop.outputs["name"].full_name)
pvolumes_dict['/marshal'] = marshal_vop.volume
volume_step_names.sort()
volume_name_parameters.sort()
setup_task = setup_op(MINIO_ACCESS_KEY, MINIO_HOST, MINIO_MODEL_BUCKET, MINIO_SECRET_KEY)\
.add_pvolumes(pvolumes_dict)\
.after()
setup_task.container.working_dir = "/home/jovyan"
setup_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'setup': '/setup.html'})
setup_task.output_artifact_paths.update(output_artifacts)
setup_task.add_pod_label("pipelines.kubeflow.org/metadata_written", "true")
dep_names = setup_task.dependent_names + volume_step_names
setup_task.add_pod_annotation("kubeflow-kale.org/dependent-templates",
json.dumps(dep_names))
if volume_name_parameters:
setup_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(volume_name_parameters))
build_model_task = build_model_op(INCOME_MODEL_PATH, MINIO_ACCESS_KEY, MINIO_HOST, MINIO_MODEL_BUCKET, MINIO_SECRET_KEY)\
.add_pvolumes(pvolumes_dict)\
.after(setup_task)
build_model_task.container.working_dir = "/home/jovyan"
build_model_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'build_model': '/build_model.html'})
build_model_task.output_artifact_paths.update(output_artifacts)
build_model_task.add_pod_label("pipelines.kubeflow.org/metadata_written",
"true")
dep_names = build_model_task.dependent_names + volume_step_names
build_model_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(dep_names))
if volume_name_parameters:
build_model_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(volume_name_parameters))
build_outlier_task = build_outlier_op(MINIO_ACCESS_KEY, MINIO_HOST, MINIO_MODEL_BUCKET, MINIO_SECRET_KEY, OUTLIER_MODEL_PATH)\
.add_pvolumes(pvolumes_dict)\
.after(build_model_task)
build_outlier_task.container.working_dir = "/home/jovyan"
build_outlier_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'build_outlier': '/build_outlier.html'})
build_outlier_task.output_artifact_paths.update(output_artifacts)
build_outlier_task.add_pod_label("pipelines.kubeflow.org/metadata_written",
"true")
dep_names = build_outlier_task.dependent_names + volume_step_names
build_outlier_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(dep_names))
if volume_name_parameters:
build_outlier_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(volume_name_parameters))
train_explainer_task = train_explainer_op(EXPLAINER_MODEL_PATH, MINIO_ACCESS_KEY, MINIO_HOST, MINIO_MODEL_BUCKET, MINIO_SECRET_KEY)\
.add_pvolumes(pvolumes_dict)\
.after(build_model_task)
train_explainer_task.container.working_dir = "/home/jovyan"
train_explainer_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'train_explainer': '/train_explainer.html'})
train_explainer_task.output_artifact_paths.update(output_artifacts)
train_explainer_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
dep_names = train_explainer_task.dependent_names + volume_step_names
train_explainer_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(dep_names))
if volume_name_parameters:
train_explainer_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(volume_name_parameters))
deploy_seldon_task = deploy_seldon_op(DEPLOY_NAMESPACE, EXPLAINER_MODEL_PATH, INCOME_MODEL_PATH, MINIO_ACCESS_KEY, MINIO_HOST, MINIO_MODEL_BUCKET, MINIO_SECRET_KEY)\
.add_pvolumes(pvolumes_dict)\
.after(train_explainer_task)
deploy_seldon_task.container.working_dir = "/home/jovyan"
deploy_seldon_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'deploy_seldon': '/deploy_seldon.html'})
deploy_seldon_task.output_artifact_paths.update(output_artifacts)
deploy_seldon_task.add_pod_label("pipelines.kubeflow.org/metadata_written",
"true")
dep_names = deploy_seldon_task.dependent_names + volume_step_names
deploy_seldon_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(dep_names))
if volume_name_parameters:
deploy_seldon_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(volume_name_parameters))
test_model_task = test_model_op(DEPLOY_NAMESPACE, MINIO_ACCESS_KEY, MINIO_HOST, MINIO_SECRET_KEY)\
.add_pvolumes(pvolumes_dict)\
.after(deploy_seldon_task)
test_model_task.container.working_dir = "/home/jovyan"
test_model_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'test_model': '/test_model.html'})
test_model_task.output_artifact_paths.update(output_artifacts)
test_model_task.add_pod_label("pipelines.kubeflow.org/metadata_written",
"true")
dep_names = test_model_task.dependent_names + volume_step_names
test_model_task.add_pod_annotation("kubeflow-kale.org/dependent-templates",
json.dumps(dep_names))
if volume_name_parameters:
test_model_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(volume_name_parameters))
deploy_outlier_task = deploy_outlier_op(DEPLOY_NAMESPACE, MINIO_ACCESS_KEY, MINIO_HOST, MINIO_MODEL_BUCKET, MINIO_SECRET_KEY, OUTLIER_MODEL_PATH)\
.add_pvolumes(pvolumes_dict)\
.after(build_outlier_task, test_model_task)
deploy_outlier_task.container.working_dir = "/home/jovyan"
deploy_outlier_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'deploy_outlier': '/deploy_outlier.html'})
deploy_outlier_task.output_artifact_paths.update(output_artifacts)
deploy_outlier_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
dep_names = deploy_outlier_task.dependent_names + volume_step_names
deploy_outlier_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(dep_names))
if volume_name_parameters:
deploy_outlier_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(volume_name_parameters))
deploy_event_display_task = deploy_event_display_op(DEPLOY_NAMESPACE, MINIO_ACCESS_KEY, MINIO_HOST, MINIO_SECRET_KEY)\
.add_pvolumes(pvolumes_dict)\
.after(deploy_outlier_task)
deploy_event_display_task.container.working_dir = "/home/jovyan"
deploy_event_display_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update(
{'deploy_event_display': '/deploy_event_display.html'})
deploy_event_display_task.output_artifact_paths.update(output_artifacts)
deploy_event_display_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
dep_names = deploy_event_display_task.dependent_names + volume_step_names
deploy_event_display_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(dep_names))
if volume_name_parameters:
deploy_event_display_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(volume_name_parameters))
test_outliers_task = test_outliers_op(DEPLOY_NAMESPACE, MINIO_ACCESS_KEY, MINIO_HOST, MINIO_SECRET_KEY)\
.add_pvolumes(pvolumes_dict)\
.after(deploy_event_display_task)
test_outliers_task.container.working_dir = "/home/jovyan"
test_outliers_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'test_outliers': '/test_outliers.html'})
test_outliers_task.output_artifact_paths.update(output_artifacts)
test_outliers_task.add_pod_label("pipelines.kubeflow.org/metadata_written",
"true")
dep_names = test_outliers_task.dependent_names + volume_step_names
test_outliers_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(dep_names))
if volume_name_parameters:
test_outliers_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(volume_name_parameters))
explain_task = explain_op(MINIO_ACCESS_KEY, MINIO_HOST, MINIO_SECRET_KEY)\
.add_pvolumes(pvolumes_dict)\
.after(train_explainer_task)
explain_task.container.working_dir = "/home/jovyan"
explain_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'explain': '/explain.html'})
explain_task.output_artifact_paths.update(output_artifacts)
explain_task.add_pod_label("pipelines.kubeflow.org/metadata_written",
"true")
dep_names = explain_task.dependent_names + volume_step_names
explain_task.add_pod_annotation("kubeflow-kale.org/dependent-templates",
json.dumps(dep_names))
if volume_name_parameters:
explain_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(volume_name_parameters))
def dispatch(job=None, namespace='default', debug=False, **kwargs):
logger.info(f"starting {job['id']}")
from kubernetes import client, config
config.load_incluster_config()
coreV1 = client.CoreV1Api()
batchV1 = client.BatchV1Api()
batchV1.create_namespaced_job(
namespace=namespace,
body=client.V1Job(
api_version='batch/v1',
kind='Job',
metadata=client.V1ObjectMeta(
name=job['id'],
annotations={
f"container.apparmor.security.beta.kubernetes.io/appyter-{job['id']}":
'unconfined'
},
),
spec=client.V1JobSpec(
template=client.V1PodTemplateSpec(spec=client.V1PodSpec(
restart_policy='Never',
containers=[
client.V1Container(
name=f"appyter-{job['id']}",
image=job['image'],
command=[
'appyter', 'orchestration', 'job',
json.dumps(job)
],
security_context=client.V1SecurityContext(
privileged=True,
capabilities=client.V1Capabilities(
add=['SYS_ADMIN'], ),
),
volume_mounts=[
client.V1VolumeMount(
name='fuse',
mount_path='/dev/fuse',
)
],
),
],
volumes=[
client.V1Volume(
name='fuse',
host_path=client.V1HostPathVolumeSource(
path='/dev/fuse', )),
],
), ),
backoff_limit=1,
),
),
)
#
for event in endless_watch(batchV1.list_namespaced_job,
namespace,
label_selector=f"job-name={job['id']}"):
logger.debug(str(event))
event_type = event['type']
event_job = event['object']
if event_type == 'MODIFIED':
if event_job.status.succeeded or event_job.status.failed:
break
#
if not debug:
batchV1.delete_namespaced_job(job['id'], namespace)
# delete associated pod(s)
for event_pod in coreV1.list_namespaced_pod(
namespace, label_selector=f"job-name={job['id']}",
watch=False).items:
coreV1.delete_namespaced_pod(event_pod.metadata.name, namespace)
#
logger.info(f"{job['id']} completed")
def _execute_blight_job(self, ident_fault: str, on: bool,
loc: orchestrator.DeviceLightLoc) -> str:
operation_id = str(hash(loc))
message = ""
# job definition
job_metadata = client.V1ObjectMeta(name=operation_id,
namespace=self.rook_env.namespace,
labels={"ident": operation_id})
pod_metadata = client.V1ObjectMeta(labels={"ident": operation_id})
pod_container = client.V1Container(
name="ceph-lsmcli-command",
security_context=client.V1SecurityContext(privileged=True),
image=self.get_ceph_image(),
command=[
"lsmcli",
],
args=[
'local-disk-%s-led-%s' % (ident_fault, 'on' if on else 'off'),
'--path',
loc.path or loc.dev,
],
volume_mounts=[
client.V1VolumeMount(name="devices", mount_path="/dev"),
client.V1VolumeMount(name="run-udev", mount_path="/run/udev")
])
pod_spec = client.V1PodSpec(
containers=[pod_container],
active_deadline_seconds=30, # Max time to terminate pod
restart_policy="Never",
node_selector={"kubernetes.io/hostname": loc.host},
volumes=[
client.V1Volume(
name="devices",
host_path=client.V1HostPathVolumeSource(path="/dev")),
client.V1Volume(
name="run-udev",
host_path=client.V1HostPathVolumeSource(path="/run/udev"))
])
pod_template = client.V1PodTemplateSpec(metadata=pod_metadata,
spec=pod_spec)
job_spec = client.V1JobSpec(
active_deadline_seconds=60, # Max time to terminate job
ttl_seconds_after_finished=
10, # Alfa. Lifetime after finishing (either Complete or Failed)
backoff_limit=0,
template=pod_template)
job = client.V1Job(api_version="batch/v1",
kind="Job",
metadata=job_metadata,
spec=job_spec)
# delete previous job if it exists
try:
try:
api_response = self.batchV1_api.delete_namespaced_job(
operation_id,
self.rook_env.namespace,
propagation_policy="Background")
except ApiException as e:
if e.status != 404: # No problem if the job does not exist
raise
# wait until the job is not present
deleted = False
retries = 0
while not deleted and retries < 10:
api_response = self.batchV1_api.list_namespaced_job(
self.rook_env.namespace,
label_selector="ident=%s" % operation_id,
timeout_seconds=10)
deleted = not api_response.items
if retries > 5:
sleep(0.1)
retries += 1
if retries == 10 and not deleted:
raise orchestrator.OrchestratorError(
"Light <{}> in <{}:{}> cannot be executed. Cannot delete previous job <{}>"
.format(on, loc.host, loc.path or loc.dev, operation_id))
# create the job
api_response = self.batchV1_api.create_namespaced_job(
self.rook_env.namespace, job)
# get the result
finished = False
while not finished:
api_response = self.batchV1_api.read_namespaced_job(
operation_id, self.rook_env.namespace)
finished = api_response.status.succeeded or api_response.status.failed
if finished:
message = api_response.status.conditions[-1].message
# get the result of the lsmcli command
api_response = self.coreV1_api.list_namespaced_pod(
self.rook_env.namespace,
label_selector="ident=%s" % operation_id,
timeout_seconds=10)
if api_response.items:
pod_name = api_response.items[-1].metadata.name
message = self.coreV1_api.read_namespaced_pod_log(
pod_name, self.rook_env.namespace)
except ApiException as e:
log.exception('K8s API failed. {}'.format(e))
raise
# Finally, delete the job.
# The job uses <ttl_seconds_after_finished>. This makes that the TTL controller delete automatically the job.
# This feature is in Alpha state, so extra explicit delete operations trying to delete the Job has been used strategically
try:
api_response = self.batchV1_api.delete_namespaced_job(
operation_id,
self.rook_env.namespace,
propagation_policy="Background")
except ApiException as e:
if e.status != 404: # No problem if the job does not exist
raise
return message
def test_sanitize_k8s_container_attribute(self):
# test cases for implicit type sanitization(conversion)
op = dsl.ContainerOp(name='echo', image='image', command=['sh', '-c'],
arguments=['echo test | tee /tmp/message.txt'],
file_outputs={'merged': '/tmp/message.txt'})
op.container \
.add_volume_mount(k8s_client.V1VolumeMount(
mount_path='/secret/gcp-credentials',
name='gcp-credentials')) \
.add_env_variable(k8s_client.V1EnvVar(
name=80,
value=80)) \
.add_env_variable(k8s_client.V1EnvVar(
name=80,
value_from=k8s_client.V1EnvVarSource(
config_map_key_ref=k8s_client.V1ConfigMapKeySelector(key=80, name=8080, optional='False'),
field_ref=k8s_client.V1ObjectFieldSelector(api_version=80, field_path=8080),
resource_field_ref=k8s_client.V1ResourceFieldSelector(container_name=80, divisor=8080, resource=8888),
secret_key_ref=k8s_client.V1SecretKeySelector(key=80, name=8080, optional='False')
)
)) \
.add_env_from(k8s_client.V1EnvFromSource(
config_map_ref=k8s_client.V1ConfigMapEnvSource(name=80, optional='True'),
prefix=999
)) \
.add_env_from(k8s_client.V1EnvFromSource(
secret_ref=k8s_client.V1SecretEnvSource(name=80, optional='True'),
prefix=888
)) \
.add_volume_mount(k8s_client.V1VolumeMount(
mount_path=111,
mount_propagation=222,
name=333,
read_only='False',
sub_path=444,
sub_path_expr=555
)) \
.add_volume_devices(k8s_client.V1VolumeDevice(
device_path=111,
name=222
)) \
.add_port(k8s_client.V1ContainerPort(
container_port='8080',
host_ip=111,
host_port='8888',
name=222,
protocol=333
)) \
.set_security_context(k8s_client.V1SecurityContext(
allow_privilege_escalation='True',
capabilities=k8s_client.V1Capabilities(add=[11, 22], drop=[33, 44]),
privileged='False',
proc_mount=111,
read_only_root_filesystem='False',
run_as_group='222',
run_as_non_root='True',
run_as_user='******',
se_linux_options=k8s_client.V1SELinuxOptions(level=11, role=22, type=33, user=44),
windows_options=k8s_client.V1WindowsSecurityContextOptions(
gmsa_credential_spec=11, gmsa_credential_spec_name=22)
)) \
.set_stdin(stdin='False') \
.set_stdin_once(stdin_once='False') \
.set_termination_message_path(termination_message_path=111) \
.set_tty(tty='False') \
.set_readiness_probe(readiness_probe=k8s_client.V1Probe(
_exec=k8s_client.V1ExecAction(command=[11, 22, 33]),
failure_threshold='111',
http_get=k8s_client.V1HTTPGetAction(
host=11,
http_headers=[k8s_client.V1HTTPHeader(name=22, value=33)],
path=44,
port='55',
scheme=66),
initial_delay_seconds='222',
period_seconds='333',
success_threshold='444',
tcp_socket=k8s_client.V1TCPSocketAction(host=555, port='666'),
timeout_seconds='777'
)) \
.set_liveness_probe(liveness_probe=k8s_client.V1Probe(
_exec=k8s_client.V1ExecAction(command=[11, 22, 33]),
failure_threshold='111',
http_get=k8s_client.V1HTTPGetAction(
host=11,
http_headers=[k8s_client.V1HTTPHeader(name=22, value=33)],
path=44,
port='55',
scheme=66),
initial_delay_seconds='222',
period_seconds='333',
success_threshold='444',
tcp_socket=k8s_client.V1TCPSocketAction(host=555, port='666'),
timeout_seconds='777'
)) \
.set_lifecycle(lifecycle=k8s_client.V1Lifecycle(
post_start=k8s_client.V1Handler(
_exec=k8s_client.V1ExecAction(command=[11, 22, 33]),
http_get=k8s_client.V1HTTPGetAction(
host=11,
http_headers=[k8s_client.V1HTTPHeader(name=22, value=33)],
path=44,
port='55',
scheme=66),
tcp_socket=k8s_client.V1TCPSocketAction(host=555, port='666')
),
pre_stop=k8s_client.V1Handler(
_exec=k8s_client.V1ExecAction(command=[11, 22, 33]),
http_get=k8s_client.V1HTTPGetAction(
host=11,
http_headers=[k8s_client.V1HTTPHeader(name=22, value=33)],
path=44,
port='55',
scheme=66),
tcp_socket=k8s_client.V1TCPSocketAction(host=555, port='666')
)
))
sanitize_k8s_object(op.container)
for e in op.container.env:
self.assertIsInstance(e.name, str)
if e.value:
self.assertIsInstance(e.value, str)
if e.value_from:
if e.value_from.config_map_key_ref:
self.assertIsInstance(e.value_from.config_map_key_ref.key, str)
if e.value_from.config_map_key_ref.name:
self.assertIsInstance(e.value_from.config_map_key_ref.name, str)
if e.value_from.config_map_key_ref.optional:
self.assertIsInstance(e.value_from.config_map_key_ref.optional, bool)
if e.value_from.field_ref:
self.assertIsInstance(e.value_from.field_ref.field_path, str)
if e.value_from.field_ref.api_version:
self.assertIsInstance(e.value_from.field_ref.api_version, str)
if e.value_from.resource_field_ref:
self.assertIsInstance(e.value_from.resource_field_ref.resource, str)
if e.value_from.resource_field_ref.container_name:
self.assertIsInstance(e.value_from.resource_field_ref.container_name, str)
if e.value_from.resource_field_ref.divisor:
self.assertIsInstance(e.value_from.resource_field_ref.divisor, str)
if e.value_from.secret_key_ref:
self.assertIsInstance(e.value_from.secret_key_ref.key, str)
if e.value_from.secret_key_ref.name:
self.assertIsInstance(e.value_from.secret_key_ref.name, str)
if e.value_from.secret_key_ref.optional:
self.assertIsInstance(e.value_from.secret_key_ref.optional, bool)
for e in op.container.env_from:
if e.prefix:
self.assertIsInstance(e.prefix, str)
if e.config_map_ref:
if e.config_map_ref.name:
self.assertIsInstance(e.config_map_ref.name, str)
if e.config_map_ref.optional:
self.assertIsInstance(e.config_map_ref.optional, bool)
if e.secret_ref:
if e.secret_ref.name:
self.assertIsInstance(e.secret_ref.name, str)
if e.secret_ref.optional:
self.assertIsInstance(e.secret_ref.optional, bool)
for e in op.container.volume_mounts:
if e.mount_path:
self.assertIsInstance(e.mount_path, str)
if e.mount_propagation:
self.assertIsInstance(e.mount_propagation, str)
if e.name:
self.assertIsInstance(e.name, str)
if e.read_only:
self.assertIsInstance(e.read_only, bool)
if e.sub_path:
self.assertIsInstance(e.sub_path, str)
if e.sub_path_expr:
self.assertIsInstance(e.sub_path_expr, str)
for e in op.container.volume_devices:
if e.device_path:
self.assertIsInstance(e.device_path, str)
if e.name:
self.assertIsInstance(e.name, str)
for e in op.container.ports:
if e.container_port:
self.assertIsInstance(e.container_port, int)
if e.host_ip:
self.assertIsInstance(e.host_ip, str)
if e.host_port:
self.assertIsInstance(e.host_port, int)
if e.name:
self.assertIsInstance(e.name, str)
if e.protocol:
self.assertIsInstance(e.protocol, str)
if op.container.security_context:
e = op.container.security_context
if e.allow_privilege_escalation:
self.assertIsInstance(e.allow_privilege_escalation, bool)
if e.capabilities:
for a in e.capabilities.add:
self.assertIsInstance(a, str)
for d in e.capabilities.drop:
self.assertIsInstance(d, str)
if e.privileged:
self.assertIsInstance(e.privileged, bool)
if e.proc_mount:
self.assertIsInstance(e.proc_mount, str)
if e.read_only_root_filesystem:
self.assertIsInstance(e.read_only_root_filesystem, bool)
if e.run_as_group:
self.assertIsInstance(e.run_as_group, int)
if e.run_as_non_root:
self.assertIsInstance(e.run_as_non_root, bool)
if e.run_as_user:
self.assertIsInstance(e.run_as_user, int)
if e.se_linux_options:
if e.se_linux_options.level:
self.assertIsInstance(e.se_linux_options.level, str)
if e.se_linux_options.role:
self.assertIsInstance(e.se_linux_options.role, str)
if e.se_linux_options.type:
self.assertIsInstance(e.se_linux_options.type, str)
if e.se_linux_options.user:
self.assertIsInstance(e.se_linux_options.user, str)
if e.windows_options:
if e.windows_options.gmsa_credential_spec:
self.assertIsInstance(e.windows_options.gmsa_credential_spec, str)
if e.windows_options.gmsa_credential_spec_name:
self.assertIsInstance(e.windows_options.gmsa_credential_spec_name, str)
if op.container.stdin:
self.assertIsInstance(op.container.stdin, bool)
if op.container.stdin_once:
self.assertIsInstance(op.container.stdin_once, bool)
if op.container.termination_message_path:
self.assertIsInstance(op.container.termination_message_path, str)
if op.container.tty:
self.assertIsInstance(op.container.tty, bool)
for e in [op.container.readiness_probe, op.container.liveness_probe]:
if e:
if e._exec:
for c in e._exec.command:
self.assertIsInstance(c, str)
if e.failure_threshold:
self.assertIsInstance(e.failure_threshold, int)
if e.http_get:
if e.http_get.host:
self.assertIsInstance(e.http_get.host, str)
if e.http_get.http_headers:
for h in e.http_get.http_headers:
if h.name:
self.assertIsInstance(h.name, str)
if h.value:
self.assertIsInstance(h.value, str)
if e.http_get.path:
self.assertIsInstance(e.http_get.path, str)
if e.http_get.port:
self.assertIsInstance(e.http_get.port, (str, int))
if e.http_get.scheme:
self.assertIsInstance(e.http_get.scheme, str)
if e.initial_delay_seconds:
self.assertIsInstance(e.initial_delay_seconds, int)
if e.period_seconds:
self.assertIsInstance(e.period_seconds, int)
if e.success_threshold:
self.assertIsInstance(e.success_threshold, int)
if e.tcp_socket:
if e.tcp_socket.host:
self.assertIsInstance(e.tcp_socket.host, str)
if e.tcp_socket.port:
self.assertIsInstance(e.tcp_socket.port, (str, int))
if e.timeout_seconds:
self.assertIsInstance(e.timeout_seconds, int)
if op.container.lifecycle:
for e in [op.container.lifecycle.post_start, op.container.lifecycle.pre_stop]:
if e:
if e._exec:
for c in e._exec.command:
self.assertIsInstance(c, str)
if e.http_get:
if e.http_get.host:
self.assertIsInstance(e.http_get.host, str)
if e.http_get.http_headers:
for h in e.http_get.http_headers:
if h.name:
self.assertIsInstance(h.name, str)
if h.value:
self.assertIsInstance(h.value, str)
if e.http_get.path:
self.assertIsInstance(e.http_get.path, str)
if e.http_get.port:
self.assertIsInstance(e.http_get.port, (str, int))
if e.http_get.scheme:
self.assertIsInstance(e.http_get.scheme, str)
if e.tcp_socket:
if e.tcp_socket.host:
self.assertIsInstance(e.tcp_socket.host, str)
if e.tcp_socket.port:
self.assertIsInstance(e.tcp_socket.port, (str, int))
# test cases for checking value after sanitization
check_value_op = dsl.ContainerOp(name='echo', image='image', command=['sh', '-c'],
arguments=['echo test | tee /tmp/message.txt'],
file_outputs={'merged': '/tmp/message.txt'})
check_value_op.container \
.add_env_variable(k8s_client.V1EnvVar(
name=80,
value=8080)) \
.set_security_context(k8s_client.V1SecurityContext(
allow_privilege_escalation='true',
capabilities=k8s_client.V1Capabilities(add=[11, 22], drop=[33, 44]),
privileged='false',
proc_mount=111,
read_only_root_filesystem='False',
run_as_group='222',
run_as_non_root='True',
run_as_user='******',
se_linux_options=k8s_client.V1SELinuxOptions(level=11, role=22, type=33, user=44),
windows_options=k8s_client.V1WindowsSecurityContextOptions(
gmsa_credential_spec=11, gmsa_credential_spec_name=22)
))
sanitize_k8s_object(check_value_op.container)
self.assertEqual(check_value_op.container.env[0].name, '80')
self.assertEqual(check_value_op.container.env[0].value, '8080')
self.assertEqual(check_value_op.container.security_context.allow_privilege_escalation, True)
self.assertEqual(check_value_op.container.security_context.capabilities.add[0], '11')
self.assertEqual(check_value_op.container.security_context.capabilities.add[1], '22')
self.assertEqual(check_value_op.container.security_context.capabilities.drop[0], '33')
self.assertEqual(check_value_op.container.security_context.capabilities.drop[1], '44')
self.assertEqual(check_value_op.container.security_context.privileged, False)
self.assertEqual(check_value_op.container.security_context.proc_mount, '111')
self.assertEqual(check_value_op.container.security_context.read_only_root_filesystem, False)
self.assertEqual(check_value_op.container.security_context.run_as_group, 222)
self.assertEqual(check_value_op.container.security_context.run_as_non_root, True)
self.assertEqual(check_value_op.container.security_context.run_as_user, 333)
self.assertEqual(check_value_op.container.security_context.se_linux_options.level, '11')
self.assertEqual(check_value_op.container.security_context.se_linux_options.role, '22')
self.assertEqual(check_value_op.container.security_context.se_linux_options.type, '33')
self.assertEqual(check_value_op.container.security_context.se_linux_options.user, '44')
self.assertEqual(check_value_op.container.security_context.windows_options.gmsa_credential_spec, '11')
self.assertEqual(check_value_op.container.security_context.windows_options.gmsa_credential_spec_name, '22')
# test cases for exception
with self.assertRaises(ValueError, msg='Invalid boolean string 2. Should be boolean.'):
exception_op = dsl.ContainerOp(name='echo', image='image')
exception_op.container \
.set_security_context(k8s_client.V1SecurityContext(
allow_privilege_escalation=1
))
sanitize_k8s_object(exception_op.container)
with self.assertRaises(ValueError, msg='Invalid boolean string Test. Should be "true" or "false".'):
exception_op = dsl.ContainerOp(name='echo', image='image')
exception_op.container \
.set_security_context(k8s_client.V1SecurityContext(
allow_privilege_escalation='Test'
))
sanitize_k8s_object(exception_op.container)
with self.assertRaises(ValueError, msg='Invalid test. Should be integer.'):
exception_op = dsl.ContainerOp(name='echo', image='image')
exception_op.container \
.set_security_context(k8s_client.V1SecurityContext(
run_as_group='test',
))
sanitize_k8s_object(exception_op.container)
def createStatefulSet(cls, cluster_object: V1MongoClusterConfiguration) -> client.V1beta1StatefulSet:
"""
Creates a the stateful set configuration for the given cluster.
:param cluster_object: The cluster object from the YAML file.
:return: The stateful set object.
"""
# Parse cluster data object.
name = cluster_object.metadata.name
namespace = cluster_object.metadata.namespace
replicas = cluster_object.spec.mongodb.replicas
storage_mount_path = cluster_object.spec.mongodb.host_path or cls.DEFAULT_STORAGE_MOUNT_PATH
host_path = cluster_object.spec.mongodb.host_path
cpu_limit = cluster_object.spec.mongodb.cpu_limit or cls.DEFAULT_CPU_LIMIT
memory_limit = cluster_object.spec.mongodb.memory_limit or cls.DEFAULT_MEMORY_LIMIT
run_as_user = cluster_object.spec.mongodb.run_as_user or cls.DEFAULT_RUN_AS_USER
service_account = cluster_object.spec.mongodb.service_account or cls.DEFAULT_SERVICE_ACCOUNT
wired_tiger_cache_size = cluster_object.spec.mongodb.wired_tiger_cache_size or cls.DEFAULT_CACHE_SIZE
secret_name = cls.ADMIN_SECRET_NAME_FORMAT.format(name)
# create container
mongo_container = client.V1Container(
name=name,
env=[client.V1EnvVar(
name="POD_IP",
value_from=client.V1EnvVarSource(
field_ref = client.V1ObjectFieldSelector(
api_version = "v1",
field_path = "status.podIP"
)
)
),
client.V1EnvVar(
name="MONGODB_PASSWORD",
value_from=client.V1EnvVarSource(
secret_key_ref=client.V1SecretKeySelector(
key="database-password",
name=secret_name
)
)
),
client.V1EnvVar(
name="MONGODB_USER",
value_from=client.V1EnvVarSource(
secret_key_ref=client.V1SecretKeySelector(
key="database-user",
name=secret_name
)
)
),
client.V1EnvVar(
name="MONGODB_DATABASE",
value_from=client.V1EnvVarSource(
secret_key_ref=client.V1SecretKeySelector(
key="database-name",
name=secret_name
)
)
),
client.V1EnvVar(
name="MONGODB_ADMIN_PASSWORD",
value_from=client.V1EnvVarSource(
secret_key_ref=client.V1SecretKeySelector(
key="database-admin-password",
name=secret_name
)
)
),
client.V1EnvVar(
name="WIREDTIGER_CACHE_SIZE",
value=wired_tiger_cache_size
),
client.V1EnvVar(
name="MONGODB_REPLICA_NAME",
value=name
),
client.V1EnvVar(
name="MONGODB_SERVICE_NAME",
value="svc-" + name + "-internal"
),
client.V1EnvVar(
name="MONGODB_KEYFILE_VALUE",
value="supersecretkeyfile123"
)],
liveness_probe=client.V1Probe(failure_threshold=3,
initial_delay_seconds=30,
period_seconds=30,
success_threshold=1,
tcp_socket=client.V1TCPSocketAction(port=cls.MONGO_PORT),
timeout_seconds=1
),
command=cls.MONGO_COMMAND.split(),
image=cls.MONGO_IMAGE,
image_pull_policy="Always",
ports=[client.V1ContainerPort(
name="mongodb",
container_port=cls.MONGO_PORT,
protocol="TCP"
)],
readiness_probe=client.V1Probe(_exec=client.V1ExecAction(command=["/bin/sh", "-i", "-c", "mongo 127.0.0.1:27017/$MONGODB_DATABASE -u $MONGODB_USER -p $MONGODB_PASSWORD --eval=\"quit()\""]),
failure_threshold=3,
initial_delay_seconds=10,
period_seconds=10,
success_threshold=1,
timeout_seconds=1
),
security_context=client.V1SecurityContext(
run_as_user=int(run_as_user),
se_linux_options=client.V1SELinuxOptions(
level="s0",
type="spc_t"
)
),
termination_message_path="/dev/termination-log",
volume_mounts=[client.V1VolumeMount(
name="mongo-data",
read_only=False,
mount_path=storage_mount_path
)],
resources=client.V1ResourceRequirements(
limits={"cpu": cpu_limit, "memory": memory_limit},
requests={"cpu": cpu_limit, "memory": memory_limit}
)
)
#create affinity rules
affinity = client.V1Affinity(
pod_anti_affinity=client.V1PodAntiAffinity(
required_during_scheduling_ignored_during_execution=[
client.V1PodAffinityTerm(label_selector=client.V1LabelSelector(
match_expressions=[client.V1LabelSelectorRequirement(
key="app",
operator="In",
values=[name]
)]
),
topology_key="kubernetes.io/hostname")
]
)
)
volumes = [client.V1Volume(
name="mongo-data",
host_path=client.V1HostPathVolumeSource(path=host_path)
)]
# Create stateful set.
return client.V1beta1StatefulSet(
metadata = client.V1ObjectMeta(annotations={"service.alpha.kubernetes.io/tolerate-unready-endpoints": "true"},
name=name,
namespace=namespace,
labels=cls.createDefaultLabels(name)),
spec = client.V1beta1StatefulSetSpec(
replicas = replicas,
service_name = "svc-" + name + "-internal",
template = client.V1PodTemplateSpec(
metadata = client.V1ObjectMeta(labels=cls.createDefaultLabels(name)),
spec = client.V1PodSpec(affinity = affinity,
containers=[mongo_container],
node_selector={"compute":"mongodb"},
service_account=service_account,
#restart_policy="Never",
volumes=volumes
)
),
),
)
def add_run_with_root_permissions(self):
"""Run interactive session with root."""
security_context = client.V1SecurityContext(run_as_user=0)
self.kubernetes_objects["deployment"].spec.template.spec.containers[
0].security_context = security_context
def deployment_object(self, instance_uuid, cnf_yaml, service_uuid, vim_uuid):
"""
CNF modeling method. This build a deployment object in kubernetes
instance_uuid: k8s deployment name
cnf_yaml: CNF Descriptor in yaml format
"""
t0 = time.time()
LOG.debug("CNFD: {}".format(cnf_yaml))
container_list = []
pod_volume_list = []
deployment_k8s = None
privileged = False
node_selector = {}
host_network = False
if "cloudnative_deployment_units" in cnf_yaml:
cdu = cnf_yaml.get('cloudnative_deployment_units')
for cdu_obj in cdu:
env_vars = env_from = cpu = memory = huge_pages = gpu = sr_iov = resources = volume_mounts = None
port_list = []
environment = []
capabilities_list = []
cdu_id = cdu_obj.get('id')
image = cdu_obj.get('image')
cdu_conex = cdu_obj.get('connection_points')
container_name = cdu_id
config_map_id = cdu_id
if cdu_obj.get('parameters'):
env_vars = cdu_obj['parameters'].get('env')
volume_mounts = cdu_obj['parameters'].get('volume_mounts')
capabilities_list = cdu_obj['parameters'].get('capabilities')
if cdu_obj['parameters'].get('privileged'):
privileged = cdu_obj['parameters'].get('privileged')
if cdu_obj['parameters'].get('host_network'):
privileged = cdu_obj['parameters']['host_network']
if not isinstance(capabilities_list, list):
capabilities_list = []
if cdu_obj['parameters'].get('node_selector'):
node_selector = cdu_obj['parameters']['node_selector']
if cdu_obj.get('resource_requirements'):
gpu = cdu_obj['resource_requirements'].get('gpu')
cpu = cdu_obj['resource_requirements'].get('cpu')
memory = cdu_obj['resource_requirements'].get('memory')
sr_iov = cdu_obj['resource_requirements'].get('sr-iov')
huge_pages = cdu_obj['resource_requirements'].get('huge-pages')
if cdu_conex:
for po in cdu_conex:
port = po.get('port')
port_name = po.get('id')
protocol = "TCP"
if po.get("protocol"):
protocol = po["protocol"]
port_list.append(client.V1ContainerPort(container_port = port, name = port_name, protocol=protocol))
limits = {}
requests = {}
if gpu:
LOG.debug("Features requested: {}".format(gpu))
# gpu_type can be amd or nvidia
for gpu_type, amount in gpu.items():
limits["{}.com/gpu".format(gpu_type)] = amount
if cpu:
# TODO
pass
if memory:
# TODO
pass
if sr_iov:
# TODO
pass
if huge_pages:
# TODO
pass
resources = client.V1ResourceRequirements(limits=limits, requests=requests)
# Environment variables from descriptor
if env_vars:
LOG.debug("Configmap: {}".format(config_map_id))
KubernetesWrapperEngine.create_configmap(self, config_map_id, instance_uuid, env_vars, service_uuid,
vim_uuid, namespace = "default")
else:
env_vars = {"sonata": "rules"}
LOG.debug("Configmap: {}".format(config_map_id))
KubernetesWrapperEngine.create_configmap(self, config_map_id, instance_uuid, env_vars, service_uuid,
vim_uuid, namespace = "default")
env_from = client.V1EnvFromSource(config_map_ref = client.V1ConfigMapEnvSource(name = config_map_id,
optional = False))
# Default static environment variables
environment.append(client.V1EnvVar(name="instance_uuid", value=instance_uuid))
environment.append(client.V1EnvVar(name="service_uuid", value=service_uuid))
environment.append(client.V1EnvVar(name="container_name", value=container_name))
environment.append(client.V1EnvVar(name="vendor", value=KubernetesWrapperEngine.normalize(self, cnf_yaml.get('vendor'))))
environment.append(client.V1EnvVar(name="name", value=KubernetesWrapperEngine.normalize(self, cnf_yaml.get('name'))))
environment.append(client.V1EnvVar(name="version", value=KubernetesWrapperEngine.normalize(self, cnf_yaml.get('version'))))
image_pull_policy = KubernetesWrapperEngine.check_connection(self)
# Volume mounts
container_volume_mount_list = []
if volume_mounts:
LOG.debug("volume mounts: {}".format(volume_mounts))
# Create the specification of volumes
for volume_mounts_item in volume_mounts:
if volume_mounts_item.get('id') and volume_mounts_item.get('location'):
if volume_mounts_item.get('persistent'):
volumes = client.V1Volume(name=volume_mounts_item['id'],
host_path=client.V1HostPathVolumeSource(path='/mnt/data', type='DirectoryOrCreate' ))
else:
volumes = client.V1Volume(name=volume_mounts_item['id'],
empty_dir=client.V1EmptyDirVolumeSource(medium='' ))
if volumes not in pod_volume_list:
pod_volume_list.append(volumes)
container_volume_mount = client.V1VolumeMount(name=volume_mounts_item['id'], mount_path=volume_mounts_item['location'] )
container_volume_mount_list.append(container_volume_mount)
LOG.debug("Security capabilities: {}, privileged: {} applied to {}".format(capabilities_list, privileged, container_name))
sec_context = client.V1SecurityContext(privileged=privileged, capabilities=client.V1Capabilities(add=capabilities_list))
# Configureate Pod template container
container = client.V1Container(
env = environment,
name = container_name,
resources = resources,
image = image,
image_pull_policy = image_pull_policy,
ports = port_list,
env_from = [env_from],
volume_mounts = container_volume_mount_list,
security_context=sec_context)
container_list.append(container)
else:
return deployment_k8s
# Create and configurate a spec section
deployment_label = ("{}-{}-{}-{}".format(cnf_yaml.get("vendor"), cnf_yaml.get("name"), cnf_yaml.get("version"),
instance_uuid.split("-")[0])).replace(".", "-")
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={'deployment': deployment_label,
'instance_uuid': cnf_yaml['instance_uuid'],
'service_uuid': service_uuid,
'sp': "sonata",
'descriptor_uuid': cnf_yaml['uuid']}
),
spec=client.V1PodSpec(containers=container_list, volumes=pod_volume_list, node_selector=node_selector,
host_network=host_network))
selector=client.V1LabelSelector(match_labels={'deployment': deployment_label,
'instance_uuid': cnf_yaml['instance_uuid'],
'service_uuid': service_uuid,
'sp': "sonata",
'descriptor_uuid': cnf_yaml['uuid']}
)
# Create the specification of deployment
spec = client.V1DeploymentSpec(
replicas=1,
template=template,
selector=selector)
# Instantiate the deployment object
deployment_k8s = client.V1Deployment(
api_version="apps/v1",
kind="Deployment",
metadata=client.V1ObjectMeta(name=deployment_label,
labels={'deployment': deployment_label,
'instance_uuid': cnf_yaml['instance_uuid'],
'service_uuid': service_uuid,
'sp': "sonata",
'descriptor_uuid': cnf_yaml['uuid']}
),
spec=spec)
LOG.info("Deployment object: {}".format(deployment_k8s))
LOG.info("CreatingDeploymentObject-time: {} ms".format(int((time.time() - t0)* 1000)))
return deployment_k8s
def _create_pod(self,
image,
pod_name,
job_name,
port=80,
cmd_string=None,
volumes=[]):
""" Create a kubernetes pod for the job.
Args:
- image (string) : Docker image to launch
- pod_name (string) : Name of the pod
- job_name (string) : App label
KWargs:
- port (integer) : Container port
Returns:
- None
"""
security_context = None
if self.user_id and self.group_id:
security_context = client.V1SecurityContext(
run_as_group=self.group_id,
run_as_user=self.user_id,
run_as_non_root=self.run_as_non_root)
# Create the enviornment variables and command to initiate IPP
environment_vars = client.V1EnvVar(name="TEST", value="SOME DATA")
launch_args = ["-c", "{0}".format(cmd_string)]
volume_mounts = []
# Create mount paths for the volumes
for volume in volumes:
volume_mounts.append(
client.V1VolumeMount(mount_path=volume[1], name=volume[0]))
resources = client.V1ResourceRequirements(limits={
'cpu': str(self.max_cpu),
'memory': self.max_mem
},
requests={
'cpu':
str(self.init_cpu),
'memory': self.init_mem
})
# Configure Pod template container
container = client.V1Container(
name=pod_name,
image=image,
resources=resources,
ports=[client.V1ContainerPort(container_port=port)],
volume_mounts=volume_mounts,
command=['/bin/bash'],
args=launch_args,
env=[environment_vars],
security_context=security_context)
# Create a secret to enable pulling images from secure repositories
secret = None
if self.secret:
secret = client.V1LocalObjectReference(name=self.secret)
# Create list of volumes from (pvc, mount) tuples
volume_defs = []
for volume in volumes:
volume_defs.append(
client.V1Volume(
name=volume[0],
persistent_volume_claim=client.
V1PersistentVolumeClaimVolumeSource(claim_name=volume[0])))
metadata = client.V1ObjectMeta(name=pod_name, labels={"app": job_name})
spec = client.V1PodSpec(containers=[container],
image_pull_secrets=[secret],
volumes=volume_defs)
pod = client.V1Pod(spec=spec, metadata=metadata)
api_response = self.kube_client.create_namespaced_pod(
namespace=self.namespace, body=pod)
logger.debug("Pod created. status='{0}'".format(
str(api_response.status)))
def deploy_k8s_nfs(self) -> bool:
"""
Deploy the NFS server in the Kubernetes orchestrator.
"""
from kubernetes import client as k8sclient
name = "nfs-server-{}".format(uuid.uuid4())
container = k8sclient.V1Container(
name=name,
image="k8s.gcr.io/volume-nfs:0.8",
ports=[
k8sclient.V1ContainerPort(name="nfs",
container_port=2049,
protocol="TCP"),
k8sclient.V1ContainerPort(name="rpcbind", container_port=111),
k8sclient.V1ContainerPort(name="mountd", container_port=20048),
],
volume_mounts=[
k8sclient.V1VolumeMount(name='nfs-host-path',
mount_path='/exports')
],
security_context=k8sclient.V1SecurityContext(privileged=True))
template = k8sclient.V1PodTemplateSpec(
metadata=k8sclient.V1ObjectMeta(labels={'app': name}),
spec=k8sclient.V1PodSpec(
containers=[container],
volumes=[
k8sclient.V1Volume(
name="nfs-host-path",
host_path=k8sclient.V1HostPathVolumeSource(
path='/tmp/nfsexports-{}'.format(uuid.uuid4())))
]))
deployment_spec = k8sclient.V1DeploymentSpec(
replicas=1,
template=template,
selector=k8sclient.V1LabelSelector(match_labels={'app': name}))
deployment = k8sclient.V1Deployment(api_version='apps/v1',
kind='Deployment',
metadata=k8sclient.V1ObjectMeta(
name=name,
labels={'app': name}),
spec=deployment_spec)
k8s_apps_v1_api_client = k8sclient.AppsV1Api()
try:
k8s_apps_v1_api_client.create_namespaced_deployment(
self.params.namespace, deployment)
self.params.name = name
except k8sclient.rest.ApiException as e:
screen.print("Got exception: %s\n while creating nfs-server", e)
return False
k8s_core_v1_api_client = k8sclient.CoreV1Api()
svc_name = "nfs-service-{}".format(uuid.uuid4())
service = k8sclient.V1Service(
api_version='v1',
kind='Service',
metadata=k8sclient.V1ObjectMeta(name=svc_name),
spec=k8sclient.V1ServiceSpec(selector={'app': self.params.name},
ports=[
k8sclient.V1ServicePort(
protocol='TCP',
port=2049,
target_port=2049)
]))
try:
svc_response = k8s_core_v1_api_client.create_namespaced_service(
self.params.namespace, service)
self.params.svc_name = svc_name
self.params.server = svc_response.spec.cluster_ip
except k8sclient.rest.ApiException as e:
screen.print(
"Got exception: %s\n while creating a service for nfs-server",
e)
return False
return True
def auto_generated_pipeline(booltest='True', d1='5', d2='6', strtest='test'):
pvolumes_dict = OrderedDict()
volume_step_names = []
volume_name_parameters = []
marshal_vop = dsl.VolumeOp(name="kale-marshal-volume",
resource_name="kale-marshal-pvc",
modes=dsl.VOLUME_MODE_RWM,
size="1Gi")
volume_step_names.append(marshal_vop.name)
volume_name_parameters.append(marshal_vop.outputs["name"].full_name)
pvolumes_dict['/marshal'] = marshal_vop.volume
volume_step_names.sort()
volume_name_parameters.sort()
create_matrix_task = create_matrix_op(d1, d2)\
.add_pvolumes(pvolumes_dict)\
.after()
step_limits = {'nvidia.com/gpu': '2'}
for k, v in step_limits.items():
create_matrix_task.container.add_resource_limit(k, v)
create_matrix_task.container.working_dir = "/kale"
create_matrix_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update({'mlpipeline-metrics': '/mlpipeline-metrics.json'})
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'create_matrix': '/create_matrix.html'})
create_matrix_task.output_artifact_paths.update(output_artifacts)
create_matrix_task.add_pod_label("pipelines.kubeflow.org/metadata_written",
"true")
dep_names = create_matrix_task.dependent_names + volume_step_names
create_matrix_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(dep_names))
if volume_name_parameters:
create_matrix_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(volume_name_parameters))
sum_matrix_task = sum_matrix_op()\
.add_pvolumes(pvolumes_dict)\
.after(create_matrix_task)
sum_matrix_task.container.working_dir = "/kale"
sum_matrix_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update({'mlpipeline-metrics': '/mlpipeline-metrics.json'})
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'sum_matrix': '/sum_matrix.html'})
sum_matrix_task.output_artifact_paths.update(output_artifacts)
sum_matrix_task.add_pod_label("pipelines.kubeflow.org/metadata_written",
"true")
dep_names = sum_matrix_task.dependent_names + volume_step_names
sum_matrix_task.add_pod_annotation("kubeflow-kale.org/dependent-templates",
json.dumps(dep_names))
if volume_name_parameters:
sum_matrix_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(volume_name_parameters))
def _build_definition(machine, config_map):
volume_mounts = []
if config_map:
# Define volume mounts for hostlab if a ConfigMap is defined.
volume_mounts.append(
client.V1VolumeMount(name="hostlab",
mount_path="/tmp/kathara"))
if Setting.get_instance().host_shared:
volume_mounts.append(
client.V1VolumeMount(name="shared", mount_path="/shared"))
# Machine must be executed in privileged mode to run sysctls.
security_context = client.V1SecurityContext(privileged=True)
ports_info = machine.get_ports()
container_ports = None
if ports_info:
container_ports = []
for (host_port, protocol), guest_port in ports_info.items():
container_ports.append(
client.V1ContainerPort(name=str(uuid.uuid4()).replace(
'-', '')[0:15],
container_port=guest_port,
host_port=host_port,
protocol=protocol.upper()))
resources = None
memory = machine.get_mem()
cpus = machine.get_cpu(multiplier=1000)
if memory or cpus:
limits = dict()
if memory:
limits["memory"] = memory.upper()
if cpus:
limits["cpu"] = "%dm" % cpus
resources = client.V1ResourceRequirements(limits=limits)
# postStart lifecycle hook is launched asynchronously by k8s master when the main container is Ready
# On Ready state, the pod has volumes and network interfaces up, so this hook is used
# to execute custom commands coming from .startup file and "exec" option
# Build the final startup commands string
sysctl_commands = "; ".join([
"sysctl -w -q %s=%d" % item
for item in machine.meta["sysctls"].items()
])
startup_commands_string = "; ".join(STARTUP_COMMANDS) \
.format(machine_name=machine.name,
sysctl_commands=sysctl_commands,
machine_commands="; ".join(machine.startup_commands)
)
post_start = client.V1Handler(_exec=client.V1ExecAction(command=[
Setting.get_instance().device_shell, "-c", startup_commands_string
]))
lifecycle = client.V1Lifecycle(post_start=post_start)
env = [
client.V1EnvVar(
"_MEGALOS_SHELL", machine.meta["shell"] if "shell"
in machine.meta else Setting.get_instance().device_shell)
]
container_definition = client.V1Container(
name=machine.meta['real_name'],
image=machine.get_image(),
lifecycle=lifecycle,
stdin=True,
tty=True,
image_pull_policy=Setting.get_instance().image_pull_policy,
ports=container_ports,
resources=resources,
volume_mounts=volume_mounts,
security_context=security_context,
env=env)
pod_annotations = {}
network_interfaces = []
for (idx, machine_link) in machine.interfaces.items():
network_interfaces.append({
"name":
machine_link.api_object["metadata"]["name"],
"namespace":
machine.lab.folder_hash,
"interface":
"net%d" % idx
})
pod_annotations["k8s.v1.cni.cncf.io/networks"] = json.dumps(
network_interfaces)
# Create labels (so Deployment can match them)
pod_labels = {"name": machine.name, "app": "kathara"}
pod_metadata = client.V1ObjectMeta(deletion_grace_period_seconds=0,
annotations=pod_annotations,
labels=pod_labels)
# Add fake DNS just to override k8s one
dns_config = client.V1PodDNSConfig(nameservers=["127.0.0.1"])
volumes = []
if config_map:
# Hostlab is the lab base64 encoded .tar.gz of the machine files, deployed as a ConfigMap in the cluster
# The base64 file is mounted into /tmp and it's extracted by the postStart hook
volumes.append(
client.V1Volume(name="hostlab",
config_map=client.V1ConfigMapVolumeSource(
name=config_map.metadata.name)))
# Container /shared mounts in /home/shared folder
if Setting.get_instance().host_shared:
volumes.append(
client.V1Volume(name="shared",
host_path=client.V1HostPathVolumeSource(
path='/home/shared',
type='DirectoryOrCreate')))
pod_spec = client.V1PodSpec(containers=[container_definition],
hostname=machine.meta['real_name'],
dns_policy="None",
dns_config=dns_config,
volumes=volumes)
pod_template = client.V1PodTemplateSpec(metadata=pod_metadata,
spec=pod_spec)
selector_rules = client.V1LabelSelector(match_labels=pod_labels)
deployment_spec = client.V1DeploymentSpec(replicas=1,
template=pod_template,
selector=selector_rules)
deployment_metadata = client.V1ObjectMeta(
name=machine.meta['real_name'], labels=pod_labels)
return client.V1Deployment(api_version="apps/v1",
kind="Deployment",
metadata=deployment_metadata,
spec=deployment_spec)
def _create_deployment_object(self):
"""Creates the deployment object for the grader service using environment variables
Returns:
V1Deployment: a valid kubernetes deployment object
"""
# Configureate Pod template container
# Volumes to mount as subPaths of PV
sub_path_grader_home = str(self.course_dir.parent).strip("/")
sub_path_exchange = str(self.exchange_dir.relative_to(EXCHANGE_MNT_ROOT))
# define the container to launch
container = client.V1Container(
name="grader-notebook",
image=GRADER_IMAGE_NAME,
image_pull_policy=GRADER_IMAGE_PULL_POLICY,
command=["start-notebook.sh", f"--group=formgrade-{self.course_id}"],
ports=[client.V1ContainerPort(container_port=8888)],
working_dir=f"/home/{self.grader_name}",
resources=client.V1ResourceRequirements(
requests={
"cpu": GRADER_REQUESTS_CPU,
"memory": GRADER_REQUESTS_MEM,
},
limits={
"cpu": GRADER_LIMITS_CPU,
"memory": GRADER_LIMITS_MEM,
},
),
security_context=client.V1SecurityContext(allow_privilege_escalation=False),
env=[
client.V1EnvVar(name="JUPYTERHUB_SERVICE_NAME", value=self.course_id),
client.V1EnvVar(
name="JUPYTERHUB_SERVICE_URL",
value=f"http://{self.course_id}.{NAMESPACE}.svc.cluster.local:8888",
),
client.V1EnvVar(name="JUPYTERHUB_API_TOKEN", value=self.grader_token),
# we're using the K8s Service name 'hub' (defined in the jhub helm chart)
# to connect from our grader-notebooks
client.V1EnvVar(name="JUPYTERHUB_API_URL", value=JUPYTERHUB_API_URL),
client.V1EnvVar(name="JUPYTERHUB_BASE_URL", value=JUPYTERHUB_BASE_URL),
client.V1EnvVar(
name="JUPYTERHUB_SERVICE_PREFIX",
value=f"/services/{self.course_id}/",
),
client.V1EnvVar(
name="JUPYTERHUB_CLIENT_ID", value=f"service-{self.course_id}"
),
client.V1EnvVar(name="JUPYTERHUB_USER", value=self.grader_name),
client.V1EnvVar(name="NAMESPACE", value=str(NAMESPACE)),
client.V1EnvVar(name="NB_UID", value=str(NB_UID)),
client.V1EnvVar(name="NB_GID", value=str(NB_GID)),
client.V1EnvVar(name="NB_USER", value=self.grader_name),
],
volume_mounts=[
client.V1VolumeMount(
mount_path=f"/home/{self.grader_name}",
name=GRADER_PVC,
sub_path=sub_path_grader_home,
),
client.V1VolumeMount(
mount_path="/srv/nbgrader/exchange",
name=GRADER_EXCHANGE_SHARED_PVC,
sub_path=sub_path_exchange,
),
],
)
# Create and configure a spec section
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(
labels={"component": self.grader_name, "app": "illumidesk"}
),
spec=client.V1PodSpec(
containers=[container],
security_context=client.V1PodSecurityContext(run_as_user=0),
volumes=[
client.V1Volume(
name=GRADER_PVC,
persistent_volume_claim=client.V1PersistentVolumeClaimVolumeSource(
claim_name=GRADER_PVC
),
),
client.V1Volume(
name=GRADER_EXCHANGE_SHARED_PVC,
persistent_volume_claim=client.V1PersistentVolumeClaimVolumeSource(
claim_name=GRADER_EXCHANGE_SHARED_PVC
),
),
],
),
)
# Create the specification of deployment
spec = client.V1DeploymentSpec(
replicas=1,
template=template,
selector={"matchLabels": {"component": self.grader_name}},
)
# Instantiate the deployment object
deployment = client.V1Deployment(
api_version="apps/v1",
kind="Deployment",
metadata=client.V1ObjectMeta(name=self.grader_name),
spec=spec,
)
return deployment
def template(context):
labels = {"app": context["name"]}
# Create volume mount lists and populate them if they are declared in the command
pod_spec_volume_mounts = []
pod_spec_volumes = []
if "volumeName" in context:
# Create volume mounts
if "subPath" in context:
pod_spec_volume_mounts = [
client.V1VolumeMount(name=context["volumeName"],
mount_path=context["mountPath"],
sub_path=context["subPath"])
]
else:
pod_spec_volume_mounts = [
client.V1VolumeMount(name=context["volumeName"],
mount_path=context["mountPath"])
]
# Create volumes
pod_spec_volumes = [
client.V1Volume(name=context["volumeName"],
nfs=client.V1NFSVolumeSource(
path=context["nfsPath"],
server=context["nfsServer"]))
]
pod_init_containers = []
if "vpn_init" in context:
for key, val in dict(
zip(context["vpn_init"].split(),
context["vpn_cmds"].split(';'))).items():
pod_init_containers.append(
client.V1Container(name=context["name"] + "-" + key,
image=context["image"],
command=val.split(),
volume_mounts=pod_spec_volume_mounts))
# Create Environment variable list and populate if it is declared in the command
env_list = []
if "env" in context:
envs = dict(zip(context["env"].split(), context["vals"].split()))
for key, val in envs.items():
env_var = client.V1EnvVar(name=key, value=val)
env_list.append(env_var)
args = []
if "attack_args" in context:
args.append(context["attack_args"].split())
# Define the template specification
template_spec = client.V1PodSpec(containers=[
client.V1Container(
name=context["name"],
image=context["image"],
env=env_list,
args=args,
security_context=client.V1SecurityContext(privileged=True),
volume_mounts=pod_spec_volume_mounts)
],
init_containers=pod_init_containers,
volumes=pod_spec_volumes)
# Create dictionary for network attachment definition
# This is required in a dictionary format for the template.metadata.annotations field
net_dict = {}
net_dict[context["netkey"]] = ', '.join(context["netval"].split())
# Return deployment specification and tie together all the above components
return client.V1Deployment(
api_version="extensions/v1beta1",
kind="Deployment",
metadata=client.V1ObjectMeta(name=context["name"]),
spec=client.V1DeploymentSpec(
replicas=int(context["replicas"]),
selector=client.V1LabelSelector(match_labels=labels),
template=client.V1PodTemplateSpec(metadata=client.V1ObjectMeta(
labels=labels, annotations=net_dict),
spec=template_spec),
),
)
def auto_generated_pipeline():
pvolumes_dict = OrderedDict()
marshal_vop = dsl.VolumeOp(name="kale_marshal_volume",
resource_name="kale-marshal-pvc",
modes=dsl.VOLUME_MODE_RWM,
size="1Gi")
pvolumes_dict['/marshal'] = marshal_vop.volume
loaddata_task = loaddata_op()\
.add_pvolumes(pvolumes_dict)\
.after()
loaddata_task.container.working_dir = "/kale"
loaddata_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'loaddata': '/loaddata.html'})
loaddata_task.output_artifact_paths.update(output_artifacts)
datapreprocessing_task = datapreprocessing_op()\
.add_pvolumes(pvolumes_dict)\
.after(loaddata_task)
datapreprocessing_task.container.working_dir = "/kale"
datapreprocessing_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'datapreprocessing': '/datapreprocessing.html'})
datapreprocessing_task.output_artifact_paths.update(output_artifacts)
featureengineering_task = featureengineering_op()\
.add_pvolumes(pvolumes_dict)\
.after(datapreprocessing_task)
featureengineering_task.container.working_dir = "/kale"
featureengineering_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'featureengineering': '/featureengineering.html'})
featureengineering_task.output_artifact_paths.update(output_artifacts)
decisiontree_task = decisiontree_op()\
.add_pvolumes(pvolumes_dict)\
.after(featureengineering_task)
decisiontree_task.container.working_dir = "/kale"
decisiontree_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'decisiontree': '/decisiontree.html'})
decisiontree_task.output_artifact_paths.update(output_artifacts)
svm_task = svm_op()\
.add_pvolumes(pvolumes_dict)\
.after(featureengineering_task)
svm_task.container.working_dir = "/kale"
svm_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'svm': '/svm.html'})
svm_task.output_artifact_paths.update(output_artifacts)
naivebayes_task = naivebayes_op()\
.add_pvolumes(pvolumes_dict)\
.after(featureengineering_task)
naivebayes_task.container.working_dir = "/kale"
naivebayes_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'naivebayes': '/naivebayes.html'})
naivebayes_task.output_artifact_paths.update(output_artifacts)
logisticregression_task = logisticregression_op()\
.add_pvolumes(pvolumes_dict)\
.after(featureengineering_task)
logisticregression_task.container.working_dir = "/kale"
logisticregression_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'logisticregression': '/logisticregression.html'})
logisticregression_task.output_artifact_paths.update(output_artifacts)
randomforest_task = randomforest_op()\
.add_pvolumes(pvolumes_dict)\
.after(featureengineering_task)
randomforest_task.container.working_dir = "/kale"
randomforest_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'randomforest': '/randomforest.html'})
randomforest_task.output_artifact_paths.update(output_artifacts)
results_task = results_op()\
.add_pvolumes(pvolumes_dict)\
.after(randomforest_task, logisticregression_task, naivebayes_task, svm_task, decisiontree_task)
results_task.container.working_dir = "/kale"
results_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'results': '/results.html'})
results_task.output_artifact_paths.update(output_artifacts)
def auto_generated_pipeline():
_kale_pvolumes_dict = OrderedDict()
_kale_volume_step_names = []
_kale_volume_name_parameters = []
_kale_marshal_vop = _kfp_dsl.VolumeOp(
name="kale-marshal-volume",
resource_name="kale-marshal-pvc",
modes=['ReadWriteMany'],
size="1Gi"
)
_kale_volume_step_names.append(_kale_marshal_vop.name)
_kale_volume_name_parameters.append(
_kale_marshal_vop.outputs["name"].full_name)
_kale_pvolumes_dict['/marshal'] = _kale_marshal_vop.volume
_kale_volume_step_names.sort()
_kale_volume_name_parameters.sort()
_kale_step1_task = _kale_step1_op()\
.add_pvolumes(_kale_pvolumes_dict)\
.after()
_kale_step_labels = {'common-label': 'true'}
for _kale_k, _kale_v in _kale_step_labels.items():
_kale_step1_task.add_pod_label(_kale_k, _kale_v)
_kale_step_limits = {'amd/gpu': '1'}
for _kale_k, _kale_v in _kale_step_limits.items():
_kale_step1_task.container.add_resource_limit(_kale_k, _kale_v)
_kale_step1_task.container.working_dir = "/test"
_kale_step1_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_step1_task.output_artifact_paths.update(_kale_output_artifacts)
_kale_step1_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_step1_task.dependent_names +
_kale_volume_step_names)
_kale_step1_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_step1_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
_kale_step2_task = _kale_step2_op()\
.add_pvolumes(_kale_pvolumes_dict)\
.after(_kale_step1_task)
_kale_step_labels = {'common-label': 'true'}
for _kale_k, _kale_v in _kale_step_labels.items():
_kale_step2_task.add_pod_label(_kale_k, _kale_v)
_kale_step2_task.set_retry_strategy(
num_retries=5,
retry_policy="Always",
backoff_duration="20",
backoff_factor=2,
backoff_max_duration=None)
_kale_step2_task.container.working_dir = "/test"
_kale_step2_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_step2_task.output_artifact_paths.update(_kale_output_artifacts)
_kale_step2_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_step2_task.dependent_names +
_kale_volume_step_names)
_kale_step2_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_step2_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
_kale_step3_task = _kale_step3_op()\
.add_pvolumes(_kale_pvolumes_dict)\
.after(_kale_step2_task, _kale_step1_task)
_kale_step_annotations = {'step3-annotation': 'test'}
for _kale_k, _kale_v in _kale_step_annotations.items():
_kale_step3_task.add_pod_annotation(_kale_k, _kale_v)
_kale_step_labels = {'common-label': 'true'}
for _kale_k, _kale_v in _kale_step_labels.items():
_kale_step3_task.add_pod_label(_kale_k, _kale_v)
_kale_step3_task.container.working_dir = "/test"
_kale_step3_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_step3_task.output_artifact_paths.update(_kale_output_artifacts)
_kale_step3_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_step3_task.dependent_names +
_kale_volume_step_names)
_kale_step3_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_step3_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
def auto_generated_pipeline(density_p='1000000000',
vol_shared_volume='hostpath-pvc'):
_kale_pvolumes_dict = OrderedDict()
_kale_volume_step_names = []
_kale_volume_name_parameters = []
_kale_annotations = {}
_kale_volume = _kfp_dsl.PipelineVolume(pvc=vol_shared_volume)
_kale_pvolumes_dict['/shared_volume'] = _kale_volume
_kale_volume_step_names.sort()
_kale_volume_name_parameters.sort()
_kale_integral1_task = _kale_integral1_op(density_p)\
.add_pvolumes(_kale_pvolumes_dict)\
.after()
#see: https://kubeflow-pipelines.readthedocs.io/en/latest/source/kfp.dsl.html#kfp.dsl.Sidecar.add_resource_limit
#can also be: _kale_step_limits = {'nvidia.com/gpu': '1'}
_kale_step_limits = {'cpu': '1', 'memory': '40Gi'}
for _kale_k, _kale_v in _kale_step_limits.items():
_kale_integral1_task.container.add_resource_limit(_kale_k, _kale_v)
_kale_integral1_task.container.working_dir = "//shared_volume"
_kale_integral1_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_output_artifacts.update({'integral1': '/integral1.html'})
_kale_integral1_task.output_artifact_paths.update(_kale_output_artifacts)
_kale_integral1_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_integral1_task.dependent_names +
_kale_volume_step_names)
_kale_integral1_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_integral1_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
_kale_integral2_task = _kale_integral2_op(density_p)\
.add_pvolumes(_kale_pvolumes_dict)\
.after()
_kale_step_limits = {'cpu': '1', 'memory': '40Gi'}
for _kale_k, _kale_v in _kale_step_limits.items():
_kale_integral2_task.container.add_resource_limit(_kale_k, _kale_v)
_kale_integral2_task.container.working_dir = "//shared_volume"
_kale_integral2_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_output_artifacts.update({'integral2': '/integral2.html'})
_kale_integral2_task.output_artifact_paths.update(_kale_output_artifacts)
_kale_integral2_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_integral2_task.dependent_names +
_kale_volume_step_names)
_kale_integral2_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_integral2_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
_kale_integral3_task = _kale_integral3_op(density_p)\
.add_pvolumes(_kale_pvolumes_dict)\
.after()
_kale_step_limits = {'cpu': '1', 'memory': '40Gi'}
for _kale_k, _kale_v in _kale_step_limits.items():
_kale_integral3_task.container.add_resource_limit(_kale_k, _kale_v)
_kale_integral3_task.container.working_dir = "//shared_volume"
_kale_integral3_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_output_artifacts.update({'integral3': '/integral3.html'})
_kale_integral3_task.output_artifact_paths.update(_kale_output_artifacts)
_kale_integral3_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_integral3_task.dependent_names +
_kale_volume_step_names)
_kale_integral3_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_integral3_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
_kale_integral4_task = _kale_integral4_op(density_p)\
.add_pvolumes(_kale_pvolumes_dict)\
.after()
_kale_step_limits = {'cpu': '1', 'memory': '40Gi'}
for _kale_k, _kale_v in _kale_step_limits.items():
_kale_integral4_task.container.add_resource_limit(_kale_k, _kale_v)
_kale_integral4_task.container.working_dir = "//shared_volume"
_kale_integral4_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_output_artifacts.update({'integral4': '/integral4.html'})
_kale_integral4_task.output_artifact_paths.update(_kale_output_artifacts)
_kale_integral4_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_integral4_task.dependent_names +
_kale_volume_step_names)
_kale_integral4_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_integral4_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
_kale_integral5_task = _kale_integral5_op(density_p)\
.add_pvolumes(_kale_pvolumes_dict)\
.after()
_kale_step_limits = {'cpu': '1', 'memory': '40Gi'}
for _kale_k, _kale_v in _kale_step_limits.items():
_kale_integral5_task.container.add_resource_limit(_kale_k, _kale_v)
_kale_integral5_task.container.working_dir = "//shared_volume"
_kale_integral5_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_output_artifacts.update({'integral5': '/integral5.html'})
_kale_integral5_task.output_artifact_paths.update(_kale_output_artifacts)
_kale_integral5_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_integral5_task.dependent_names +
_kale_volume_step_names)
_kale_integral5_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_integral5_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
_kale_integral6_task = _kale_integral6_op(density_p)\
.add_pvolumes(_kale_pvolumes_dict)\
.after()
_kale_step_limits = {'cpu': '1', 'memory': '40Gi'}
for _kale_k, _kale_v in _kale_step_limits.items():
_kale_integral6_task.container.add_resource_limit(_kale_k, _kale_v)
_kale_integral6_task.container.working_dir = "//shared_volume"
_kale_integral6_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_output_artifacts.update({'integral6': '/integral6.html'})
_kale_integral6_task.output_artifact_paths.update(_kale_output_artifacts)
_kale_integral6_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_integral6_task.dependent_names +
_kale_volume_step_names)
_kale_integral6_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_integral6_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
def _create_deployment_object(self):
"""Creates the deployment object for the grader service using environment variables
Returns:
V1Deployment: a valid kubernetes deployment object
"""
# Configureate Pod template container
# Volumes to mount as subPaths of PV
sub_path_grader_home = str(self.course_dir.parent).strip('/')
sub_path_exchange = str(
self.exchange_dir.relative_to(EXCHANGE_MNT_ROOT))
# define the container to launch
container = client.V1Container(
name='grader-notebook',
image=GRADER_IMAGE_NAME,
command=[
'start-notebook.sh', f'--group=formgrade-{self.course_id}'
],
ports=[client.V1ContainerPort(container_port=8888)],
working_dir=f'/home/{self.grader_name}',
resources=client.V1ResourceRequirements(requests={
"cpu": "100m",
"memory": "200Mi"
},
limits={
"cpu": "500m",
"memory": "1G"
}),
security_context=client.V1SecurityContext(
allow_privilege_escalation=False),
env=[
client.V1EnvVar(name='JUPYTERHUB_SERVICE_NAME',
value=self.course_id),
client.V1EnvVar(name='JUPYTERHUB_API_TOKEN',
value=self.grader_token),
# we're using the K8s Service name 'hub' (defined in the jhub helm chart)
# to connect from our grader-notebooks
client.V1EnvVar(name='JUPYTERHUB_API_URL',
value='http://hub:8081/hub/api'),
client.V1EnvVar(name='JUPYTERHUB_BASE_URL', value='/'),
client.V1EnvVar(name='JUPYTERHUB_SERVICE_PREFIX',
value=f'/services/{self.course_id}/'),
client.V1EnvVar(name='JUPYTERHUB_CLIENT_ID',
value=f'service-{self.course_id}'),
client.V1EnvVar(name='JUPYTERHUB_USER',
value=self.grader_name),
client.V1EnvVar(name='NB_UID', value=str(NB_UID)),
client.V1EnvVar(name='NB_GID', value=str(NB_GID)),
client.V1EnvVar(name='NB_USER', value=self.grader_name),
],
volume_mounts=[
client.V1VolumeMount(mount_path=f'/home/{self.grader_name}',
name=GRADER_PVC,
sub_path=sub_path_grader_home),
client.V1VolumeMount(mount_path='/srv/nbgrader/exchange',
name=GRADER_EXCHANGE_SHARED_PVC,
sub_path=sub_path_exchange),
],
)
# Create and configurate a spec section
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={
'component': self.grader_name,
'app': 'illumidesk'
}),
spec=client.V1PodSpec(
containers=[container],
security_context=client.V1PodSecurityContext(run_as_user=0),
volumes=[
client.V1Volume(
name=GRADER_PVC,
persistent_volume_claim=client.
V1PersistentVolumeClaimVolumeSource(
claim_name=GRADER_PVC),
),
client.V1Volume(
name=GRADER_EXCHANGE_SHARED_PVC,
persistent_volume_claim=client.
V1PersistentVolumeClaimVolumeSource(
claim_name=GRADER_EXCHANGE_SHARED_PVC),
),
],
),
)
# Create the specification of deployment
spec = client.V1DeploymentSpec(
replicas=1,
template=template,
selector={'matchLabels': {
'component': self.grader_name
}})
# Instantiate the deployment object
deployment = client.V1Deployment(
api_version="apps/v1",
kind="Deployment",
metadata=client.V1ObjectMeta(name=self.grader_name),
spec=spec)
return deployment
def auto_generated_pipeline(a='1', b='Some string', c='5'):
_kale_pvolumes_dict = OrderedDict()
_kale_volume_step_names = []
_kale_volume_name_parameters = []
_kale_marshal_vop = _kfp_dsl.VolumeOp(
name="kale-marshal-volume",
resource_name="kale-marshal-pvc",
modes=['ReadWriteMany'],
size="1Gi"
)
_kale_volume_step_names.append(_kale_marshal_vop.name)
_kale_volume_name_parameters.append(
_kale_marshal_vop.outputs["name"].full_name)
_kale_pvolumes_dict['/marshal'] = _kale_marshal_vop.volume
_kale_volume_step_names.sort()
_kale_volume_name_parameters.sort()
_kale_step1_task = _kale_step1_op()\
.add_pvolumes(_kale_pvolumes_dict)\
.after()
_kale_step1_task.container.working_dir = "/test"
_kale_step1_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_step1_task.output_artifact_paths.update(_kale_output_artifacts)
_kale_step1_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_step1_task.dependent_names +
_kale_volume_step_names)
_kale_step1_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_step1_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
_kale_step3_task = _kale_step3_op(b)\
.add_pvolumes(_kale_pvolumes_dict)\
.after(_kale_step1_task)
_kale_step3_task.container.working_dir = "/test"
_kale_step3_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_step3_task.output_artifact_paths.update(_kale_output_artifacts)
_kale_step3_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_step3_task.dependent_names +
_kale_volume_step_names)
_kale_step3_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_step3_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
_kale_step2_task = _kale_step2_op(a, c)\
.add_pvolumes(_kale_pvolumes_dict)\
.after(_kale_step1_task)
_kale_step2_task.container.working_dir = "/test"
_kale_step2_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_step2_task.output_artifact_paths.update(_kale_output_artifacts)
_kale_step2_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_step2_task.dependent_names +
_kale_volume_step_names)
_kale_step2_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_step2_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
_kale_final_auto_snapshot_task = _kale_final_auto_snapshot_op()\
.add_pvolumes(_kale_pvolumes_dict)\
.after(_kale_step3_task, _kale_step2_task)
_kale_final_auto_snapshot_task.container.working_dir = "/test"
_kale_final_auto_snapshot_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_final_auto_snapshot_task.output_artifact_paths.update(
_kale_output_artifacts)
_kale_final_auto_snapshot_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_final_auto_snapshot_task.dependent_names +
_kale_volume_step_names)
_kale_final_auto_snapshot_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_final_auto_snapshot_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
'-y', '%s' % get_base64_encoded_str(namelist_input),
'-k', '/wrf/config/gcs.json',
'-v', 'curwsl_nfs_1:/wrf/output',
'-v', 'curwsl_archive_1:/wrf/archive',
]
logging.info('Creating container spec for image ' + image)
container = client.V1Container(name='test-container',
image_pull_policy='IfNotPresent')
container.image = image
container.command = command
container.args = command_args
container.volume_mounts = [client.V1VolumeMount(mount_path='/wrf/geog', name='test-vol'),
client.V1VolumeMount(mount_path='/wrf/config', name='test-sec-vol')]
container.resources = client.V1ResourceRequirements(requests={'cpu': 4})
container.security_context = client.V1SecurityContext(privileged=True)
logging.info('Initializing pod')
pod = client.V1Pod()
logging.info('Creating pod metadata ')
pod_metadata = client.V1ObjectMeta(name='test-pod')
pod.metadata = pod_metadata
logging.info('Creating volume list')
vols = [client.V1Volume(name='test-vol',
gce_persistent_disk=client.V1GCEPersistentDiskVolumeSource(pd_name='wrf-geog-disk1',
read_only=True)),
client.V1Volume(name='test-sec-vol',
secret=client.V1SecretVolumeSource(secret_name='google-app-creds'))]
def _create_deployment_object(
self,
job_name,
job_image,
deployment_name,
port=80,
replicas=1,
cmd_string=None,
engine_json_file='~/.ipython/profile_default/security/ipcontroller-engine.json',
engine_dir='.'):
""" Create a kubernetes deployment for the job.
Args:
- job_name (string) : Name of the job and deployment
- job_image (string) : Docker image to launch
KWargs:
- port (integer) : Container port
- replicas : Number of replica containers to maintain
Returns:
- True: The deployment object to launch
"""
# sorry, quick hack that doesn't pass this stuff through to test it works.
# TODO it also doesn't only add what is set :(
security_context = None
if 'security' in self.config['execution']:
security_context = client.V1SecurityContext(
run_as_group=self.group_id,
run_as_user=self.user_id,
run_as_non_root=self.run_as_non_root)
# self.user_id = None
# self.group_id = None
# self.run_as_non_root = None
# Create the enviornment variables and command to initiate IPP
environment_vars = client.V1EnvVar(name="TEST", value="SOME DATA")
launch_args = ["-c", "{0}; /app/deploy.sh;".format(cmd_string)]
print(launch_args)
# Configureate Pod template container
container = None
if security_context:
container = client.V1Container(
name=job_name,
image=job_image,
ports=[client.V1ContainerPort(container_port=port)],
command=['/bin/bash'],
args=launch_args,
env=[environment_vars],
security_context=security_context)
else:
container = client.V1Container(
name=job_name,
image=job_image,
ports=[client.V1ContainerPort(container_port=port)],
command=['/bin/bash'],
args=launch_args,
env=[environment_vars])
# Create a secret to enable pulling images from secure repositories
secret = None
if self.secret:
secret = client.V1LocalObjectReference(name=self.secret)
# Create and configurate a spec section
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels={"app": job_name}),
spec=client.V1PodSpec(containers=[container],
image_pull_secrets=[secret]))
# Create the specification of deployment
spec = client.ExtensionsV1beta1DeploymentSpec(replicas=replicas,
template=template)
# Instantiate the deployment object
deployment = client.ExtensionsV1beta1Deployment(
api_version="extensions/v1beta1",
kind="Deployment",
metadata=client.V1ObjectMeta(name=deployment_name),
spec=spec)
return deployment
def _create_job_spec(
self,
name,
command=None,
image=None,
env_vars=None,
overwrite_input_parameters=None,
overwrite_operational_options=None,
):
"""Instantiate a Kubernetes job.
:param name: Name of the job.
:param image: Docker image to use to run the job on.
:param command: List of commands to run on the given job.
:param env_vars: List of environment variables (dictionaries) to
inject into the workflow engine container.
:param interactive_session_type: One of the available interactive
session types.
:param overwrite_input_params: Dictionary with parameters to be
overwritten or added to the current workflow run.
:param type: Dict
:param overwrite_operational_options: Dictionary with operational
options to be overwritten or added to the current workflow run.
:param type: Dict
"""
image = image or self._workflow_engine_image()
command = command or self._workflow_engine_command(
overwrite_input_parameters=overwrite_input_parameters,
overwrite_operational_options=overwrite_operational_options,
)
workflow_engine_env_vars = env_vars or self._workflow_engine_env_vars()
job_controller_env_vars = []
owner_id = str(self.workflow.owner_id)
command = format_cmd(command)
workspace_mount, workspace_volume = get_workspace_volume(
self.workflow.workspace_path)
db_mount, shared_volume = get_shared_volume("db")
workflow_metadata = client.V1ObjectMeta(
name=name,
labels={
"reana_workflow_mode": "batch",
"reana-run-batch-workflow-uuid": str(self.workflow.id_),
},
namespace=REANA_RUNTIME_KUBERNETES_NAMESPACE,
)
job = client.V1Job()
job.api_version = "batch/v1"
job.kind = "Job"
job.metadata = workflow_metadata
spec = client.V1JobSpec(template=client.V1PodTemplateSpec())
spec.template.metadata = workflow_metadata
workflow_engine_container = client.V1Container(
name=current_app.config["WORKFLOW_ENGINE_NAME"],
image=image,
image_pull_policy="IfNotPresent",
env=[],
volume_mounts=[],
command=["/bin/bash", "-c"],
args=command,
)
workflow_engine_env_vars.extend([
{
"name": "REANA_JOB_CONTROLLER_SERVICE_PORT_HTTP",
"value":
str(current_app.config["JOB_CONTROLLER_CONTAINER_PORT"]),
},
{
"name": "REANA_JOB_CONTROLLER_SERVICE_HOST",
"value": "localhost"
},
{
"name": "REANA_COMPONENT_PREFIX",
"value": REANA_COMPONENT_PREFIX
},
{
"name": "REANA_COMPONENT_NAMING_SCHEME",
"value": REANA_COMPONENT_NAMING_SCHEME,
},
{
"name": "REANA_INFRASTRUCTURE_KUBERNETES_NAMESPACE",
"value": REANA_INFRASTRUCTURE_KUBERNETES_NAMESPACE,
},
{
"name": "REANA_RUNTIME_KUBERNETES_NAMESPACE",
"value": REANA_RUNTIME_KUBERNETES_NAMESPACE,
},
{
"name": "REANA_JOB_CONTROLLER_CONNECTION_CHECK_SLEEP",
"value": str(REANA_JOB_CONTROLLER_CONNECTION_CHECK_SLEEP),
},
])
workflow_engine_container.env.extend(workflow_engine_env_vars)
workflow_engine_container.security_context = client.V1SecurityContext(
run_as_group=WORKFLOW_RUNTIME_USER_GID,
run_as_user=WORKFLOW_RUNTIME_USER_UID,
)
workflow_engine_container.volume_mounts = [workspace_mount]
secrets_store = REANAUserSecretsStore(owner_id)
job_controller_env_secrets = secrets_store.get_env_secrets_as_k8s_spec(
)
user = secrets_store.get_secret_value(
"CERN_USER") or WORKFLOW_RUNTIME_USER_NAME
job_controller_container = client.V1Container(
name=current_app.config["JOB_CONTROLLER_NAME"],
image=current_app.config["JOB_CONTROLLER_IMAGE"],
image_pull_policy="IfNotPresent",
env=[],
volume_mounts=[],
command=["/bin/bash", "-c"],
args=self._create_job_controller_startup_cmd(user),
ports=[],
)
job_controller_env_vars.extend([
{
"name": "REANA_USER_ID",
"value": owner_id
},
{
"name": "CERN_USER",
"value": user
},
{
"name": "USER",
"value": user
}, # Required by HTCondor
{
"name": "K8S_CERN_EOS_AVAILABLE",
"value": K8S_CERN_EOS_AVAILABLE
},
{
"name": "IMAGE_PULL_SECRETS",
"value": ",".join(IMAGE_PULL_SECRETS)
},
{
"name": "REANA_SQLALCHEMY_DATABASE_URI",
"value": SQLALCHEMY_DATABASE_URI,
},
{
"name": "REANA_STORAGE_BACKEND",
"value": REANA_STORAGE_BACKEND
},
{
"name": "REANA_COMPONENT_PREFIX",
"value": REANA_COMPONENT_PREFIX
},
{
"name": "REANA_COMPONENT_NAMING_SCHEME",
"value": REANA_COMPONENT_NAMING_SCHEME,
},
{
"name": "REANA_INFRASTRUCTURE_KUBERNETES_NAMESPACE",
"value": REANA_INFRASTRUCTURE_KUBERNETES_NAMESPACE,
},
{
"name": "REANA_RUNTIME_KUBERNETES_NAMESPACE",
"value": REANA_RUNTIME_KUBERNETES_NAMESPACE,
},
{
"name": "REANA_JOB_HOSTPATH_MOUNTS",
"value": json.dumps(REANA_JOB_HOSTPATH_MOUNTS),
},
{
"name":
"REANA_RUNTIME_KUBERNETES_KEEP_ALIVE_JOBS_WITH_STATUSES",
"value":
",".join(
REANA_RUNTIME_KUBERNETES_KEEP_ALIVE_JOBS_WITH_STATUSES),
},
{
"name": "REANA_KUBERNETES_JOBS_MEMORY_LIMIT",
"value": REANA_KUBERNETES_JOBS_MEMORY_LIMIT,
},
{
"name": "REANA_KUBERNETES_JOBS_MAX_USER_MEMORY_LIMIT",
"value": REANA_KUBERNETES_JOBS_MAX_USER_MEMORY_LIMIT,
},
{
"name": "REANA_KUBERNETES_JOBS_TIMEOUT_LIMIT",
"value": REANA_KUBERNETES_JOBS_TIMEOUT_LIMIT,
},
{
"name": "REANA_KUBERNETES_JOBS_MAX_USER_TIMEOUT_LIMIT",
"value": REANA_KUBERNETES_JOBS_MAX_USER_TIMEOUT_LIMIT,
},
{
"name": "WORKSPACE_PATHS",
"value": json.dumps(WORKSPACE_PATHS)
},
])
job_controller_container.env.extend(job_controller_env_vars)
job_controller_container.env.extend(job_controller_env_secrets)
if REANA_RUNTIME_JOBS_KUBERNETES_NODE_LABEL:
job_controller_container.env.append(
{
"name": "REANA_RUNTIME_JOBS_KUBERNETES_NODE_LABEL",
"value":
os.getenv("REANA_RUNTIME_JOBS_KUBERNETES_NODE_LABEL"),
}, )
secrets_volume_mount = secrets_store.get_secrets_volume_mount_as_k8s_spec(
)
job_controller_container.volume_mounts = [workspace_mount, db_mount]
job_controller_container.volume_mounts.append(secrets_volume_mount)
job_controller_container.ports = [{
"containerPort":
current_app.config["JOB_CONTROLLER_CONTAINER_PORT"]
}]
containers = [workflow_engine_container, job_controller_container]
spec.template.spec = client.V1PodSpec(
containers=containers,
node_selector=REANA_RUNTIME_BATCH_KUBERNETES_NODE_LABEL,
)
spec.template.spec.service_account_name = (
REANA_RUNTIME_KUBERNETES_SERVICEACCOUNT_NAME)
volumes = [
workspace_volume,
shared_volume,
secrets_store.get_file_secrets_volume_as_k8s_specs(),
]
# filter out volumes with the same name
spec.template.spec.volumes = list({v["name"]: v
for v in volumes}.values())
if os.getenv("FLASK_ENV") == "development":
code_volume_name = "reana-code"
code_mount_path = "/code"
k8s_code_volume = client.V1Volume(name=code_volume_name)
k8s_code_volume.host_path = client.V1HostPathVolumeSource(
code_mount_path)
spec.template.spec.volumes.append(k8s_code_volume)
for container in spec.template.spec.containers:
container.env.extend(current_app.config["DEBUG_ENV_VARS"])
sub_path = f"reana-{container.name}"
if container.name == "workflow-engine":
sub_path += f"-{self.workflow.type_}"
container.volume_mounts.append({
"name": code_volume_name,
"mountPath": code_mount_path,
"subPath": sub_path,
})
job.spec = spec
job.spec.template.spec.restart_policy = "Never"
job.spec.backoff_limit = 0
return job
def auto_generated_pipeline(booltest='True', d1='5', d2='6', strtest='test'):
_kale_pvolumes_dict = OrderedDict()
_kale_volume_step_names = []
_kale_volume_name_parameters = []
_kale_marshal_vop = _kfp_dsl.VolumeOp(name="kale-marshal-volume",
resource_name="kale-marshal-pvc",
modes=['ReadWriteMany'],
size="1Gi")
_kale_volume_step_names.append(_kale_marshal_vop.name)
_kale_volume_name_parameters.append(
_kale_marshal_vop.outputs["name"].full_name)
_kale_pvolumes_dict['/marshal'] = _kale_marshal_vop.volume
_kale_volume_step_names.sort()
_kale_volume_name_parameters.sort()
_kale_create_matrix_task = _kale_create_matrix_op(d1, d2)\
.add_pvolumes(_kale_pvolumes_dict)\
.after()
_kale_step_limits = {'nvidia.com/gpu': '2'}
for _kale_k, _kale_v in _kale_step_limits.items():
_kale_create_matrix_task.container.add_resource_limit(_kale_k, _kale_v)
_kale_create_matrix_task.container.working_dir = "/kale"
_kale_create_matrix_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-metrics': '/tmp/mlpipeline-metrics.json'})
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_output_artifacts.update({'create_matrix': '/create_matrix.html'})
_kale_create_matrix_task.output_artifact_paths.update(
_kale_output_artifacts)
_kale_create_matrix_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_create_matrix_task.dependent_names +
_kale_volume_step_names)
_kale_create_matrix_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_create_matrix_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
_kale_sum_matrix_task = _kale_sum_matrix_op()\
.add_pvolumes(_kale_pvolumes_dict)\
.after(_kale_create_matrix_task)
_kale_sum_matrix_task.container.working_dir = "/kale"
_kale_sum_matrix_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-metrics': '/tmp/mlpipeline-metrics.json'})
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_output_artifacts.update({'sum_matrix': '/sum_matrix.html'})
_kale_sum_matrix_task.output_artifact_paths.update(_kale_output_artifacts)
_kale_sum_matrix_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_sum_matrix_task.dependent_names +
_kale_volume_step_names)
_kale_sum_matrix_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_sum_matrix_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
def auto_generated_pipeline(vol_shared_volume_kale_volumes='titanic'):
#pvolumes_dict = OrderedDict()
#
#annotations = {}
#
#volume = dsl.PipelineVolume(pvc=vol_shared_volume_kale_volumes)
#
#pvolumes_dict['/shared_volume/kale_volumes/'] = volume
#
#marshal_vop = dsl.VolumeOp(
# name="kale_marshal_volume",
# resource_name="kale-marshal-pvc",
# modes=dsl.VOLUME_MODE_RWM,
# size="1Gi"
#)
#pvolumes_dict['/marshal'] = marshal_vop.volume
#loaddata_task = loaddata_op(vol_shared_volume_kale_volumes)\
# .add_pvolumes(pvolumes_dict)\
# .after()
#loaddata_task.container.working_dir = "/shared_volume/notebooks"
#loaddata_task.container.set_security_context(
# k8s_client.V1SecurityContext(run_as_user=0))
#
#datapreprocessing_task = datapreprocessing_op(vol_shared_volume_kale_volumes)\
# .add_pvolumes(pvolumes_dict)\
# .after(loaddata_task)
#datapreprocessing_task.container.working_dir = "/shared_volume/notebooks"
#datapreprocessing_task.container.set_security_context(
# k8s_client.V1SecurityContext(run_as_user=0))
#
#featureengineering_task = featureengineering_op(vol_shared_volume_kale_volumes)\
# .add_pvolumes(pvolumes_dict)\
# .after(datapreprocessing_task)
#featureengineering_task.container.working_dir = "/shared_volume/notebooks"
#featureengineering_task.container.set_security_context(
# k8s_client.V1SecurityContext(run_as_user=0))
#
#decisiontree_task = decisiontree_op(vol_shared_volume_kale_volumes)\
# .add_pvolumes(pvolumes_dict)\
# .after(featureengineering_task)
#decisiontree_task.container.working_dir = "/shared_volume/notebooks"
#decisiontree_task.container.set_security_context(
# k8s_client.V1SecurityContext(run_as_user=0))
#
#svm_task = svm_op(vol_shared_volume_kale_volumes)\
# .add_pvolumes(pvolumes_dict)\
# .after(featureengineering_task)
#svm_task.container.working_dir = "/shared_volume/notebooks"
#svm_task.container.set_security_context(
# k8s_client.V1SecurityContext(run_as_user=0))
#
#naivebayes_task = naivebayes_op(vol_shared_volume_kale_volumes)\
# .add_pvolumes(pvolumes_dict)\
# .after(featureengineering_task)
#naivebayes_task.container.working_dir = "/shared_volume/notebooks"
#naivebayes_task.container.set_security_context(
# k8s_client.V1SecurityContext(run_as_user=0))
#
#logisticregression_task = logisticregression_op(vol_shared_volume_kale_volumes)\
# .add_pvolumes(pvolumes_dict)\
# .after(featureengineering_task)
#logisticregression_task.container.working_dir = "/shared_volume/notebooks"
#logisticregression_task.container.set_security_context(
# k8s_client.V1SecurityContext(run_as_user=0))
#
#randomforest_task = randomforest_op(vol_shared_volume_kale_volumes)\
# .add_pvolumes(pvolumes_dict)\
# .after(featureengineering_task)
#randomforest_task.container.working_dir = "/shared_volume/notebooks"
#randomforest_task.container.set_security_context(
# k8s_client.V1SecurityContext(run_as_user=0))
#
#results_task = results_op(vol_shared_volume_kale_volumes)\
# .add_pvolumes(pvolumes_dict)\
# .after(randomforest_task, logisticregression_task, naivebayes_task, svm_task, decisiontree_task)
#results_task.container.working_dir = "/shared_volume/notebooks"
#results_task.container.set_security_context(
# k8s_client.V1SecurityContext(run_as_user=0))
loaddata_task = loaddata_op(vol_shared_volume_kale_volumes)\
.add_volume(k8s_client.V1Volume(name='efs-pvc', persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource(claim_name='efs')))\
.add_volume_mount(k8s_client.V1VolumeMount(mount_path='/shared_volume',
name='efs-pvc'))\
.after()
loaddata_task.add_pod_annotation('iam.amazonaws.com/role',
'arn:aws:iam::<micuenta>:role/<mirol>')
loaddata_task.container.working_dir = "/shared_volume/notebooks"
loaddata_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
datapreprocessing_task = datapreprocessing_op(vol_shared_volume_kale_volumes)\
.add_volume(k8s_client.V1Volume(name='efs-pvc', persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource(claim_name='efs')))\
.add_volume_mount(k8s_client.V1VolumeMount(mount_path='/shared_volume',
name='efs-pvc'))\
.after(loaddata_task)
datapreprocessing_task.add_pod_annotation(
'iam.amazonaws.com/role', 'arn:aws:iam::<micuenta>:role/<mirol>')
datapreprocessing_task.container.working_dir = "/shared_volume/notebooks"
datapreprocessing_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
featureengineering_task = featureengineering_op(vol_shared_volume_kale_volumes)\
.add_volume(k8s_client.V1Volume(name='efs-pvc', persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource(claim_name='efs')))\
.add_volume_mount(k8s_client.V1VolumeMount(mount_path='/shared_volume',
name='efs-pvc'))\
.after(datapreprocessing_task)
featureengineering_task.add_pod_annotation(
'iam.amazonaws.com/role', 'arn:aws:iam::<micuenta>:role/<mirol>')
featureengineering_task.container.working_dir = "/shared_volume/notebooks"
featureengineering_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
decisiontree_task = decisiontree_op(vol_shared_volume_kale_volumes)\
.add_volume(k8s_client.V1Volume(name='efs-pvc', persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource(claim_name='efs')))\
.add_volume_mount(k8s_client.V1VolumeMount(mount_path='/shared_volume',
name='efs-pvc'))\
.after(featureengineering_task)
decisiontree_task.add_pod_annotation(
'iam.amazonaws.com/role', 'arn:aws:iam::<micuenta>:role/<mirol>')
decisiontree_task.container.working_dir = "/shared_volume/notebooks"
decisiontree_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
svm_task = svm_op(vol_shared_volume_kale_volumes)\
.add_volume(k8s_client.V1Volume(name='efs-pvc', persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource(claim_name='efs')))\
.add_volume_mount(k8s_client.V1VolumeMount(mount_path='/shared_volume',
name='efs-pvc'))\
.after(featureengineering_task)
svm_task.add_pod_annotation('iam.amazonaws.com/role',
'arn:aws:iam::<micuenta>:role/<mirol>')
svm_task.container.working_dir = "/shared_volume/notebooks"
svm_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
#
naivebayes_task = naivebayes_op(vol_shared_volume_kale_volumes)\
.add_volume(k8s_client.V1Volume(name='efs-pvc', persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource(claim_name='efs')))\
.add_volume_mount(k8s_client.V1VolumeMount(mount_path='/shared_volume',
name='efs-pvc'))\
.after(featureengineering_task)
naivebayes_task.add_pod_annotation('iam.amazonaws.com/role',
'arn:aws:iam::<micuenta>:role/<mirol>')
naivebayes_task.container.working_dir = "/shared_volume/notebooks"
naivebayes_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
#
logisticregression_task = logisticregression_op(vol_shared_volume_kale_volumes)\
.add_volume(k8s_client.V1Volume(name='efs-pvc', persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource(claim_name='efs')))\
.add_volume_mount(k8s_client.V1VolumeMount(mount_path='/shared_volume',
name='efs-pvc'))\
.after(featureengineering_task)
logisticregression_task.add_pod_annotation(
'iam.amazonaws.com/role', 'arn:aws:iam::<micuenta>:role/<mirol>')
logisticregression_task.container.working_dir = "/shared_volume/notebooks"
logisticregression_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
#
randomforest_task = randomforest_op(vol_shared_volume_kale_volumes)\
.add_volume(k8s_client.V1Volume(name='efs-pvc', persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource(claim_name='efs')))\
.add_volume_mount(k8s_client.V1VolumeMount(mount_path='/shared_volume',
name='efs-pvc'))\
.after(featureengineering_task)
randomforest_task.add_pod_annotation(
'iam.amazonaws.com/role', 'arn:aws:iam::<micuenta>:role/<mirol>')
randomforest_task.container.working_dir = "/shared_volume/notebooks"
randomforest_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
#
results_task = results_op(vol_shared_volume_kale_volumes)\
.add_volume(k8s_client.V1Volume(name='efs-pvc', persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource(claim_name='efs')))\
.add_volume_mount(k8s_client.V1VolumeMount(mount_path='/shared_volume',
name='efs-pvc'))\
.after(randomforest_task, logisticregression_task, naivebayes_task, svm_task, decisiontree_task)
results_task.add_pod_annotation('iam.amazonaws.com/role',
'arn:aws:iam::<micuenta>:role/<mirol>')
results_task.container.working_dir = "/shared_volume/notebooks"
results_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
def auto_generated_pipeline():
pvolumes_dict = OrderedDict()
volume_step_names = []
volume_name_parameters = []
marshal_vop = dsl.VolumeOp(
name="kale-marshal-volume",
resource_name="kale-marshal-pvc",
modes=dsl.VOLUME_MODE_RWM,
size="1Gi"
)
volume_step_names.append(marshal_vop.name)
volume_name_parameters.append(marshal_vop.outputs["name"].full_name)
pvolumes_dict['/marshal'] = marshal_vop.volume
volume_step_names.sort()
volume_name_parameters.sort()
loaddata_task = loaddata_op()\
.add_pvolumes(pvolumes_dict)\
.after()
loaddata_task.container.working_dir = "/kale"
loaddata_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'loaddata': '/loaddata.html'})
loaddata_task.output_artifact_paths.update(output_artifacts)
loaddata_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
dep_names = loaddata_task.dependent_names + volume_step_names
loaddata_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(dep_names))
if volume_name_parameters:
loaddata_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(volume_name_parameters))
datapreprocessing_task = datapreprocessing_op()\
.add_pvolumes(pvolumes_dict)\
.after(loaddata_task)
datapreprocessing_task.container.working_dir = "/kale"
datapreprocessing_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'datapreprocessing': '/datapreprocessing.html'})
datapreprocessing_task.output_artifact_paths.update(output_artifacts)
datapreprocessing_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
dep_names = datapreprocessing_task.dependent_names + volume_step_names
datapreprocessing_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(dep_names))
if volume_name_parameters:
datapreprocessing_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(volume_name_parameters))
featureengineering_task = featureengineering_op()\
.add_pvolumes(pvolumes_dict)\
.after(datapreprocessing_task)
featureengineering_task.container.working_dir = "/kale"
featureengineering_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'featureengineering': '/featureengineering.html'})
featureengineering_task.output_artifact_paths.update(output_artifacts)
featureengineering_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
dep_names = featureengineering_task.dependent_names + volume_step_names
featureengineering_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(dep_names))
if volume_name_parameters:
featureengineering_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(volume_name_parameters))
decisiontree_task = decisiontree_op()\
.add_pvolumes(pvolumes_dict)\
.after(featureengineering_task)
decisiontree_task.container.working_dir = "/kale"
decisiontree_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'decisiontree': '/decisiontree.html'})
decisiontree_task.output_artifact_paths.update(output_artifacts)
decisiontree_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
dep_names = decisiontree_task.dependent_names + volume_step_names
decisiontree_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(dep_names))
if volume_name_parameters:
decisiontree_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(volume_name_parameters))
svm_task = svm_op()\
.add_pvolumes(pvolumes_dict)\
.after(featureengineering_task)
svm_task.container.working_dir = "/kale"
svm_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'svm': '/svm.html'})
svm_task.output_artifact_paths.update(output_artifacts)
svm_task.add_pod_label("pipelines.kubeflow.org/metadata_written", "true")
dep_names = svm_task.dependent_names + volume_step_names
svm_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(dep_names))
if volume_name_parameters:
svm_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(volume_name_parameters))
naivebayes_task = naivebayes_op()\
.add_pvolumes(pvolumes_dict)\
.after(featureengineering_task)
naivebayes_task.container.working_dir = "/kale"
naivebayes_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'naivebayes': '/naivebayes.html'})
naivebayes_task.output_artifact_paths.update(output_artifacts)
naivebayes_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
dep_names = naivebayes_task.dependent_names + volume_step_names
naivebayes_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(dep_names))
if volume_name_parameters:
naivebayes_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(volume_name_parameters))
logisticregression_task = logisticregression_op()\
.add_pvolumes(pvolumes_dict)\
.after(featureengineering_task)
logisticregression_task.container.working_dir = "/kale"
logisticregression_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'logisticregression': '/logisticregression.html'})
logisticregression_task.output_artifact_paths.update(output_artifacts)
logisticregression_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
dep_names = logisticregression_task.dependent_names + volume_step_names
logisticregression_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(dep_names))
if volume_name_parameters:
logisticregression_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(volume_name_parameters))
randomforest_task = randomforest_op()\
.add_pvolumes(pvolumes_dict)\
.after(featureengineering_task)
randomforest_task.container.working_dir = "/kale"
randomforest_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'randomforest': '/randomforest.html'})
randomforest_task.output_artifact_paths.update(output_artifacts)
randomforest_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
dep_names = randomforest_task.dependent_names + volume_step_names
randomforest_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(dep_names))
if volume_name_parameters:
randomforest_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(volume_name_parameters))
results_task = results_op()\
.add_pvolumes(pvolumes_dict)\
.after(randomforest_task, logisticregression_task, naivebayes_task, svm_task, decisiontree_task)
results_task.container.working_dir = "/kale"
results_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
output_artifacts = {}
output_artifacts.update(
{'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'})
output_artifacts.update({'results': '/results.html'})
results_task.output_artifact_paths.update(output_artifacts)
results_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
dep_names = results_task.dependent_names + volume_step_names
results_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(dep_names))
if volume_name_parameters:
results_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(volume_name_parameters))
def template(context):
"""
handle yml env
"""
name = context.get("name")
version = context.get("version")
labels = {
"app": name,
"version": "v1"
} if not version else {
"app": name.strip("-v2"),
"version": version
}
image_tag = context["name"].split('-')[1]
image = context["image_namespace"] + image_tag + ":" + context[
"image_branch"]
args = [arg for arg in context["args"]] if context.get("args") else None
limits, requests = context["resources"]["limits"], context["resources"][
"requests"]
replicas = context.get("replicas", 1)
workingDir = context["workingDir"]
if name == "backend-logproxy":
annotations = {"sidecar.istio.io/inject": "false"}
else:
annotations = {"traffic.sidecar.istio.io/excludeOutboundPorts": "6379"}
"""
handle cmdb env
"""
filename = "env_" + name.split("-")[1] + ".yml"
env = handle_env("/tmp/{}".format(filename))
"""
k8s yaml 组件模块
"""
#从svn分支configmap目录中获取相关的目录结构
parentDir, subdir = handle_configmap("configmap")
volumemounts = [
client.V1VolumeMount(mount_path="/{}".format(parentDir),
name="mainfiles")
]
volumes = [
client.V1Volume(
name="mainfiles",
config_map=client.V1ConfigMapVolumeSource(name="mainfiles"))
]
for dir in subdir:
volumemounts.append(
client.V1VolumeMount(mount_path="/{}/{}".format(parentDir, dir),
name=dir))
volumes.append(
client.V1Volume(
name=dir, config_map=client.V1ConfigMapVolumeSource(name=dir)))
if name.startswith("frontend-dispatch"):
containers = [
client.V1Container(
name=name,
image=image,
env=env,
args=args,
volume_mounts=volumemounts,
image_pull_policy="Always",
lifecycle=client.V1Lifecycle(pre_stop=client.V1Handler(
_exec=client.V1ExecAction(
command=["nginx", "-s", "quit"]))),
readiness_probe=client.V1Probe(_exec=client.V1ExecAction(
command=['cat', '/tmp/container_ready']),
initial_delay_seconds=10,
period_seconds=5),
resources=client.V1ResourceRequirements(limits=limits,
requests=requests),
security_context=client.V1SecurityContext(privileged=True),
working_dir=workingDir,
)
]
else:
containers = [
client.V1Container(
name=name,
image=image,
env=env,
args=args,
volume_mounts=volumemounts,
image_pull_policy="Always",
readiness_probe=client.V1Probe(_exec=client.V1ExecAction(
command=['cat', '/tmp/container_ready']),
initial_delay_seconds=10,
period_seconds=5),
resources=client.V1ResourceRequirements(limits=limits,
requests=requests),
security_context=client.V1SecurityContext(privileged=True),
working_dir=workingDir,
)
]
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(labels=labels, annotations=annotations),
spec=client.V1PodSpec(
containers=containers,
dns_policy="ClusterFirst",
image_pull_secrets=[
client.V1LocalObjectReference(name="image-pull-secret")
],
restart_policy="Always",
volumes=volumes))
spec = client.V1DeploymentSpec(
replicas=replicas,
selector=client.V1LabelSelector(match_labels=labels),
template=template,
strategy=client.ExtensionsV1beta1DeploymentStrategy(
rolling_update=client.ExtensionsV1beta1RollingUpdateDeployment(
max_surge=1, max_unavailable='25%'),
type="RollingUpdate",
),
)
return client.V1Deployment(api_version="apps/v1",
kind="Deployment",
metadata=client.V1ObjectMeta(name=name,
labels=labels),
spec=spec)
def get_pod_manifest(self):
"""Include volume with the git repository."""
repository = yield self.git_repository()
options = self.user_options
# make sure the pod name is less than 64 characters - if longer, keep
# the last 16 untouched since it is the server hash
if len(self.pod_name) > 63:
self.pod_name = self.pod_name[:47] + self.pod_name[-16:]
# Process the requested server options
server_options = options.get("server_options", {})
self.default_url = server_options.get("defaultUrl")
self.cpu_guarantee = float(server_options.get("cpu_request", 0.1))
# Make the user pods be in Guaranteed QoS class if the user
# had specified a memory request. Otherwise use a sensible default.
self.mem_guarantee = server_options.get("mem_request", "500M")
self.mem_limit = server_options.get("mem_request", "1G")
gpu = server_options.get("gpu_request", {})
if gpu:
self.extra_resource_limits = {"nvidia.com/gpu": str(gpu)}
# Configure the git repository volume
git_volume_name = self.pod_name[:54] + "-git-repo"
# 1. Define a new empty volume.
self.volumes = [
volume for volume in self.volumes
if volume["name"] != git_volume_name
]
volume = {"name": git_volume_name, "emptyDir": {}}
self.volumes.append(volume)
# 2. Define a volume mount for both init and notebook containers.
mount_path = f'/work/{options["project"]}'
volume_mount = {"mountPath": mount_path, "name": git_volume_name}
# 3. Configure the init container
init_container_name = "git-clone"
self.init_containers = [
container for container in self.init_containers
if not container.name.startswith(init_container_name)
]
lfs_auto_fetch = server_options.get("lfs_auto_fetch")
init_container = client.V1Container(
name=init_container_name,
env=[
client.V1EnvVar(name="MOUNT_PATH", value=mount_path),
client.V1EnvVar(name="REPOSITORY", value=repository),
client.V1EnvVar(
name="LFS_AUTO_FETCH",
value="1" if lfs_auto_fetch else "0",
),
client.V1EnvVar(name="COMMIT_SHA",
value=str(options.get("commit_sha"))),
client.V1EnvVar(name="BRANCH",
value=options.get("branch", "master")),
client.V1EnvVar(name="JUPYTERHUB_USER", value=self.user.name),
],
image=options.get("git_clone_image"),
volume_mounts=[volume_mount],
working_dir=mount_path,
security_context=client.V1SecurityContext(run_as_user=0),
)
self.init_containers.append(init_container)
# 4. Configure notebook container git repo volume mount
self.volume_mounts = [
volume_mount for volume_mount in self.volume_mounts
if volume_mount["mountPath"] != mount_path
]
self.volume_mounts.append(volume_mount)
# 5. Configure autosaving script execution hook
self.lifecycle_hooks = {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/usr/local/bin/pre-stop.sh",
"||",
"true",
]
}
}
}
# Finalize the pod configuration
# Set the repository path to the working directory
self.working_dir = mount_path
self.notebook_dir = mount_path
# add git project-specific annotations
self.extra_annotations = {
RENKU_ANNOTATION_PREFIX + "namespace":
options.get("namespace"),
RENKU_ANNOTATION_PREFIX + "projectName":
options.get("project"),
RENKU_ANNOTATION_PREFIX + "projectId":
"{}".format(options.get("project_id")),
RENKU_ANNOTATION_PREFIX + "branch":
options.get("branch"),
RENKU_ANNOTATION_PREFIX + "commit-sha":
options.get("commit_sha"),
}
# add username to labels
safe_username = escapism.escape(self.user.name,
escape_char="-").lower()
self.extra_labels = {
RENKU_ANNOTATION_PREFIX + "username": safe_username
}
self.delete_grace_period = 30
pod = yield super().get_pod_manifest()
# Because repository comes from a coroutine, we can't put it simply in `get_env()`
pod.spec.containers[0].env.append(
client.V1EnvVar("CI_REPOSITORY_URL", repository))
# Add image pull secrets
if options.get("image_pull_secrets"):
secrets = [
client.V1LocalObjectReference(name=name)
for name in options.get("image_pull_secrets")
]
pod.spec.image_pull_secrets = secrets
return pod
def auto_generated_pipeline():
_kale_pvolumes_dict = OrderedDict()
_kale_volume_step_names = []
_kale_volume_name_parameters = []
_kale_marshal_vop = _kfp_dsl.VolumeOp(name="kale-marshal-volume",
resource_name="kale-marshal-pvc",
modes=_kfp_dsl.VOLUME_MODE_RWM,
size="1Gi")
_kale_volume_step_names.append(_kale_marshal_vop.name)
_kale_volume_name_parameters.append(
_kale_marshal_vop.outputs["name"].full_name)
_kale_pvolumes_dict['/marshal'] = _kale_marshal_vop.volume
_kale_volume_step_names.sort()
_kale_volume_name_parameters.sort()
_kale_loaddata_task = _kale_loaddata_op()\
.add_pvolumes(_kale_pvolumes_dict)\
.after()
_kale_loaddata_task.container.working_dir = "/Users/animeshsingh/go/src/github.com/kubeflow/kale/~/go/src/github.com/kubeflow/kale/examples/titanic-ml-dataset"
_kale_loaddata_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_output_artifacts.update({'loaddata': '/tmp/loaddata.html'})
_kale_loaddata_task.output_artifact_paths.update(_kale_output_artifacts)
_kale_loaddata_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_loaddata_task.dependent_names +
_kale_volume_step_names)
_kale_loaddata_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_loaddata_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
_kale_datapreprocessing_task = _kale_datapreprocessing_op()\
.add_pvolumes(_kale_pvolumes_dict)\
.after(_kale_loaddata_task)
_kale_datapreprocessing_task.container.working_dir = "/Users/animeshsingh/go/src/github.com/kubeflow/kale/~/go/src/github.com/kubeflow/kale/examples/titanic-ml-dataset"
_kale_datapreprocessing_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_output_artifacts.update(
{'datapreprocessing': '/tmp/datapreprocessing.html'})
_kale_datapreprocessing_task.output_artifact_paths.update(
_kale_output_artifacts)
_kale_datapreprocessing_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_datapreprocessing_task.dependent_names +
_kale_volume_step_names)
_kale_datapreprocessing_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_datapreprocessing_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
_kale_featureengineering_task = _kale_featureengineering_op()\
.add_pvolumes(_kale_pvolumes_dict)\
.after(_kale_datapreprocessing_task)
_kale_featureengineering_task.container.working_dir = "/Users/animeshsingh/go/src/github.com/kubeflow/kale/~/go/src/github.com/kubeflow/kale/examples/titanic-ml-dataset"
_kale_featureengineering_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_output_artifacts.update(
{'featureengineering': '/tmp/featureengineering.html'})
_kale_featureengineering_task.output_artifact_paths.update(
_kale_output_artifacts)
_kale_featureengineering_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_featureengineering_task.dependent_names +
_kale_volume_step_names)
_kale_featureengineering_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_featureengineering_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
_kale_decisiontree_task = _kale_decisiontree_op()\
.add_pvolumes(_kale_pvolumes_dict)\
.after(_kale_featureengineering_task)
_kale_decisiontree_task.container.working_dir = "/Users/animeshsingh/go/src/github.com/kubeflow/kale/~/go/src/github.com/kubeflow/kale/examples/titanic-ml-dataset"
_kale_decisiontree_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_output_artifacts.update({'decisiontree': '/tmp/decisiontree.html'})
_kale_decisiontree_task.output_artifact_paths.update(
_kale_output_artifacts)
_kale_decisiontree_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_decisiontree_task.dependent_names +
_kale_volume_step_names)
_kale_decisiontree_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_decisiontree_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
_kale_svm_task = _kale_svm_op()\
.add_pvolumes(_kale_pvolumes_dict)\
.after(_kale_featureengineering_task)
_kale_svm_task.container.working_dir = "/Users/animeshsingh/go/src/github.com/kubeflow/kale/~/go/src/github.com/kubeflow/kale/examples/titanic-ml-dataset"
_kale_svm_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_output_artifacts.update({'svm': '/tmp/svm.html'})
_kale_svm_task.output_artifact_paths.update(_kale_output_artifacts)
_kale_svm_task.add_pod_label("pipelines.kubeflow.org/metadata_written",
"true")
_kale_dep_names = (_kale_svm_task.dependent_names +
_kale_volume_step_names)
_kale_svm_task.add_pod_annotation("kubeflow-kale.org/dependent-templates",
json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_svm_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
_kale_naivebayes_task = _kale_naivebayes_op()\
.add_pvolumes(_kale_pvolumes_dict)\
.after(_kale_featureengineering_task)
_kale_naivebayes_task.container.working_dir = "/Users/animeshsingh/go/src/github.com/kubeflow/kale/~/go/src/github.com/kubeflow/kale/examples/titanic-ml-dataset"
_kale_naivebayes_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_output_artifacts.update({'naivebayes': '/tmp/naivebayes.html'})
_kale_naivebayes_task.output_artifact_paths.update(_kale_output_artifacts)
_kale_naivebayes_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_naivebayes_task.dependent_names +
_kale_volume_step_names)
_kale_naivebayes_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_naivebayes_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
_kale_logisticregression_task = _kale_logisticregression_op()\
.add_pvolumes(_kale_pvolumes_dict)\
.after(_kale_featureengineering_task)
_kale_logisticregression_task.container.working_dir = "/Users/animeshsingh/go/src/github.com/kubeflow/kale/~/go/src/github.com/kubeflow/kale/examples/titanic-ml-dataset"
_kale_logisticregression_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_output_artifacts.update(
{'logisticregression': '/tmp/logisticregression.html'})
_kale_logisticregression_task.output_artifact_paths.update(
_kale_output_artifacts)
_kale_logisticregression_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_logisticregression_task.dependent_names +
_kale_volume_step_names)
_kale_logisticregression_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_logisticregression_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
_kale_randomforest_task = _kale_randomforest_op()\
.add_pvolumes(_kale_pvolumes_dict)\
.after(_kale_featureengineering_task)
_kale_randomforest_task.container.working_dir = "/Users/animeshsingh/go/src/github.com/kubeflow/kale/~/go/src/github.com/kubeflow/kale/examples/titanic-ml-dataset"
_kale_randomforest_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_output_artifacts.update({'randomforest': '/tmp/randomforest.html'})
_kale_randomforest_task.output_artifact_paths.update(
_kale_output_artifacts)
_kale_randomforest_task.add_pod_label(
"pipelines.kubeflow.org/metadata_written", "true")
_kale_dep_names = (_kale_randomforest_task.dependent_names +
_kale_volume_step_names)
_kale_randomforest_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_randomforest_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
_kale_results_task = _kale_results_op()\
.add_pvolumes(_kale_pvolumes_dict)\
.after(_kale_randomforest_task, _kale_logisticregression_task, _kale_naivebayes_task, _kale_svm_task, _kale_decisiontree_task)
_kale_results_task.container.working_dir = "/Users/animeshsingh/go/src/github.com/kubeflow/kale/~/go/src/github.com/kubeflow/kale/examples/titanic-ml-dataset"
_kale_results_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
_kale_output_artifacts = {}
_kale_output_artifacts.update(
{'mlpipeline-ui-metadata': '/tmp/mlpipeline-ui-metadata.json'})
_kale_output_artifacts.update({'results': '/tmp/results.html'})
_kale_results_task.output_artifact_paths.update(_kale_output_artifacts)
_kale_results_task.add_pod_label("pipelines.kubeflow.org/metadata_written",
"true")
_kale_dep_names = (_kale_results_task.dependent_names +
_kale_volume_step_names)
_kale_results_task.add_pod_annotation(
"kubeflow-kale.org/dependent-templates", json.dumps(_kale_dep_names))
if _kale_volume_name_parameters:
_kale_results_task.add_pod_annotation(
"kubeflow-kale.org/volume-name-parameters",
json.dumps(_kale_volume_name_parameters))
cloud_metadata = get_config('singleuser.cloudMetadata', {})
if not cloud_metadata.get('enabled', False):
# Use iptables to block access to cloud metadata by default
network_tools_image_name = get_config('singleuser.networkTools.image.name')
network_tools_image_tag = get_config('singleuser.networkTools.image.tag')
ip_block_container = client.V1Container(
name="block-cloud-metadata",
image=f"{network_tools_image_name}:{network_tools_image_tag}",
command=[
'iptables', '-A', 'OUTPUT', '-d',
cloud_metadata.get('ip', '169.254.169.254'), '-j', 'DROP'
],
security_context=client.V1SecurityContext(
privileged=True,
run_as_user=0,
capabilities=client.V1Capabilities(add=['NET_ADMIN'])))
c.KubeSpawner.init_containers.append(ip_block_container)
if get_config('debug.enabled', False):
c.JupyterHub.log_level = 'DEBUG'
c.Spawner.debug = True
extra_config = get_config('hub.extraConfig', {})
if isinstance(extra_config, str):
from textwrap import indent, dedent
msg = dedent("""
hub.extraConfig should be a dict of strings,
but found a single string instead.
def auto_generated_pipeline(vol_shared_volume='efs'):
pvolumes_dict = OrderedDict()
annotations = {}
volume = dsl.PipelineVolume(pvc=vol_shared_volume)
pvolumes_dict['/shared_volume/'] = volume
loaddata_task = loaddata_op(vol_shared_volume)\
.add_pvolumes(pvolumes_dict)\
.after()
loaddata_task.container.working_dir = "/shared_volume/notebooks/titanic"
loaddata_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
datapreprocessing_task = datapreprocessing_op(vol_shared_volume)\
.add_pvolumes(pvolumes_dict)\
.after(loaddata_task)
datapreprocessing_task.container.working_dir = "/shared_volume/notebooks/titanic"
datapreprocessing_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
featureengineering_task = featureengineering_op(vol_shared_volume)\
.add_pvolumes(pvolumes_dict)\
.after(datapreprocessing_task)
featureengineering_task.container.working_dir = "/shared_volume/notebooks/titanic"
featureengineering_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
decisiontree_task = decisiontree_op(vol_shared_volume)\
.add_pvolumes(pvolumes_dict)\
.after(featureengineering_task)
decisiontree_task.container.working_dir = "/shared_volume/notebooks/titanic"
decisiontree_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
svm_task = svm_op(vol_shared_volume)\
.add_pvolumes(pvolumes_dict)\
.after(featureengineering_task)
svm_task.container.working_dir = "/shared_volume/notebooks/titanic"
svm_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
naivebayes_task = naivebayes_op(vol_shared_volume)\
.add_pvolumes(pvolumes_dict)\
.after(featureengineering_task)
naivebayes_task.container.working_dir = "/shared_volume/notebooks/titanic"
naivebayes_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
logisticregression_task = logisticregression_op(vol_shared_volume)\
.add_pvolumes(pvolumes_dict)\
.after(featureengineering_task)
logisticregression_task.container.working_dir = "/shared_volume/notebooks/titanic"
logisticregression_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
randomforest_task = randomforest_op(vol_shared_volume)\
.add_pvolumes(pvolumes_dict)\
.after(featureengineering_task)
randomforest_task.container.working_dir = "/shared_volume/notebooks/titanic"
randomforest_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
results_task = results_op(vol_shared_volume)\
.add_pvolumes(pvolumes_dict)\
.after(randomforest_task, logisticregression_task, naivebayes_task, svm_task, decisiontree_task)
results_task.container.working_dir = "/shared_volume/notebooks/titanic"
results_task.container.set_security_context(
k8s_client.V1SecurityContext(run_as_user=0))
