class EnvironmentSchema(BaseCamelSchema):
labels = fields.Dict(values=fields.Str(),
keys=fields.Str(),
allow_none=True)
annotations = fields.Dict(values=fields.Str(),
keys=fields.Str(),
allow_none=True)
node_selector = fields.Dict(values=fields.Str(),
keys=fields.Str(),
allow_none=True)
affinity = SwaggerField(cls=k8s_schemas.V1Affinity, allow_none=True)
tolerations = fields.List(SwaggerField(cls=k8s_schemas.V1Toleration),
allow_none=True)
node_name = fields.Str(allow_none=True)
service_account_name = fields.Str(allow_none=True)
host_aliases = fields.List(SwaggerField(cls=k8s_schemas.V1HostAlias),
allow_none=True)
security_context = SwaggerField(cls=k8s_schemas.V1SecurityContext,
allow_none=True)
image_pull_secrets = fields.List(fields.Str(), allow_none=True)
host_network = fields.Bool(allow_none=True)
dns_policy = fields.Str(allow_none=True)
dns_config = SwaggerField(cls=k8s_schemas.V1PodDNSConfig, allow_none=True)
scheduler_name = fields.Str(allow_none=True)
priority_class_name = fields.Str(allow_none=True)
priority = fields.Int(allow_none=True)
restart_policy = fields.Str(allow_none=True,
validate=validate.OneOf(
["Always", "OnFailure", "Never"]))
@staticmethod
def schema_config():
return V1Environment
class SparkReplicaSchema(BaseCamelSchema):
replicas = fields.Int(allow_none=True)
environment = fields.Nested(EnvironmentSchema, allow_none=True)
init = fields.List(fields.Nested(InitSchema), allow_none=True)
sidecars = fields.List(SwaggerField(cls=k8s_schemas.V1Container),
allow_none=True)
container = SwaggerField(
cls=k8s_schemas.V1Container,
defaults={"name": MAIN_JOB_CONTAINER},
allow_none=True,
)
@staticmethod
def schema_config():
return V1SparkReplica
class HyperbandSchema(BaseCamelSchema):
kind = fields.Str(allow_none=True,
validate=validate.Equal(V1MatrixKind.HYPERBAND))
params = fields.Dict(keys=fields.Str(),
values=fields.Nested(HpParamSchema),
allow_none=True)
max_iterations = RefOrObject(fields.Int(validate=validate.Range(min=1)))
eta = RefOrObject(fields.Float(validate=validate.Range(min=0)))
resource = fields.Nested(OptimizationResourceSchema)
metric = fields.Nested(OptimizationMetricSchema)
resume = RefOrObject(fields.Boolean(allow_none=True))
seed = RefOrObject(fields.Int(allow_none=True))
concurrency = fields.Int(allow_none=True)
container = SwaggerField(
cls=k8s_schemas.V1Container,
defaults={"name": MAIN_JOB_CONTAINER},
allow_none=True,
)
early_stopping = fields.Nested(EarlyStoppingSchema,
many=True,
allow_none=True)
@staticmethod
def schema_config():
return V1Hyperband
class SparkSchema(BaseCamelSchema):
kind = fields.Str(allow_none=True,
validate=validate.Equal(V1RunKind.SPARK))
connections = fields.List(fields.Str(), allow_none=True)
volumes = fields.List(SwaggerField(cls=k8s_schemas.V1Volume),
allow_none=True)
type = fields.Str(allow_none=True,
validate=validate.OneOf(V1SparkType.allowable_values))
spark_version = fields.Str(allow_none=True)
python_version = fields.Str(allow_none=True,
validate=validate.OneOf(
V1SparkDeploy.allowable_values))
deploy_mode = fields.Str(allow_none=True)
main_class = fields.Str(allow_none=True)
main_application_file = fields.Str(allow_none=True)
arguments = fields.List(fields.Str(), allow_none=True)
hadoop_conf = fields.Dict(keys=fields.Str(),
values=fields.Str(),
allow_none=True)
spark_conf = fields.Dict(keys=fields.Str(),
values=fields.Str(),
allow_none=True)
hadoop_config_map = fields.Str(allow_none=True)
spark_config_map = fields.Str(allow_none=True)
executor = fields.Nested(SparkReplicaSchema, allow_none=True)
driver = fields.Nested(SparkReplicaSchema, allow_none=True)
@staticmethod
def schema_config():
return V1Spark
class HyperoptSchema(BaseCamelSchema):
kind = fields.Str(allow_none=True,
validate=validate.Equal(V1MatrixKind.HYPEROPT))
max_iterations = RefOrObject(fields.Int(allow_none=True))
algorithm = fields.Str(allow_none=True,
validate=validate.OneOf(["tpe", "rand", "anneal"]))
params = fields.Dict(keys=fields.Str(),
values=fields.Nested(HpParamSchema),
required=True)
num_runs = RefOrObject(fields.Int(required=True,
validate=validate.Range(min=1)),
required=True)
seed = RefOrObject(fields.Int(allow_none=True))
concurrency = RefOrObject(fields.Int(allow_none=True))
container = SwaggerField(
cls=k8s_schemas.V1Container,
defaults={"name": MAIN_JOB_CONTAINER},
allow_none=True,
)
early_stopping = fields.List(fields.Nested(EarlyStoppingSchema),
allow_none=True)
@staticmethod
def schema_config():
return V1Hyperopt
class JobSchema(BaseCamelSchema):
kind = fields.Str(allow_none=True, validate=validate.Equal(V1RunKind.JOB))
environment = fields.Nested(EnvironmentSchema, allow_none=True)
connections = fields.List(fields.Str(), allow_none=True)
volumes = fields.List(SwaggerField(cls=k8s_schemas.V1Volume), allow_none=True)
init = fields.List(fields.Nested(InitSchema), allow_none=True)
sidecars = fields.List(SwaggerField(cls=k8s_schemas.V1Container), allow_none=True)
container = SwaggerField(
cls=k8s_schemas.V1Container,
defaults={"name": MAIN_JOB_CONTAINER},
allow_none=True,
)
@staticmethod
def schema_config():
return V1Job
class BayesSchema(BaseCamelSchema):
kind = fields.Str(allow_none=True,
validate=validate.Equal(V1MatrixKind.BAYES))
utility_function = fields.Nested(UtilityFunctionSchema, allow_none=True)
num_initial_runs = RefOrObject(fields.Int(), required=True)
max_iterations = RefOrObject(fields.Int(validate=validate.Range(min=1)))
metric = fields.Nested(OptimizationMetricSchema, required=True)
params = fields.Dict(keys=fields.Str(),
values=fields.Nested(HpParamSchema),
required=True)
seed = RefOrObject(fields.Int(allow_none=True))
concurrency = fields.Int(allow_none=True)
container = SwaggerField(
cls=k8s_schemas.V1Container,
defaults={"name": MAIN_JOB_CONTAINER},
allow_none=True,
)
early_stopping = fields.Nested(EarlyStoppingSchema,
many=True,
allow_none=True)
@staticmethod
def schema_config():
return V1Bayes
@validates_schema
def validate_matrix(self, data, **kwargs):
"""Validates matrix data and creates the config objects"""
validate_matrix(data.get("params"))
class ServiceSchema(BaseCamelSchema):
kind = fields.Str(allow_none=True, validate=validate.Equal(V1RunKind.SERVICE))
environment = fields.Nested(EnvironmentSchema, allow_none=True)
connections = fields.List(fields.Str(), allow_none=True)
volumes = fields.List(SwaggerField(cls=k8s_schemas.V1Volume), allow_none=True)
init = fields.List(fields.Nested(InitSchema), allow_none=True)
sidecars = fields.List(SwaggerField(cls=k8s_schemas.V1Container), allow_none=True)
container = SwaggerField(
cls=k8s_schemas.V1Container,
defaults={"name": MAIN_JOB_CONTAINER},
allow_none=True,
)
ports = RefOrObject(fields.List(fields.Int(), allow_none=True))
rewrite_path = RefOrObject(fields.Bool(allow_none=True))
@staticmethod
def schema_config():
return V1Service
class PolyaxonInitContainerSchema(BaseCamelSchema):
image = fields.Str(required=True)
image_tag = fields.Str(required=True)
image_pull_policy = fields.Str(allow_none=True)
resources = SwaggerField(cls=k8s_schemas.V1ResourceRequirements, allow_none=True)
@staticmethod
def schema_config():
return V1PolyaxonInitContainer
class EnvironmentSchema(BaseCamelSchema):
labels = fields.Dict(allow_none=True)
annotations = fields.Dict(allow_none=True)
node_selector = fields.Dict(allow_none=True)
affinity = SwaggerField(cls=k8s_schemas.V1Affinity, allow_none=True)
tolerations = fields.List(SwaggerField(cls=k8s_schemas.V1Toleration),
allow_none=True)
node_name = fields.Str(allow_none=True)
service_account_name = fields.Str(allow_none=True)
host_aliases = fields.List(SwaggerField(cls=k8s_schemas.V1HostAlias),
allow_none=True)
security_context = SwaggerField(cls=k8s_schemas.V1SecurityContext,
allow_none=True)
image_pull_secrets = fields.List(fields.Str(), allow_none=True)
host_network = fields.Bool(allow_none=True)
host_pid = fields.Bool(allow_none=True, data_key="hostPID")
dns_policy = fields.Str(allow_none=True)
dns_config = SwaggerField(cls=k8s_schemas.V1PodDNSConfig, allow_none=True)
scheduler_name = fields.Str(allow_none=True)
priority_class_name = fields.Str(allow_none=True)
priority = fields.Int(allow_none=True)
restart_policy = fields.Str(allow_none=True,
validate=validate.OneOf(
["Always", "OnFailure", "Never"]))
@staticmethod
def schema_config():
return V1Environment
@validates_schema
@check_partial
def validate_param(self, values, **kwargs):
labels = values.get("labels")
if labels:
values["labels"] = sanitize_string_dict(labels)
annotations = values.get("annotations")
if annotations:
values["annotations"] = sanitize_string_dict(annotations)
node_selector = sanitize_string_dict(values.get("nodeSelector"))
if node_selector:
values["nodeSelector"] = sanitize_string_dict(node_selector)
return values
class PolyaxonSidecarContainerSchema(BaseCamelSchema):
image = fields.Str(allow_none=True)
image_tag = fields.Str(allow_none=True)
image_pull_policy = fields.Str(allow_none=True)
resources = SwaggerField(cls=k8s_schemas.V1ResourceRequirements,
allow_none=True)
sleep_interval = fields.Int(allow_none=True)
sync_interval = fields.Int(allow_none=True)
monitor_logs = fields.Bool(allow_none=True)
@staticmethod
def schema_config():
return V1PolyaxonSidecarContainer
class DagSchema(BaseCamelSchema):
kind = fields.Str(allow_none=True, validate=validate.Equal(V1RunKind.DAG))
operations = fields.List(fields.Nested("OperationSchema"))
components = fields.List(fields.Nested("ComponentSchema"))
environment = fields.Nested(EnvironmentSchema, allow_none=True)
connections = fields.List(fields.Str(), allow_none=True)
volumes = fields.List(SwaggerField(cls=k8s_schemas.V1Volume), allow_none=True)
concurrency = fields.Int(allow_none=True)
early_stopping = fields.List(fields.Nested(EarlyStoppingSchema), allow_none=True)
@staticmethod
def schema_config():
return V1Dag
class DaskSchema(BaseCamelSchema):
kind = fields.Str(allow_none=True, validate=validate.Equal(V1RunKind.DASK))
threads = fields.Int(allow_none=True)
scale = fields.Int(allow_none=True)
adapt_min = fields.Int(allow_none=True)
adapt_max = fields.Int(allow_none=True)
adapt_interval = fields.Int(allow_none=True)
environment = fields.Nested(EnvironmentSchema, allow_none=True)
connections = fields.List(fields.Str(), allow_none=True)
volumes = fields.List(SwaggerField(cls=k8s_schemas.V1Volume),
allow_none=True)
init = fields.List(fields.Nested(InitSchema), allow_none=True)
sidecars = fields.List(SwaggerField(cls=k8s_schemas.V1Container),
allow_none=True)
container = SwaggerField(
cls=k8s_schemas.V1Container,
defaults={"name": MAIN_JOB_CONTAINER},
allow_none=True,
)
@staticmethod
def schema_config():
return V1Dask
class ServiceSchema(BaseCamelSchema):
enabled = fields.Bool(allow_none=True)
image = fields.Str(allow_none=True)
image_tag = fields.Str(allow_none=True)
image_pull_policy = fields.Str(allow_none=True)
replicas = fields.Int(allow_none=True)
concurrency = fields.Int(allow_none=True)
resources = SwaggerField(cls=k8s_schemas.V1ResourceRequirements, allow_none=True)
class Meta:
unknown = EXCLUDE
@staticmethod
def schema_config():
return Service
class InitSchema(BaseCamelSchema):
artifacts = fields.Nested(ArtifactsTypeSchema, allow_none=True)
git = fields.Nested(GitTypeSchema, allow_none=True)
dockerfile = fields.Nested(DockerfileTypeSchema, allow_none=True)
file = fields.Nested(FileTypeSchema, allow_none=True)
connection = fields.Str(allow_none=True)
path = fields.Str(allow_none=True)
container = SwaggerField(
cls=k8s_schemas.V1Container,
defaults={
"name": generate_container_name(prefix=POLYAXON_INIT_PREFIX)
},
allow_none=True,
)
@staticmethod
def schema_config():
return V1Init
@validates_schema
@check_partial
def validate_init(self, data, **kwargs):
artifacts = data.get("artifacts")
git = data.get("git")
dockerfile = data.get("dockerfile")
file = data.get("file")
connection = data.get("connection")
schemas = 0
if artifacts:
schemas += 1
if git:
schemas += 1
if dockerfile:
schemas += 1
if file:
schemas += 1
if schemas > 1:
raise ValidationError(
"One of artifacts, git, file, or dockerfile can be set")
if not connection and git and not git.url:
raise ValidationError(
"git field without a valid url requires a connection is required to be passed."
)
class IterativeSchema(BaseCamelSchema):
kind = fields.Str(allow_none=True, validate=validate.Equal(V1MatrixKind.ITERATIVE))
num_iterations = RefOrObject(
fields.Int(required=True, validate=validate.Range(min=1)), required=True
)
concurrency = fields.Int(allow_none=True)
params = fields.Dict(
keys=fields.Str(), values=fields.Nested(MatrixSchema), allow_none=True
)
seed = RefOrObject(fields.Int(allow_none=True))
container = SwaggerField(
cls=k8s_schemas.V1Container,
defaults={"name": MAIN_JOB_CONTAINER},
allow_none=True,
)
early_stopping = fields.Nested(EarlyStoppingSchema, many=True, allow_none=True)
@staticmethod
def schema_config():
return V1Iterative
class InitSchema(BaseCamelSchema):
artifacts = fields.Nested(ArtifactsTypeSchema, allow_none=True)
git = fields.Nested(GitTypeSchema, allow_none=True)
dockerfile = fields.Nested(DockerfileTypeSchema, allow_none=True)
connection = fields.Str(allow_none=True)
path = fields.Str(allow_none=True)
container = SwaggerField(
cls=k8s_schemas.V1Container,
defaults={"name": INIT_CONTAINER.format(random.randint(1, 100))},
allow_none=True,
)
@staticmethod
def schema_config():
return V1Init
@validates_schema
def validate_init(self, data, **kwargs):
artifacts = data.get("artifacts")
git = data.get("git")
dockerfile = data.get("dockerfile")
connection = data.get("connection")
schemas = 0
if artifacts:
schemas += 1
if git:
schemas += 1
if dockerfile:
schemas += 1
if schemas > 1:
raise ValidationError(
"One of artifacts, git, or dockerfile can be set")
if not connection and git and not git.url:
raise ValidationError(
"git field without a valid url requires a connection is required to be passed."
)