class JobLabelSchema(Schema):
app = fields.Str(allow_none=True)
project_name = fields.Str()
experiment_group_name = fields.Str(allow_none=True)
experiment_name = fields.Str(allow_none=True)
job_name = fields.Str(allow_none=True)
project_uuid = UUID()
experiment_group_uuid = UUID(allow_none=True)
experiment_uuid = UUID(allow_none=True)
job_uuid = UUID(allow_none=True)
task_type = fields.Str(allow_none=True)
task_idx = fields.Str(allow_none=True)
role = fields.Str()
type = fields.Str()
class Meta:
ordered = True
@post_load
def make(self, data):
return JobLabelConfig(**data)
@post_dump
def unmake(self, data):
return JobLabelConfig.remove_reduced_attrs(data)
class ExperimentJobSchema(Schema):
sequence = fields.Int(allow_none=True)
uuid = UUID()
unique_name = fields.Str(allow_none=True)
role = fields.Str(allow_none=True)
experiment = UUID()
experiment_name = fields.Str()
last_status = fields.Str(allow_none=True)
is_running = fields.Bool(allow_none=True)
is_done = fields.Bool(allow_none=True)
created_at = fields.LocalDateTime()
updated_at = fields.LocalDateTime()
started_at = fields.LocalDateTime(allow_none=True)
finished_at = fields.LocalDateTime(allow_none=True)
total_run = fields.Str(allow_none=True)
resources = fields.Nested(PodResourcesSchema, allow_none=True)
definition = fields.Dict(allow_none=True)
class Meta:
ordered = True
@post_load
def make(self, data):
return ExperimentJobConfig(**data)
@post_dump
def unmake(self, data):
return ExperimentJobConfig.remove_reduced_attrs(data)
class ExperimentGroupSchema(Schema):
sequence = fields.Int(allow_none=True)
uuid = UUID(allow_none=True)
unique_name = fields.Str(allow_none=True)
user = fields.Str(validate=validate.Regexp(regex=r'^[-a-zA-Z0-9_]+\Z'),
allow_none=True)
project = UUID(allow_none=True)
project_name = fields.Str(allow_none=True)
description = fields.Str(allow_none=True)
content = fields.Str()
created_at = fields.LocalDateTime(allow_none=True)
updated_at = fields.LocalDateTime(allow_none=True)
concurrency = fields.Int(allow_none=True)
num_experiments = fields.Int(allow_none=True)
num_scheduled_experiments = fields.Int(allow_none=True)
num_pending_experiments = fields.Int(allow_none=True)
num_running_experiments = fields.Int(allow_none=True)
num_succeeded_experiments = fields.Int(allow_none=True)
num_failed_experiments = fields.Int(allow_none=True)
num_stopped_experiments = fields.Int(allow_none=True)
experiments = fields.Nested(ExperimentSchema, many=True, allow_none=True)
class Meta:
ordered = True
@post_load
def make(self, data):
return ExperimentGroupConfig(**data)
@post_dump
def unmake(self, data):
return ExperimentGroupConfig.remove_reduced_attrs(data)
class ClusterNodeSchema(Schema):
uuid = UUID()
name = fields.Str(allow_none=True)
cluster = UUID()
hostname = fields.Str(allow_none=True)
role = fields.Str()
docker_version = fields.Str(allow_none=True)
kubelet_version = fields.Str()
os_image = fields.Str()
kernel_version = fields.Str()
schedulable_taints = fields.Bool()
schedulable_state = fields.Bool()
memory = fields.Int()
n_cpus = fields.Int()
n_gpus = fields.Int()
status = fields.Str()
gpus = fields.Nested(NodeGPUSchema, many=True, allow_none=True)
@post_load
def make(self, data):
return ClusterNodeConfig(**data)
@post_dump
def unmake(self, data):
return ClusterNodeConfig.remove_reduced_attrs(data)
class ExperimentSchema(Schema):
sequence = fields.Int(allow_none=True)
uuid = UUID(allow_none=True)
user = fields.Str(validate=validate.Regexp(regex=r'^[-a-zA-Z0-9_]+\Z'),
allow_none=True)
project = UUID(allow_none=True)
group = UUID(allow_none=True)
description = fields.Str(allow_none=True)
last_status = fields.Str(allow_none=True)
started_at = fields.DateTime(allow_none=True)
finished_at = fields.DateTime(allow_none=True)
is_clone = fields.Bool(allow_none=True)
config = fields.Dict(allow_none=True)
content = fields.Str(allow_none=True)
num_jobs = fields.Int(allow_none=True)
jobs = fields.Nested(ExperimentJobSchema, many=True, allow_none=True)
class Meta:
ordered = True
@post_load
def make(self, data):
return ExperimentConfig(**data)
@post_dump
def unmake(self, data):
return ExperimentConfig.remove_reduced_attrs(data)
class NodeGPUSchema(BaseSchema):
index = fields.Int()
name = fields.Str()
uuid = UUID()
memory = fields.Int()
serial = fields.Str()
cluster_node = UUID()
@staticmethod
def schema_config():
return NodeGPUConfig
class NodeGPUSchema(Schema):
uuid = UUID()
serial = fields.Str()
name = fields.Str()
device = fields.Str()
memory = fields.Int()
cluster_node = UUID()
@post_load
def make(self, data):
return NodeGPUConfig(**data)
@post_dump
def unmake(self, data):
return NodeGPUConfig.remove_reduced_attrs(data)
class ProjectSchema(BaseSchema):
id = fields.Int(allow_none=True)
name = fields.Str(validate=validate.Regexp(regex=r'^[-a-zA-Z0-9_]+\Z'))
user = fields.Str(validate=validate.Regexp(regex=r'^[-a-zA-Z0-9_]+\Z'),
allow_none=True)
owner = fields.Str(validate=validate.Regexp(regex=r'^[-a-zA-Z0-9_]+\Z'),
allow_none=True)
unique_name = fields.Str(allow_none=True)
uuid = UUID(allow_none=True)
description = fields.Str(allow_none=True)
tags = fields.List(fields.Str(), allow_none=True)
is_public = fields.Boolean(allow_none=True)
has_code = fields.Bool(allow_none=True)
created_at = fields.LocalDateTime(allow_none=True)
updated_at = fields.LocalDateTime(allow_none=True)
num_experiments = fields.Int(allow_none=True)
num_independent_experiments = fields.Int(allow_none=True)
num_experiment_groups = fields.Int(allow_none=True)
num_jobs = fields.Int(allow_none=True)
num_builds = fields.Int(allow_none=True)
has_tensorboard = fields.Bool(allow_none=True)
has_notebook = fields.Bool(allow_none=True)
experiment_groups = fields.Nested(GroupSchema, many=True, allow_none=True)
experiments = fields.Nested(ExperimentSchema, many=True, allow_none=True)
@staticmethod
def schema_config():
return ProjectConfig
class BaseJobSchema(BaseSchema):
id = fields.Int(allow_none=True)
uuid = UUID(allow_none=True)
unique_name = fields.Str(allow_none=True)
pod_id = fields.Str(allow_none=True)
name = fields.Str(validate=validate.Regexp(regex=r'^[-a-zA-Z0-9_]+\Z'),
allow_none=True)
user = fields.Str(validate=validate.Regexp(regex=r'^[-a-zA-Z0-9_]+\Z'),
allow_none=True)
project = fields.Str(allow_none=True)
build_job = fields.Str(allow_none=True)
description = fields.Str(allow_none=True)
tags = fields.List(fields.Str(), allow_none=True)
last_status = fields.Str(allow_none=True)
created_at = fields.LocalDateTime(allow_none=True)
updated_at = fields.LocalDateTime(allow_none=True)
started_at = fields.LocalDateTime(allow_none=True)
finished_at = fields.LocalDateTime(allow_none=True)
total_run = fields.Str(allow_none=True)
is_clone = fields.Bool(allow_none=True)
content = fields.Str(allow_none=True)
is_managed = fields.Bool(allow_none=True)
ttl = fields.Int(allow_none=True)
resources = fields.Nested(PodResourcesSchema, allow_none=True)
definition = fields.Dict(allow_none=True)
@staticmethod
def schema_config():
return BaseJobConfig
class EnvironmentSchema(Schema):
cluster_uuid = UUID(allow_none=True)
resources = fields.Nested(PodResourcesSchema, allow_none=True)
tensorflow = fields.Nested(TensorflowSchema, allow_none=True)
horovod = fields.Nested(HorovodSchema, allow_none=True)
mxnet = fields.Nested(MXNetSchema, allow_none=True)
pytorch = fields.Nested(PytorchSchema, allow_none=True)
class Meta:
ordered = True
@post_load
def make(self, data):
return EnvironmentConfig(**data)
@post_dump
def unmake(self, data):
return EnvironmentConfig.remove_reduced_attrs(data)
@validates_schema
def validate_quantity(self, data):
validate_frameworks([data.get('tensorflow'),
data.get('mxnet'),
data.get('pytorch'),
data.get('horovod')])
class ProjectSchema(Schema):
name = fields.Str(validate=validate.Regexp(regex=r'^[-a-zA-Z0-9_]+\Z'))
user = fields.Str(validate=validate.Regexp(regex=r'^[-a-zA-Z0-9_]+\Z'),
allow_none=True)
uuid = UUID(allow_none=True)
description = fields.Str(allow_none=True)
is_public = fields.Boolean(allow_none=True)
has_code = fields.Bool(allow_none=True)
created_at = fields.DateTime(allow_none=True)
num_experiments = fields.Int(allow_none=True)
num_experiment_groups = fields.Int(allow_none=True)
experiment_groups = fields.Nested(ExperimentGroupSchema,
many=True,
allow_none=True)
experiments = fields.Nested(ExperimentSchema, many=True, allow_none=True)
class Meta:
ordered = True
@post_load
def make(self, data):
return ProjectConfig(**data)
@post_dump
def unmake(self, data):
return ProjectConfig.remove_reduced_attrs(data)
class ExperimentJobSchema(Schema):
uuid = UUID()
experiment = UUID()
created_at = fields.DateTime()
definition = fields.Str()
class Meta:
ordered = True
@post_load
def make(self, data):
return ExperimentJobConfig(**data)
@post_dump
def unmake(self, data):
return ExperimentJobConfig.remove_reduced_attrs(data)
class ExperimentGroupSchema(Schema):
uuid = UUID(allow_none=True)
content = fields.Str()
project = UUID(allow_none=True)
experiments = fields.Nested(ExperimentSchema, many=True, allow_none=True)
class Meta:
ordered = True
@post_load
def make(self, data):
return ExperimentGroupConfig(**data)
@post_dump
def unmake(self, data):
return ExperimentGroupConfig.remove_reduced_attrs(data)
class ExperimentMetricSchema(Schema):
uuid = UUID()
experiment = UUID()
created_at = fields.LocalDateTime()
values = fields.Dict()
class Meta:
ordered = True
@post_load
def make(self, data):
return ExperimentMetricConfig(**data)
@post_dump
def unmake(self, data):
return ExperimentMetricConfig.remove_reduced_attrs(data)
class PolyaxonClusterSchema(Schema):
uuid = UUID()
user = UUID()
version_api = fields.Dict()
nodes = fields.Nested(ClusterNodeSchema, many=True, allow_none=True)
class Meta:
ordered = True
@post_load
def make(self, data):
return PolyaxonClusterConfig(**data)
@post_dump
def unmake(self, data):
return PolyaxonClusterConfig.remove_reduced_attrs(data)
class EnvironmentSchema(Schema):
cluster_uuid = UUID(allow_none=True)
n_workers = fields.Int(allow_none=True)
n_ps = fields.Int(allow_none=True)
delay_workers_by_global_step = fields.Bool(allow_none=True)
run_config = fields.Nested(RunSchema, allow_none=True)
resources = fields.Nested(PodResourcesSchema, allow_none=True)
master_resources = fields.Nested(PodResourcesSchema, allow_none=True)
default_worker_config = fields.Nested(SessionSchema, allow_none=True)
default_worker_resources = fields.Nested(PodResourcesSchema,
allow_none=True)
default_ps_config = fields.Nested(SessionSchema, allow_none=True)
default_ps_resources = fields.Nested(PodResourcesSchema, allow_none=True)
worker_configs = fields.Nested(SessionSchema, many=True, allow_none=True)
worker_resources = fields.Nested(PodResourcesSchema,
many=True,
allow_none=True)
ps_configs = fields.Nested(SessionSchema, many=True, allow_none=True)
ps_resources = fields.Nested(PodResourcesSchema,
many=True,
allow_none=True)
class Meta:
ordered = True
@post_load
def make(self, data):
return EnvironmentConfig(**data)
@post_dump
def unmake(self, data):
return EnvironmentConfig.remove_reduced_attrs(data)
class ContainerResourcesSchema(BaseSchema):
job_uuid = UUID()
experiment_uuid = UUID()
job_name = fields.Str()
container_id = fields.Str()
n_cpus = fields.Int()
cpu_percentage = fields.Float()
percpu_percentage = fields.List(fields.Float(), allow_none=True)
memory_used = fields.Int()
memory_limit = fields.Int()
gpu_resources = fields.Nested(ContainerGPUResourcesSchema,
many=True,
allow_none=True)
@staticmethod
def schema_config():
return ContainerResourcesConfig
class ExperimentStatusSchema(Schema):
uuid = UUID()
experiment = UUID()
created_at = fields.LocalDateTime()
status = fields.Str()
message = fields.Str(allow_none=True)
class Meta:
ordered = True
@post_load
def make(self, data):
return ExperimentStatusConfig(**data)
@post_dump
def unmake(self, data):
return ExperimentStatusConfig.remove_reduced_attrs(data)
class ExperimentSchema(Schema):
name = fields.Str()
uuid = UUID(allow_none=True)
project = UUID(allow_none=True)
group = UUID(allow_none=True)
jobs = fields.Nested(ExperimentJobSchema, many=True, allow_none=True)
description = fields.Str(allow_none=True)
class Meta:
ordered = True
@post_load
def make(self, data):
return ExperimentConfig(**data)
@post_dump
def unmake(self, data):
return ExperimentConfig.remove_reduced_attrs(data)
class JobLabelSchema(BaseSchema):
app = fields.Str(allow_none=True)
project_name = fields.Str()
experiment_group_name = fields.Str(allow_none=True)
experiment_name = fields.Str(allow_none=True)
job_name = fields.Str(allow_none=True)
project_uuid = UUID()
experiment_group_uuid = UUID(allow_none=True)
experiment_uuid = UUID(allow_none=True)
job_uuid = UUID(allow_none=True)
task_type = fields.Str(allow_none=True)
task_idx = fields.Str(allow_none=True)
role = fields.Str()
type = fields.Str()
@staticmethod
def schema_config():
return JobLabelConfig
class DatasetSchema(BaseSchema):
uuid = UUID()
name = fields.Str()
description = fields.Str(allow_none=True)
is_public = fields.Boolean()
@staticmethod
def schema_config():
return DatasetConfig
class EnvironmentSchema(PodEnvironmentSchema):
cluster_uuid = UUID(allow_none=True)
persistence = fields.Nested(PersistenceSchema, allow_none=True)
outputs = fields.Nested(OutputsSchema, allow_none=True)
secret_refs = fields.List(fields.Str(), allow_none=True)
configmap_refs = fields.List(fields.Str(), allow_none=True)
@staticmethod
def schema_config():
return EnvironmentConfig
class ExperimentMetricSchema(BaseSchema):
id = fields.Int()
uuid = UUID()
experiment = fields.Int()
created_at = fields.LocalDateTime()
values = fields.Dict()
@staticmethod
def schema_config():
return ExperimentMetricConfig
class ContainerResourcesSchema(Schema):
job_uuid = UUID()
experiment_uuid = UUID()
container_id = fields.Str()
cpu_percentage = fields.Float()
percpu_percentage = fields.List(fields.Float(), allow_none=True)
memory_used = fields.Int()
memory_limit = fields.Int()
gpu_resources = fields.Nested(ContainerGPUResourcesSchema, allow_none=True)
class Meta:
ordered = True
@post_load
def make(self, data):
return ContainerResourcesConfig(**data)
@post_dump
def unmake(self, data):
return ContainerResourcesConfig.remove_reduced_attrs(data)
class ExperimentJobStatusSchema(BaseSchema):
id = fields.Int()
uuid = UUID()
job = fields.Int()
created_at = fields.LocalDateTime()
status = fields.Str()
message = fields.Str(allow_none=True)
details = fields.Dict(allow_none=True)
@staticmethod
def schema_config():
return ExperimentJobStatusConfig
class ExperimentStatusSchema(BaseSchema):
id = fields.Int()
uuid = UUID()
experiment = fields.Int()
created_at = fields.LocalDateTime()
status = fields.Str()
message = fields.Str(allow_none=True)
traceback = fields.Str(allow_none=True)
@staticmethod
def schema_config():
return ExperimentStatusConfig
class DataSchema(BaseSchema):
uuid = UUID()
name = fields.Str()
created_at = fields.LocalDateTime()
description = fields.Str(allow_none=True)
details = fields.Nested(DataDetailsSchema)
version = fields.Str(allow_none=True)
resource_id = fields.Str(allow_none=True)
@staticmethod
def schema_config():
return DataConfig
class ExperimentGroupSchema(Schema):
uuid = UUID(allow_none=True)
sequence = fields.Int(allow_none=True)
user = fields.Str(validate=validate.Regexp(regex=r'^[-a-zA-Z0-9_]+\Z'),
allow_none=True)
description = fields.Str(allow_none=True)
content = fields.Str()
project = UUID(allow_none=True)
created_at = fields.DateTime(allow_none=True)
num_experiments = fields.Int(allow_none=True)
experiments = fields.Nested(ExperimentSchema, many=True, allow_none=True)
class Meta:
ordered = True
@post_load
def make(self, data):
return ExperimentGroupConfig(**data)
@post_dump
def unmake(self, data):
return ExperimentGroupConfig.remove_reduced_attrs(data)
class ExperimentSchema(Schema):
sequence = fields.Int(allow_none=True)
uuid = UUID(allow_none=True)
unique_name = fields.Str(allow_none=True)
user = fields.Str(validate=validate.Regexp(regex=r'^[-a-zA-Z0-9_]+\Z'),
allow_none=True)
project = UUID(allow_none=True)
project_name = fields.Str(allow_none=True)
experiment_group = UUID(allow_none=True)
experiment_group_name = fields.Str(allow_none=True)
description = fields.Str(allow_none=True)
last_status = fields.Str(allow_none=True)
last_metric = fields.Dict(allow_none=True)
created_at = fields.LocalDateTime(allow_none=True)
updated_at = fields.LocalDateTime(allow_none=True)
started_at = fields.LocalDateTime(allow_none=True)
finished_at = fields.LocalDateTime(allow_none=True)
total_run = fields.Str(allow_none=True)
is_running = fields.Bool(allow_none=True)
is_done = fields.Bool(allow_none=True)
is_clone = fields.Bool(allow_none=True)
config = fields.Dict(allow_none=True)
content = fields.Str(allow_none=True)
num_jobs = fields.Int(allow_none=True)
declarations = fields.Dict(allow_none=True)
resources = fields.Nested(PodResourcesSchema, allow_none=True)
jobs = fields.Nested(ExperimentJobSchema, many=True, allow_none=True)
class Meta:
ordered = True
@post_load
def make(self, data):
return ExperimentConfig(**data)
@post_dump
def unmake(self, data):
return ExperimentConfig.remove_reduced_attrs(data)
class DatasetSchema(Schema):
uuid = UUID()
name = fields.Str()
description = fields.Str(allow_none=True)
is_public = fields.Boolean()
class Meta:
ordered = True
@post_load
def make(self, data):
return DatasetConfig(**data)
@post_dump
def unmake(self, data):
return DatasetConfig.remove_reduced_attrs(data)