def create_specification(
cls, # pylint:disable=arguments-differ
build_config,
run_config,
to_dict=True):
try:
specification = BaseRunSpecification.create_specification(
build_config=build_config, to_dict=True)
except PolyaxonConfigurationError:
raise PolyaxonConfigurationError(
'Create specification expects a dict or an instance of BuildConfig.'
)
if isinstance(run_config, RunConfig):
r_config = run_config.to_light_dict()
elif isinstance(run_config, Mapping):
r_config = RunConfig.from_dict(run_config, unknown=EXCLUDE)
r_config = r_config.to_light_dict()
else:
raise PolyaxonConfigurationError(
'Create specification expects a dict or an instance of RunConfig.'
)
specification[cls.KIND] = cls._SPEC_KIND
specification[cls.RUN] = r_config
if to_dict:
return specification
return cls.read(specification)
def __init__(self, values):
self._values = to_list(values)
self._data = reader.read(self._values)
Parser.check_data(spec=self, data=self._data)
headers = Parser.get_headers(spec=self, data=self._data)
matrix = Parser.get_matrix(spec=self, data=self._data)
try:
self._matrix = validator.validate_matrix(matrix)
except ValidationError as e:
raise PolyaxonConfigurationError(e)
try:
self._headers = validator.validate_headers(spec=self, data=headers)
except ValidationError as e:
raise PolyaxonConfigurationError(e)
self._parsed_data = []
self._validated_data = []
self._experiment_specs = []
matrix_declarations = self.matrix_declarations if self.matrix_declarations else [
{}
]
for i, matrix_declaration in enumerate(matrix_declarations):
parsed_data = Parser.parse(self, self._data, matrix_declaration)
self._validated_data.append(
validator.validate(spec=self, data=parsed_data))
self._parsed_data.append(parsed_data)
self._experiment_specs.append(
Specification(experiment=i, values=parsed_data))
def read(config_values):
"""Reads an ordered list of configuration values and deep merge the values in reverse order."""
if not config_values:
raise PolyaxonConfigurationError(
'Cannot read config_value: `{}`'.format(config_values))
config_values = to_list(config_values)
config = {}
for config_value in config_values:
if not isinstance(config_value, (Mapping, six.string_types)):
raise PolyaxonConfigurationError(
"Expects Mapping, string, or list of Mapping/string instances, "
"received {} instead".format(type(config_value)))
if isinstance(config_value, Mapping):
config_results = config_value
elif os.path.isfile(config_value):
config_results = _read_from_file(config_value)
else:
# try reading a stream of yaml or json
try:
config_results = _read_from_stream(config_value)
except ScannerError:
raise PolyaxonConfigurationError(
'Received non valid yaml stream: `{}`'.format(
config_value))
if config_results and isinstance(config_results, Mapping):
config = deep_update(config, config_results)
else:
raise PolyaxonConfigurationError(
'Cannot read config_value: `{}`'.format(config_value))
return config
def _read_from_json(f_path, is_stream=False):
if is_stream:
try:
return json.loads(f_path)
except ValueError as e:
raise PolyaxonConfigurationError(e)
try:
return json.loads(open(f_path).read())
except ValueError as e:
raise PolyaxonConfigurationError(e)
def create_specification(
cls, # pylint:disable=arguments-differ
build_config,
to_dict=True):
from polyaxon_schemas.ops.build import BuildConfig
if isinstance(build_config, BuildConfig):
b_config = build_config.to_light_dict()
elif isinstance(build_config, Mapping):
# Since the objective is to create the build spec from other specs
# we drop any extra attrs
b_config = BuildConfig.from_dict(build_config, unknown=EXCLUDE)
b_config = b_config.to_light_dict()
else:
raise PolyaxonConfigurationError(
'Create specification expects a dict or an instance of BuildConfig.'
)
specification = {
cls.VERSION: 1,
cls.KIND: cls._SPEC_KIND,
cls.BUILD: b_config,
}
if to_dict:
return specification
return cls.read(specification)
def get_persistent_volume_spec(namespace,
volume,
run_type,
access_modes='ReadWriteOnce',
persistent_volume_reclaim_policy='Recycle'):
capacity = {'storage': STORAGE_BY_VOLUME[volume]}
access_modes = to_list(access_modes)
vol_path = get_vol_path(volume, run_type)
if run_type == RunTypes.MINIKUBE:
params = get_host_path_pvol(vol_path)
elif run_type == RunTypes.KUBERNETES:
params = get_nfs_pvol(vol_path)
else:
raise PolyaxonConfigurationError(
'Run type `{}` is not allowed.'.format(run_type))
volc_name = constants.VOLUME_CLAIM_NAME.format(vol_name=volume)
claim_ref = client.V1ObjectReference(
api_version=k8s_constants.K8S_API_VERSION_V1,
kind=k8s_constants.K8S_PERSISTENT_VOLUME_CLAIM_KIND,
name=volc_name,
namespace=namespace)
return client.V1PersistentVolumeSpec(
capacity=capacity,
access_modes=access_modes,
persistent_volume_reclaim_policy=persistent_volume_reclaim_policy,
claim_ref=claim_ref,
**params)
def create_specification(cls,
build_config,
secret_refs=None,
configmap_refs=None,
to_dict=True):
if isinstance(build_config, BuildConfig):
config = build_config.to_light_dict()
elif isinstance(build_config, Mapping):
# Since the objective is to create the build spec from other specs
# we drop any extra attrs
config = BuildConfig.from_dict(build_config, unknown=EXCLUDE)
config = config.to_light_dict()
else:
raise PolyaxonConfigurationError(
'Create specification expects a dict or an instance of BuildConfig.'
)
specification = {
cls.VERSION: 1,
cls.KIND: cls._SPEC_KIND,
}
specification.update(config)
env = {}
if secret_refs:
env['secret_refs'] = secret_refs
if configmap_refs:
env['configmap_refs'] = configmap_refs
if env:
specification[cls.ENVIRONMENT] = env
if to_dict:
return specification
return cls.read(specification)
def _read_from_file(f_path):
_, ext = os.path.splitext(f_path)
if ext in ('.yml', '.yaml'):
return _read_from_yml(f_path)
elif ext == '.json':
return _read_from_json(f_path)
raise PolyaxonConfigurationError(
"Expects a file with extension: `.yml`, `.yaml`, or `json`, "
"received instead `{}`".format(ext))
def __init__(self, values):
self._values = to_list(values)
try:
self._data = rhea.read(self._values)
except rhea.RheaError as e:
raise PolyaxonConfigurationError(e)
self.check_data()
headers = Parser.get_headers(spec=self, data=self._data)
try:
self._headers = validator.validate_headers(spec=self, data=headers)
except ValidationError as e:
raise PolyaxonConfigurationError(e)
self._parsed_data = None
self._validated_data = None
self._config = None
self._set_parsed_data()
self._extra_validation()
def search_algorithm(self):
if not self.matrix:
raise PolyaxonConfigurationError(
'a search algorithm requires a matrix definition.')
if self.settings.random_search:
return SearchAlgorithms.RANDOM
if self.settings.hyperband:
return SearchAlgorithms.HYPERBAND
# Default value
return SearchAlgorithms.GRID
def __init__(self,
namespace,
project_name,
experiment_group_name,
experiment_name,
project_uuid,
experiment_group_uuid,
experiment_uuid,
original_name=None,
cloning_strategy=None,
job_container_name=None,
job_docker_image=None,
sidecar_container_name=None,
sidecar_docker_image=None,
init_container_name=None,
init_docker_image=None,
role_label=None,
type_label=None,
ports=None,
use_sidecar=False,
sidecar_config=None,
log_level=None,
declarations=None):
self.namespace = namespace
self.project_name = project_name
self.experiment_group_name = experiment_group_name
self.experiment_name = experiment_name
self.project_uuid = project_uuid
self.experiment_group_uuid = experiment_group_uuid
self.experiment_uuid = experiment_uuid
self.original_name = original_name
self.cloning_strategy = cloning_strategy
self.job_container_name = job_container_name or settings.CONTAINER_NAME_EXPERIMENT_JOB
self.job_docker_image = job_docker_image or settings.JOB_DOCKER_NAME
self.sidecar_container_name = sidecar_container_name or settings.CONTAINER_NAME_SIDECAR
self.sidecar_docker_image = sidecar_docker_image or settings.JOB_SIDECAR_DOCKER_IMAGE
self.init_container_name = init_container_name or settings.CONTAINER_NAME_INIT
self.init_docker_image = init_docker_image or settings.JOB_INIT_DOCKER_IMAGE
self.role_label = role_label or settings.ROLE_LABELS_WORKER
self.type_label = type_label or settings.TYPE_LABELS_EXPERIMENT
self.app_label = settings.APP_LABELS_EXPERIMENT
self.ports = ports or [constants.DEFAULT_PORT]
self.use_sidecar = use_sidecar
if use_sidecar and not sidecar_config:
raise PolyaxonConfigurationError(
'In order to use a `sidecar_config` is required. '
'The `sidecar_config` must correspond to the sidecar docker image used.'
)
self.sidecar_config = sidecar_config
self.log_level = log_level
self.declarations = declarations
self.experiment_labels = self.get_experiment_labels()
self.cluster_def = None
def read(config_values):
"""Reads an ordered list of configuration values and deep merge the values in reverse order."""
if not config_values:
return None
if not isinstance(config_values, (np.ndarray, list, tuple)):
config_values = [config_values]
config = {}
for config_value in config_values:
if not isinstance(config_value, (Mapping, six.string_types)):
raise PolyaxonConfigurationError(
"Expects Mapping, string, or list of Mapping/string instances, "
"received {} instead".format(type(config_value)))
if isinstance(config_value, Mapping):
config = _deep_update(config, config_value)
elif os.path.isfile(config_value):
config = _deep_update(config, _read_from_file(config_value))
else:
PolyaxonConfigurationError('Cannot read config_value: `{}`'.format(config_value))
return config
def _setup_polyflow(self, polyflow_operations):
if not polyflow_operations:
return {}
ops = {}
for schema_name, schema_path in polyflow_operations:
schema = import_string(schema_path)
if not issubclass(schema, BaseOpSchema):
raise PolyaxonConfigurationError(
'Schema {} at path: {} is not a valid operation.'.format(
schema_name, schema_path))
ops[schema_name] = schema
return ops
def __init__(self, values):
self._values = to_list(values)
self._data = reader.read(self._values)
Parser.check_data(spec=self, data=self._data)
headers = Parser.get_headers(spec=self, data=self._data)
try:
self._headers = validator.validate_headers(spec=self, data=headers)
except ValidationError as e:
raise PolyaxonConfigurationError(e)
parsed_data = Parser.parse(self, self._data, None)
self._validated_data = validator.validate(spec=self, data=parsed_data)
self._parsed_data = parsed_data
def matrix_declarations(self):
if not self.matrix:
return []
declarations = []
keys = list(six.iterkeys(self.matrix))
values = [v.to_numpy() for v in six.itervalues(self.matrix)]
for v in itertools.product(*values):
declarations.append(dict(zip(keys, v)))
if len(declarations) != self.matrix_space:
raise PolyaxonConfigurationError(
'The matrix declaration is not valid.')
return declarations
def __init__(self,
namespace,
name,
project_name,
project_uuid,
job_name,
job_uuid,
job_docker_image,
job_container_name=None,
sidecar_container_name=None,
sidecar_docker_image=None,
init_container_name=None,
init_docker_image=None,
role_label=None,
type_label=None,
ports=None,
use_sidecar=False,
sidecar_config=None,
log_level=None):
self.namespace = namespace
self.name = name
self.project_name = project_name
self.project_uuid = project_uuid
self.job_name = job_name
self.job_uuid = job_uuid
self.job_container_name = job_container_name or settings.CONTAINER_NAME_JOB
self.job_docker_image = job_docker_image
self.sidecar_container_name = sidecar_container_name or settings.CONTAINER_NAME_SIDECAR
self.sidecar_docker_image = sidecar_docker_image or settings.JOB_SIDECAR_DOCKER_IMAGE
self.init_container_name = init_container_name or settings.CONTAINER_NAME_INIT
self.init_docker_image = init_docker_image or settings.JOB_INIT_DOCKER_IMAGE
self.role_label = role_label or settings.ROLE_LABELS_WORKER
self.type_label = type_label or settings.TYPE_LABELS_EXPERIMENT
self.app_label = settings.APP_LABELS_JOB
self.labels = self.get_labels()
self.k8s_job_name = self.get_k8s_job_name()
self.ports = to_list(ports) if ports else []
self.use_sidecar = use_sidecar
if use_sidecar and not sidecar_config:
raise PolyaxonConfigurationError(
'In order to use a `sidecar_config` is required. '
'The `sidecar_config` must correspond to the sidecar docker image used.'
)
self.sidecar_config = sidecar_config
self.log_level = log_level
def __init__(self,
namespace,
project_name,
experiment_group_name,
experiment_name,
project_uuid,
experiment_group_uuid,
experiment_uuid,
job_container_name=None,
job_docker_image=None,
sidecar_container_name=None,
sidecar_docker_image=None,
role_label=None,
type_label=None,
ports=None,
use_sidecar=False,
sidecar_config=None):
self.namespace = namespace
self.project_name = project_name
self.experiment_group_name = experiment_group_name
self.experiment_name = experiment_name
self.project_uuid = project_uuid
self.experiment_group_uuid = experiment_group_uuid
self.experiment_uuid = experiment_uuid
self.job_container_name = job_container_name or settings.JOB_CONTAINER_NAME
self.job_docker_image = job_docker_image or settings.JOB_DOCKER_NAME
self.sidecar_container_name = sidecar_container_name or settings.JOB_SIDECAR_CONTAINER_NAME
self.sidecar_docker_image = sidecar_docker_image or settings.JOB_SIDECAR_DOCKER_IMAGE
self.role_label = role_label or settings.ROLE_LABELS_WORKER
self.type_label = type_label or settings.TYPE_LABELS_EXPERIMENT
self.ports = ports or [constants.DEFAULT_PORT]
self.use_sidecar = use_sidecar
if use_sidecar and not sidecar_config:
raise PolyaxonConfigurationError(
'In order to use a `sidecar_config` is required. '
'The `sidecar_config` must correspond to the sidecar docker image used.'
)
self.sidecar_config = sidecar_config
def _extra_validation(self):
if not self.matrix:
raise PolyaxonConfigurationError(
'A matrix definition is required for group specification.')
def _read_from_json(f_path):
try:
return json.loads(open(f_path).read())
except ValueError as e:
raise PolyaxonConfigurationError(e)
def _extra_validation(self):
if self.settings and self.settings.matrix:
raise PolyaxonConfigurationError(
'ExperimentSpecification cannot contain a `matrix` section, you should '
'use a GroupSpecification instead.')
def _extra_validation(self):
if (self.RUN_EXEC not in self.validated_data or
not self.validated_data[self.RUN_EXEC] or
self.validated_data[self.RUN_EXEC].cmd is not None):
raise PolyaxonConfigurationError(
'Plugin specification must contain a valid `run` section.')