def create_specification( cls, # pylint:disable=arguments-differ build_config, run_config, to_dict=True): try: specification = BaseRunSpecification.create_specification( build_config=build_config, to_dict=True) except PolyaxonConfigurationError: raise PolyaxonConfigurationError( 'Create specification expects a dict or an instance of BuildConfig.' ) if isinstance(run_config, RunConfig): r_config = run_config.to_light_dict() elif isinstance(run_config, Mapping): r_config = RunConfig.from_dict(run_config, unknown=EXCLUDE) r_config = r_config.to_light_dict() else: raise PolyaxonConfigurationError( 'Create specification expects a dict or an instance of RunConfig.' ) specification[cls.KIND] = cls._SPEC_KIND specification[cls.RUN] = r_config if to_dict: return specification return cls.read(specification)
def __init__(self, values): self._values = to_list(values) self._data = reader.read(self._values) Parser.check_data(spec=self, data=self._data) headers = Parser.get_headers(spec=self, data=self._data) matrix = Parser.get_matrix(spec=self, data=self._data) try: self._matrix = validator.validate_matrix(matrix) except ValidationError as e: raise PolyaxonConfigurationError(e) try: self._headers = validator.validate_headers(spec=self, data=headers) except ValidationError as e: raise PolyaxonConfigurationError(e) self._parsed_data = [] self._validated_data = [] self._experiment_specs = [] matrix_declarations = self.matrix_declarations if self.matrix_declarations else [ {} ] for i, matrix_declaration in enumerate(matrix_declarations): parsed_data = Parser.parse(self, self._data, matrix_declaration) self._validated_data.append( validator.validate(spec=self, data=parsed_data)) self._parsed_data.append(parsed_data) self._experiment_specs.append( Specification(experiment=i, values=parsed_data))
def read(config_values): """Reads an ordered list of configuration values and deep merge the values in reverse order.""" if not config_values: raise PolyaxonConfigurationError( 'Cannot read config_value: `{}`'.format(config_values)) config_values = to_list(config_values) config = {} for config_value in config_values: if not isinstance(config_value, (Mapping, six.string_types)): raise PolyaxonConfigurationError( "Expects Mapping, string, or list of Mapping/string instances, " "received {} instead".format(type(config_value))) if isinstance(config_value, Mapping): config_results = config_value elif os.path.isfile(config_value): config_results = _read_from_file(config_value) else: # try reading a stream of yaml or json try: config_results = _read_from_stream(config_value) except ScannerError: raise PolyaxonConfigurationError( 'Received non valid yaml stream: `{}`'.format( config_value)) if config_results and isinstance(config_results, Mapping): config = deep_update(config, config_results) else: raise PolyaxonConfigurationError( 'Cannot read config_value: `{}`'.format(config_value)) return config
def _read_from_json(f_path, is_stream=False): if is_stream: try: return json.loads(f_path) except ValueError as e: raise PolyaxonConfigurationError(e) try: return json.loads(open(f_path).read()) except ValueError as e: raise PolyaxonConfigurationError(e)
def create_specification( cls, # pylint:disable=arguments-differ build_config, to_dict=True): from polyaxon_schemas.ops.build import BuildConfig if isinstance(build_config, BuildConfig): b_config = build_config.to_light_dict() elif isinstance(build_config, Mapping): # Since the objective is to create the build spec from other specs # we drop any extra attrs b_config = BuildConfig.from_dict(build_config, unknown=EXCLUDE) b_config = b_config.to_light_dict() else: raise PolyaxonConfigurationError( 'Create specification expects a dict or an instance of BuildConfig.' ) specification = { cls.VERSION: 1, cls.KIND: cls._SPEC_KIND, cls.BUILD: b_config, } if to_dict: return specification return cls.read(specification)
def get_persistent_volume_spec(namespace, volume, run_type, access_modes='ReadWriteOnce', persistent_volume_reclaim_policy='Recycle'): capacity = {'storage': STORAGE_BY_VOLUME[volume]} access_modes = to_list(access_modes) vol_path = get_vol_path(volume, run_type) if run_type == RunTypes.MINIKUBE: params = get_host_path_pvol(vol_path) elif run_type == RunTypes.KUBERNETES: params = get_nfs_pvol(vol_path) else: raise PolyaxonConfigurationError( 'Run type `{}` is not allowed.'.format(run_type)) volc_name = constants.VOLUME_CLAIM_NAME.format(vol_name=volume) claim_ref = client.V1ObjectReference( api_version=k8s_constants.K8S_API_VERSION_V1, kind=k8s_constants.K8S_PERSISTENT_VOLUME_CLAIM_KIND, name=volc_name, namespace=namespace) return client.V1PersistentVolumeSpec( capacity=capacity, access_modes=access_modes, persistent_volume_reclaim_policy=persistent_volume_reclaim_policy, claim_ref=claim_ref, **params)
def create_specification(cls, build_config, secret_refs=None, configmap_refs=None, to_dict=True): if isinstance(build_config, BuildConfig): config = build_config.to_light_dict() elif isinstance(build_config, Mapping): # Since the objective is to create the build spec from other specs # we drop any extra attrs config = BuildConfig.from_dict(build_config, unknown=EXCLUDE) config = config.to_light_dict() else: raise PolyaxonConfigurationError( 'Create specification expects a dict or an instance of BuildConfig.' ) specification = { cls.VERSION: 1, cls.KIND: cls._SPEC_KIND, } specification.update(config) env = {} if secret_refs: env['secret_refs'] = secret_refs if configmap_refs: env['configmap_refs'] = configmap_refs if env: specification[cls.ENVIRONMENT] = env if to_dict: return specification return cls.read(specification)
def _read_from_file(f_path): _, ext = os.path.splitext(f_path) if ext in ('.yml', '.yaml'): return _read_from_yml(f_path) elif ext == '.json': return _read_from_json(f_path) raise PolyaxonConfigurationError( "Expects a file with extension: `.yml`, `.yaml`, or `json`, " "received instead `{}`".format(ext))
def __init__(self, values): self._values = to_list(values) try: self._data = rhea.read(self._values) except rhea.RheaError as e: raise PolyaxonConfigurationError(e) self.check_data() headers = Parser.get_headers(spec=self, data=self._data) try: self._headers = validator.validate_headers(spec=self, data=headers) except ValidationError as e: raise PolyaxonConfigurationError(e) self._parsed_data = None self._validated_data = None self._config = None self._set_parsed_data() self._extra_validation()
def search_algorithm(self): if not self.matrix: raise PolyaxonConfigurationError( 'a search algorithm requires a matrix definition.') if self.settings.random_search: return SearchAlgorithms.RANDOM if self.settings.hyperband: return SearchAlgorithms.HYPERBAND # Default value return SearchAlgorithms.GRID
def __init__(self, namespace, project_name, experiment_group_name, experiment_name, project_uuid, experiment_group_uuid, experiment_uuid, original_name=None, cloning_strategy=None, job_container_name=None, job_docker_image=None, sidecar_container_name=None, sidecar_docker_image=None, init_container_name=None, init_docker_image=None, role_label=None, type_label=None, ports=None, use_sidecar=False, sidecar_config=None, log_level=None, declarations=None): self.namespace = namespace self.project_name = project_name self.experiment_group_name = experiment_group_name self.experiment_name = experiment_name self.project_uuid = project_uuid self.experiment_group_uuid = experiment_group_uuid self.experiment_uuid = experiment_uuid self.original_name = original_name self.cloning_strategy = cloning_strategy self.job_container_name = job_container_name or settings.CONTAINER_NAME_EXPERIMENT_JOB self.job_docker_image = job_docker_image or settings.JOB_DOCKER_NAME self.sidecar_container_name = sidecar_container_name or settings.CONTAINER_NAME_SIDECAR self.sidecar_docker_image = sidecar_docker_image or settings.JOB_SIDECAR_DOCKER_IMAGE self.init_container_name = init_container_name or settings.CONTAINER_NAME_INIT self.init_docker_image = init_docker_image or settings.JOB_INIT_DOCKER_IMAGE self.role_label = role_label or settings.ROLE_LABELS_WORKER self.type_label = type_label or settings.TYPE_LABELS_EXPERIMENT self.app_label = settings.APP_LABELS_EXPERIMENT self.ports = ports or [constants.DEFAULT_PORT] self.use_sidecar = use_sidecar if use_sidecar and not sidecar_config: raise PolyaxonConfigurationError( 'In order to use a `sidecar_config` is required. ' 'The `sidecar_config` must correspond to the sidecar docker image used.' ) self.sidecar_config = sidecar_config self.log_level = log_level self.declarations = declarations self.experiment_labels = self.get_experiment_labels() self.cluster_def = None
def read(config_values): """Reads an ordered list of configuration values and deep merge the values in reverse order.""" if not config_values: return None if not isinstance(config_values, (np.ndarray, list, tuple)): config_values = [config_values] config = {} for config_value in config_values: if not isinstance(config_value, (Mapping, six.string_types)): raise PolyaxonConfigurationError( "Expects Mapping, string, or list of Mapping/string instances, " "received {} instead".format(type(config_value))) if isinstance(config_value, Mapping): config = _deep_update(config, config_value) elif os.path.isfile(config_value): config = _deep_update(config, _read_from_file(config_value)) else: PolyaxonConfigurationError('Cannot read config_value: `{}`'.format(config_value)) return config
def _setup_polyflow(self, polyflow_operations): if not polyflow_operations: return {} ops = {} for schema_name, schema_path in polyflow_operations: schema = import_string(schema_path) if not issubclass(schema, BaseOpSchema): raise PolyaxonConfigurationError( 'Schema {} at path: {} is not a valid operation.'.format( schema_name, schema_path)) ops[schema_name] = schema return ops
def __init__(self, values): self._values = to_list(values) self._data = reader.read(self._values) Parser.check_data(spec=self, data=self._data) headers = Parser.get_headers(spec=self, data=self._data) try: self._headers = validator.validate_headers(spec=self, data=headers) except ValidationError as e: raise PolyaxonConfigurationError(e) parsed_data = Parser.parse(self, self._data, None) self._validated_data = validator.validate(spec=self, data=parsed_data) self._parsed_data = parsed_data
def matrix_declarations(self): if not self.matrix: return [] declarations = [] keys = list(six.iterkeys(self.matrix)) values = [v.to_numpy() for v in six.itervalues(self.matrix)] for v in itertools.product(*values): declarations.append(dict(zip(keys, v))) if len(declarations) != self.matrix_space: raise PolyaxonConfigurationError( 'The matrix declaration is not valid.') return declarations
def __init__(self, namespace, name, project_name, project_uuid, job_name, job_uuid, job_docker_image, job_container_name=None, sidecar_container_name=None, sidecar_docker_image=None, init_container_name=None, init_docker_image=None, role_label=None, type_label=None, ports=None, use_sidecar=False, sidecar_config=None, log_level=None): self.namespace = namespace self.name = name self.project_name = project_name self.project_uuid = project_uuid self.job_name = job_name self.job_uuid = job_uuid self.job_container_name = job_container_name or settings.CONTAINER_NAME_JOB self.job_docker_image = job_docker_image self.sidecar_container_name = sidecar_container_name or settings.CONTAINER_NAME_SIDECAR self.sidecar_docker_image = sidecar_docker_image or settings.JOB_SIDECAR_DOCKER_IMAGE self.init_container_name = init_container_name or settings.CONTAINER_NAME_INIT self.init_docker_image = init_docker_image or settings.JOB_INIT_DOCKER_IMAGE self.role_label = role_label or settings.ROLE_LABELS_WORKER self.type_label = type_label or settings.TYPE_LABELS_EXPERIMENT self.app_label = settings.APP_LABELS_JOB self.labels = self.get_labels() self.k8s_job_name = self.get_k8s_job_name() self.ports = to_list(ports) if ports else [] self.use_sidecar = use_sidecar if use_sidecar and not sidecar_config: raise PolyaxonConfigurationError( 'In order to use a `sidecar_config` is required. ' 'The `sidecar_config` must correspond to the sidecar docker image used.' ) self.sidecar_config = sidecar_config self.log_level = log_level
def __init__(self, namespace, project_name, experiment_group_name, experiment_name, project_uuid, experiment_group_uuid, experiment_uuid, job_container_name=None, job_docker_image=None, sidecar_container_name=None, sidecar_docker_image=None, role_label=None, type_label=None, ports=None, use_sidecar=False, sidecar_config=None): self.namespace = namespace self.project_name = project_name self.experiment_group_name = experiment_group_name self.experiment_name = experiment_name self.project_uuid = project_uuid self.experiment_group_uuid = experiment_group_uuid self.experiment_uuid = experiment_uuid self.job_container_name = job_container_name or settings.JOB_CONTAINER_NAME self.job_docker_image = job_docker_image or settings.JOB_DOCKER_NAME self.sidecar_container_name = sidecar_container_name or settings.JOB_SIDECAR_CONTAINER_NAME self.sidecar_docker_image = sidecar_docker_image or settings.JOB_SIDECAR_DOCKER_IMAGE self.role_label = role_label or settings.ROLE_LABELS_WORKER self.type_label = type_label or settings.TYPE_LABELS_EXPERIMENT self.ports = ports or [constants.DEFAULT_PORT] self.use_sidecar = use_sidecar if use_sidecar and not sidecar_config: raise PolyaxonConfigurationError( 'In order to use a `sidecar_config` is required. ' 'The `sidecar_config` must correspond to the sidecar docker image used.' ) self.sidecar_config = sidecar_config
def _extra_validation(self): if not self.matrix: raise PolyaxonConfigurationError( 'A matrix definition is required for group specification.')
def _read_from_json(f_path): try: return json.loads(open(f_path).read()) except ValueError as e: raise PolyaxonConfigurationError(e)
def _extra_validation(self): if self.settings and self.settings.matrix: raise PolyaxonConfigurationError( 'ExperimentSpecification cannot contain a `matrix` section, you should ' 'use a GroupSpecification instead.')
def _extra_validation(self): if (self.RUN_EXEC not in self.validated_data or not self.validated_data[self.RUN_EXEC] or self.validated_data[self.RUN_EXEC].cmd is not None): raise PolyaxonConfigurationError( 'Plugin specification must contain a valid `run` section.')