def cluster_def(self): cluster = { TaskType.MASTER: 1, } is_distributed = False environment = self.environment if not environment: return cluster, is_distributed if environment.tensorflow: return TensorflowSpecification.get_cluster_def( cluster=cluster, tensorflow_config=environment.tensorflow) if environment.horovod: return HorovodSpecification.get_cluster_def( cluster=cluster, horovod_config=environment.horovod) if environment.mxnet: return MXNetSpecification.get_cluster_def( cluster=cluster, mxnet_config=environment.mxnet) if environment.pytorch: return PytorchSpecification.get_cluster_def( cluster=cluster, pytorch_config=environment.pytorch) # No specified framework, It should return default standalone mode cluster definition return cluster, is_distributed
def node_selectors(self): cluster, is_distributed, = self.spec.cluster_def worker_node_selectors = TensorflowSpecification.get_worker_node_selectors( environment=self.spec.environment, cluster=cluster, is_distributed=is_distributed) ps_node_selectors = TensorflowSpecification.get_ps_node_selectors( environment=self.spec.environment, cluster=cluster, is_distributed=is_distributed) return { TaskType.MASTER: { 0: self.spec.master_node_selectors }, TaskType.WORKER: worker_node_selectors, TaskType.PS: ps_node_selectors, }
def affinities(self): cluster, is_distributed, = self.spec.cluster_def worker_affinities = TensorflowSpecification.get_worker_affinities( environment=self.spec.environment, cluster=cluster, is_distributed=is_distributed) ps_affinities = TensorflowSpecification.get_ps_affinities( environment=self.spec.environment, cluster=cluster, is_distributed=is_distributed) return { TaskType.MASTER: { 0: self.spec.master_affinity }, TaskType.WORKER: worker_affinities, TaskType.PS: ps_affinities, }
def handle_tensorflow_experiment(experiment, spawner, response): # Get the number of jobs this experiment started master = response[TaskType.MASTER] job_uuid = master['pod']['metadata']['labels']['job_uuid'] job_uuid = uuid.UUID(job_uuid) create_job(job_uuid=job_uuid, experiment=experiment, definition=get_job_definition(master), resources=spawner.spec.master_resources) cluster, is_distributed, = spawner.spec.cluster_def worker_resources = TensorflowSpecification.get_worker_resources( environment=spawner.spec.environment, cluster=cluster, is_distributed=is_distributed ) ps_resources = TensorflowSpecification.get_ps_resources( environment=spawner.spec.environment, cluster=cluster, is_distributed=is_distributed ) for i, worker in enumerate(response[TaskType.WORKER]): job_uuid = worker['pod']['metadata']['labels']['job_uuid'] job_uuid = uuid.UUID(job_uuid) create_job(job_uuid=job_uuid, experiment=experiment, definition=get_job_definition(worker), role=TaskType.WORKER, resources=worker_resources.get(i)) for i, ps in enumerate(response[TaskType.PS]): job_uuid = ps['pod']['metadata']['labels']['job_uuid'] job_uuid = uuid.UUID(job_uuid) create_job(job_uuid=job_uuid, experiment=experiment, definition=get_job_definition(ps), role=TaskType.PS, resources=ps_resources.get(i))
def handle_tensorflow_experiment(experiment, spawner, response): # Get the number of jobs this experiment started master = response[TaskType.MASTER] job_uuid = master['pod']['metadata']['labels']['job_uuid'] job_uuid = uuid.UUID(job_uuid) create_job(job_uuid=job_uuid, experiment=experiment, definition=get_job_definition(master), resources=spawner.spec.master_resources) cluster, is_distributed, = spawner.spec.cluster_def worker_resources = TensorflowSpecification.get_worker_resources( environment=spawner.spec.environment, cluster=cluster, is_distributed=is_distributed ) ps_resources = TensorflowSpecification.get_ps_resources( environment=spawner.spec.environment, cluster=cluster, is_distributed=is_distributed ) for i, worker in enumerate(response[TaskType.WORKER]): job_uuid = worker['pod']['metadata']['labels']['job_uuid'] job_uuid = uuid.UUID(job_uuid) create_job(job_uuid=job_uuid, experiment=experiment, definition=get_job_definition(worker), role=TaskType.WORKER, resources=worker_resources.get(i)) for i, ps in enumerate(response[TaskType.PS]): job_uuid = ps['pod']['metadata']['labels']['job_uuid'] job_uuid = uuid.UUID(job_uuid) create_job(job_uuid=job_uuid, experiment=experiment, definition=get_job_definition(ps), role=TaskType.PS, resources=ps_resources.get(i))
def total_resources(self): environment = self.environment if not environment: return None cluster, is_distributed = self.cluster_def # Check if any framework is defined if environment.tensorflow: return TensorflowSpecification.get_total_resources( master_resources=self.master_resources, environment=environment, cluster=cluster, is_distributed=is_distributed) if environment.horovod: return HorovodSpecification.get_total_resources( master_resources=self.master_resources, environment=environment, cluster=cluster, is_distributed=is_distributed) if environment.mxnet: return MXNetSpecification.get_total_resources( master_resources=self.master_resources, environment=environment, cluster=cluster, is_distributed=is_distributed) if environment.pytorch: return PytorchSpecification.get_total_resources( master_resources=self.master_resources, environment=environment, cluster=cluster, is_distributed=is_distributed) # default value is the master resources return self.master_resources
def test_advanced_file_passes(self): plxfile = PolyaxonFile( os.path.abspath('tests/fixtures/advanced_file.yml')) spec = plxfile.specification assert spec.version == 1 assert spec.project.name == 'project1' assert isinstance(spec.settings, SettingsConfig) assert isinstance(spec.settings.logging, LoggingConfig) assert spec.settings.matrix is None assert spec.is_runnable assert isinstance(spec.environment, EnvironmentConfig) assert spec.framework == Frameworks.TENSORFLOW assert spec.environment.tensorflow.n_workers == 5 assert spec.environment.tensorflow.n_ps == 10 assert spec.environment.tensorflow.delay_workers_by_global_step is True assert isinstance(spec.environment.tensorflow.run_config, RunConfig) assert spec.environment.tensorflow.run_config.tf_random_seed == 100 assert spec.environment.tensorflow.run_config.save_summary_steps == 100 assert spec.environment.tensorflow.run_config.save_checkpoints_secs == 60 assert isinstance(spec.environment.tensorflow.run_config.session, SessionConfig) assert spec.environment.tensorflow.run_config.session.allow_soft_placement is True assert spec.environment.tensorflow.run_config.session.intra_op_parallelism_threads == 2 assert spec.environment.tensorflow.run_config.session.inter_op_parallelism_threads == 2 # check properties for returning worker configs and resources assert spec.environment.tensorflow.worker_configs is None assert spec.environment.tensorflow.ps_configs is None assert spec.environment.tensorflow.worker_resources is None assert spec.environment.tensorflow.ps_resources is None cluster, is_distributed = spec.cluster_def assert TensorflowSpecification.get_worker_configs( environment=spec.environment, cluster=cluster, is_distributed=is_distributed) == {} assert TensorflowSpecification.get_ps_configs( environment=spec.environment, cluster=cluster, is_distributed=is_distributed) == {} assert TensorflowSpecification.get_worker_resources( environment=spec.environment, cluster=cluster, is_distributed=is_distributed) == {} assert TensorflowSpecification.get_ps_resources( environment=spec.environment, cluster=cluster, is_distributed=is_distributed) == {} assert spec.cluster_def == ({ TaskType.MASTER: 1, TaskType.WORKER: 5, TaskType.PS: 10 }, True) assert isinstance(spec.model, ClassifierConfig) assert isinstance(spec.model.loss, MeanSquaredErrorConfig) assert isinstance(spec.model.optimizer, AdamConfig) assert spec.model.optimizer.learning_rate == 0.21 assert isinstance(spec.model.graph, GraphConfig) assert len(spec.model.graph.layers) == 7 assert spec.model.graph.input_layers == [['images', 0, 0]] assert len(spec.model.graph.output_layers) == 3 assert ['super_dense', 0, 0] in spec.model.graph.output_layers assert isinstance(spec.train.data_pipeline, TFRecordImagePipelineConfig) assert len(spec.train.data_pipeline.feature_processors. feature_processors) == 1 assert isinstance(spec.eval.data_pipeline, TFRecordImagePipelineConfig) assert spec.eval.data_pipeline.feature_processors is None
def test_distributed_tensorflow_passes(self): plxfile = PolyaxonFile( os.path.abspath('tests/fixtures/distributed_tensorflow_file.yml')) spec = plxfile.specification assert spec.version == 1 assert spec.project.name == 'project1' assert isinstance(spec.settings, SettingsConfig) assert isinstance(spec.settings.logging, LoggingConfig) assert spec.settings.matrix is None assert isinstance(spec.environment, EnvironmentConfig) assert spec.is_runnable assert spec.framework == Frameworks.TENSORFLOW assert spec.environment.tensorflow.n_workers == 5 assert spec.environment.tensorflow.n_ps == 10 assert isinstance(spec.environment.resources, PodResourcesConfig) assert isinstance(spec.environment.resources.cpu, K8SResourcesConfig) assert spec.environment.resources.cpu.requests == 1 assert spec.environment.resources.cpu.limits == 2 assert isinstance(spec.environment.tensorflow.default_worker_resources, PodResourcesConfig) assert isinstance( spec.environment.tensorflow.default_worker_resources.cpu, K8SResourcesConfig) assert spec.environment.tensorflow.default_worker_resources.cpu.requests == 3 assert spec.environment.tensorflow.default_worker_resources.cpu.limits == 3 assert isinstance( spec.environment.tensorflow.default_worker_resources.memory, K8SResourcesConfig) assert spec.environment.tensorflow.default_worker_resources.memory.requests == 256 assert spec.environment.tensorflow.default_worker_resources.memory.limits == 256 assert isinstance(spec.environment.tensorflow.worker_resources[0], PodResourcesConfig) assert isinstance( spec.environment.tensorflow.worker_resources[0].memory, K8SResourcesConfig) assert spec.environment.tensorflow.worker_resources[0].index == 3 assert spec.environment.tensorflow.worker_resources[ 0].memory.requests == 300 assert spec.environment.tensorflow.worker_resources[ 0].memory.limits == 300 assert isinstance(spec.environment.tensorflow.default_ps_resources, PodResourcesConfig) assert isinstance(spec.environment.tensorflow.default_ps_resources.cpu, K8SResourcesConfig) assert spec.environment.tensorflow.default_ps_resources.cpu.requests == 2 assert spec.environment.tensorflow.default_ps_resources.cpu.limits == 4 assert isinstance(spec.environment.tensorflow.ps_resources[0], PodResourcesConfig) assert isinstance(spec.environment.tensorflow.ps_resources[0].memory, K8SResourcesConfig) assert spec.environment.tensorflow.ps_resources[0].index == 9 assert spec.environment.tensorflow.ps_resources[ 0].memory.requests == 512 assert spec.environment.tensorflow.ps_resources[ 0].memory.limits == 1024 # check that properties for return list of configs and resources is working cluster, is_distributed = spec.cluster_def worker_resources = TensorflowSpecification.get_worker_resources( environment=spec.environment, cluster=cluster, is_distributed=is_distributed) assert len(worker_resources) == spec.environment.tensorflow.n_workers assert set(worker_resources.values()) == { spec.environment.tensorflow.default_worker_resources, spec.environment.tensorflow.worker_resources[0] } ps_resources = TensorflowSpecification.get_ps_resources( environment=spec.environment, cluster=cluster, is_distributed=is_distributed) assert len(ps_resources) == spec.environment.tensorflow.n_ps assert set(ps_resources.values()) == { spec.environment.tensorflow.default_ps_resources, spec.environment.tensorflow.ps_resources[0] } # Check total resources assert spec.total_resources == { 'cpu': { 'requests': 1 + 3 * 4 + 2 * 9, 'limits': 2 + 3 * 4 + 4 * 9 }, 'memory': { 'requests': 300 + 256 * 4 + 512, 'limits': 300 + 256 * 4 + 1024 }, 'gpu': None } assert spec.cluster_def == ({ TaskType.MASTER: 1, TaskType.WORKER: 5, TaskType.PS: 10 }, True)
def test_advanced_file_with_custom_configs_and_resources_passes(self): plxfile = PolyaxonFile( os.path.abspath( 'tests/fixtures/advanced_file_with_custom_configs_and_resources.yml' )) spec = plxfile.specification assert spec.version == 1 assert spec.project.name == 'project1' assert isinstance(spec.settings, SettingsConfig) assert isinstance(spec.settings.logging, LoggingConfig) assert spec.settings.matrix is None assert isinstance(spec.environment, EnvironmentConfig) assert spec.is_runnable assert spec.framework == Frameworks.TENSORFLOW assert spec.environment.tensorflow.n_workers == 5 assert spec.environment.tensorflow.n_ps == 10 assert spec.environment.tensorflow.delay_workers_by_global_step is True assert isinstance(spec.environment.tensorflow.run_config, RunConfig) assert spec.environment.tensorflow.run_config.tf_random_seed == 100 assert spec.environment.tensorflow.run_config.save_summary_steps == 100 assert spec.environment.tensorflow.run_config.save_checkpoints_secs == 60 assert isinstance(spec.environment.resources, PodResourcesConfig) assert isinstance(spec.environment.resources.cpu, K8SResourcesConfig) assert spec.environment.resources.cpu.requests == 1 assert spec.environment.resources.cpu.limits == 2 assert isinstance(spec.environment.tensorflow.run_config.session, SessionConfig) assert spec.environment.tensorflow.run_config.session.allow_soft_placement is True assert spec.environment.tensorflow.run_config.session.intra_op_parallelism_threads == 2 assert spec.environment.tensorflow.run_config.session.inter_op_parallelism_threads == 2 assert isinstance(spec.environment.tensorflow.default_worker_config, SessionConfig) assert spec.environment.tensorflow.default_worker_config.allow_soft_placement is True assert spec.environment.tensorflow.default_worker_config.intra_op_parallelism_threads == 1 assert spec.environment.tensorflow.default_worker_config.inter_op_parallelism_threads == 1 assert isinstance(spec.environment.tensorflow.worker_configs[0], SessionConfig) assert spec.environment.tensorflow.worker_configs[0].index == 3 assert spec.environment.tensorflow.worker_configs[ 0].allow_soft_placement is False assert spec.environment.tensorflow.worker_configs[ 0].intra_op_parallelism_threads == 5 assert spec.environment.tensorflow.worker_configs[ 0].inter_op_parallelism_threads == 5 assert spec.environment.tensorflow.ps_configs is None assert spec.environment.tensorflow.worker_resources is None assert isinstance(spec.environment.tensorflow.default_ps_resources, PodResourcesConfig) assert isinstance(spec.environment.tensorflow.default_ps_resources.cpu, K8SResourcesConfig) assert spec.environment.tensorflow.default_ps_resources.cpu.requests == 2 assert spec.environment.tensorflow.default_ps_resources.cpu.limits == 4 assert isinstance(spec.environment.tensorflow.ps_resources[0], PodResourcesConfig) assert isinstance(spec.environment.tensorflow.ps_resources[0].memory, K8SResourcesConfig) assert spec.environment.tensorflow.ps_resources[0].index == 9 assert spec.environment.tensorflow.ps_resources[ 0].memory.requests == 512 assert spec.environment.tensorflow.ps_resources[ 0].memory.limits == 1024 # check that properties for return list of configs and resources is working cluster, is_distributed = spec.cluster_def worker_configs = TensorflowSpecification.get_worker_configs( environment=spec.environment, cluster=cluster, is_distributed=is_distributed) assert len(worker_configs) == spec.environment.tensorflow.n_workers assert set(worker_configs.values()) == { spec.environment.tensorflow.default_worker_config, spec.environment.tensorflow.worker_configs[0] } assert TensorflowSpecification.get_ps_configs( environment=spec.environment, cluster=cluster, is_distributed=is_distributed) == {} assert TensorflowSpecification.get_worker_resources( environment=spec.environment, cluster=cluster, is_distributed=is_distributed) == {} ps_resources = TensorflowSpecification.get_ps_resources( environment=spec.environment, cluster=cluster, is_distributed=is_distributed) assert len(ps_resources) == spec.environment.tensorflow.n_ps assert set(ps_resources.values()) == { spec.environment.tensorflow.default_ps_resources, spec.environment.tensorflow.ps_resources[0] } # Check total resources assert spec.total_resources == { 'cpu': { 'requests': 1 + 2 * 9, 'limits': 2 + 4 * 9 }, 'memory': { 'requests': 512, 'limits': 1024 }, 'gpu': None } assert spec.cluster_def == ({ TaskType.MASTER: 1, TaskType.WORKER: 5, TaskType.PS: 10 }, True) assert isinstance(spec.model, ClassifierConfig) assert isinstance(spec.model.loss, MeanSquaredErrorConfig) assert isinstance(spec.model.optimizer, AdamConfig) assert spec.model.optimizer.learning_rate == 0.21 assert isinstance(spec.model.graph, GraphConfig) assert len(spec.model.graph.layers) == 7 assert spec.model.graph.input_layers == [['images', 0, 0]] assert len(spec.model.graph.output_layers) == 3 assert ['super_dense', 0, 0] in spec.model.graph.output_layers assert isinstance(spec.train.data_pipeline, TFRecordImagePipelineConfig) assert len(spec.train.data_pipeline.feature_processors. feature_processors) == 1 assert isinstance(spec.eval.data_pipeline, TFRecordImagePipelineConfig) assert spec.eval.data_pipeline.feature_processors is None