def handle_horovod_experiment(experiment, spawner, response):
# Get the number of jobs this experiment started
master = response[TaskType.MASTER]
job_uuid = master['pod']['metadata']['labels']['job_uuid']
job_uuid = uuid.UUID(job_uuid)
create_job(job_uuid=job_uuid,
experiment=experiment,
definition=get_job_definition(master),
resources=spawner.spec.master_resources)
cluster, is_distributed, = spawner.spec.cluster_def
worker_resources = HorovodSpecification.get_worker_resources(
environment=spawner.spec.environment,
cluster=cluster,
is_distributed=is_distributed)
for i, worker in enumerate(response[TaskType.WORKER]):
job_uuid = worker['pod']['metadata']['labels']['job_uuid']
job_uuid = uuid.UUID(job_uuid)
create_job(job_uuid=job_uuid,
experiment=experiment,
definition=get_job_definition(worker),
role=TaskType.WORKER,
resources=worker_resources.get(i))
def handle_horovod_experiment(experiment, spawner, response):
# Get the number of jobs this experiment started
master = response[TaskType.MASTER]
job_uuid = master['pod']['metadata']['labels']['job_uuid']
job_uuid = uuid.UUID(job_uuid)
create_job(job_uuid=job_uuid,
experiment=experiment,
definition=get_job_definition(master),
resources=spawner.spec.master_resources)
cluster, is_distributed, = spawner.spec.cluster_def
worker_resources = HorovodSpecification.get_worker_resources(
environment=spawner.spec.environment,
cluster=cluster,
is_distributed=is_distributed
)
for i, worker in enumerate(response[TaskType.WORKER]):
job_uuid = worker['pod']['metadata']['labels']['job_uuid']
job_uuid = uuid.UUID(job_uuid)
create_job(job_uuid=job_uuid,
experiment=experiment,
definition=get_job_definition(worker),
role=TaskType.WORKER,
resources=worker_resources.get(i))
def resources(self):
cluster, is_distributed, = self.spec.cluster_def
worker_resources = HorovodSpecification.get_worker_resources(
environment=self.spec.environment,
cluster=cluster,
is_distributed=is_distributed
)
return {
TaskType.MASTER: {0: self.spec.master_resources},
TaskType.WORKER: worker_resources,
}
def resources(self):
cluster, is_distributed, = self.spec.cluster_def
worker_resources = HorovodSpecification.get_worker_resources(
environment=self.spec.environment,
cluster=cluster,
is_distributed=is_distributed
)
return {
TaskType.MASTER: {0: self.spec.master_resources},
TaskType.WORKER: worker_resources,
}
def test_distributed_horovod_passes(self):
plxfile = PolyaxonFile(
os.path.abspath('tests/fixtures/distributed_horovod_file.yml'))
spec = plxfile.specification
assert spec.version == 1
assert spec.project.name == 'project1'
assert isinstance(spec.settings, SettingsConfig)
assert isinstance(spec.settings.logging, LoggingConfig)
assert spec.settings.matrix is None
assert isinstance(spec.environment, EnvironmentConfig)
assert spec.is_runnable
assert spec.framework == Frameworks.HOROVOD
assert spec.environment.horovod.n_workers == 5
assert isinstance(spec.environment.resources, PodResourcesConfig)
assert isinstance(spec.environment.resources.cpu, K8SResourcesConfig)
assert spec.environment.resources.cpu.requests == 1
assert spec.environment.resources.cpu.limits == 2
assert isinstance(spec.environment.horovod.default_worker_resources,
PodResourcesConfig)
assert isinstance(
spec.environment.horovod.default_worker_resources.cpu,
K8SResourcesConfig)
assert spec.environment.horovod.default_worker_resources.cpu.requests == 3
assert spec.environment.horovod.default_worker_resources.cpu.limits == 3
assert isinstance(
spec.environment.horovod.default_worker_resources.memory,
K8SResourcesConfig)
assert spec.environment.horovod.default_worker_resources.memory.requests == 256
assert spec.environment.horovod.default_worker_resources.memory.limits == 256
assert isinstance(spec.environment.horovod.worker_resources[0],
PodResourcesConfig)
assert isinstance(spec.environment.horovod.worker_resources[0].memory,
K8SResourcesConfig)
assert spec.environment.horovod.worker_resources[0].index == 3
assert spec.environment.horovod.worker_resources[
0].memory.requests == 300
assert spec.environment.horovod.worker_resources[
0].memory.limits == 300
# check that properties for return list of configs and resources is working
cluster, is_distributed = spec.cluster_def
worker_resources = HorovodSpecification.get_worker_resources(
environment=spec.environment,
cluster=cluster,
is_distributed=is_distributed)
assert len(worker_resources) == spec.environment.horovod.n_workers
assert set(worker_resources.values()) == {
spec.environment.horovod.default_worker_resources,
spec.environment.horovod.worker_resources[0]
}
# Check total resources
assert spec.total_resources == {
'cpu': {
'requests': 1 + 3 * 4,
'limits': 2 + 3 * 4
},
'memory': {
'requests': 300 + 256 * 4,
'limits': 300 + 256 * 4
},
'gpu': None
}
assert spec.cluster_def == ({
TaskType.MASTER: 1,
TaskType.WORKER: 5
}, True)