def print_cluster_conf(cluster_conf: ClusterConfiguration, wait: bool):
user_configuration = cluster_conf.user_configuration
log.info("-------------------------------------------")
log.info("cluster id: %s", cluster_conf.cluster_id)
log.info("cluster toolkit: %s %s", cluster_conf.toolkit.software,
cluster_conf.toolkit.version)
log.info("cluster size: %s",
cluster_conf.size + cluster_conf.size_low_priority)
log.info("> dedicated: %s", cluster_conf.size)
log.info("> low priority: %s", cluster_conf.size_low_priority)
log.info("cluster vm size: %s", cluster_conf.vm_size)
log.info("subnet ID: %s", cluster_conf.subnet_id)
log.info(
"file shares: %s",
len(cluster_conf.file_shares)
if cluster_conf.file_shares is not None else 0)
log.info("gpu enabled: %s", str(cluster_conf.gpu_enabled()))
log.info("docker repo name: %s", cluster_conf.get_docker_repo())
if cluster_conf.get_docker_run_options():
log.info("docker run options: %s",
cluster_conf.get_docker_run_options())
log.info("wait for cluster: %s", wait)
if user_configuration:
log.info("username: %s", user_configuration.username)
if user_configuration.password:
log.info("Password: %s", '*' * len(user_configuration.password))
log.info("Plugins:")
if not cluster_conf.plugins:
log.info(" None Configured")
else:
for plugin in cluster_conf.plugins:
log.info(" - %s", plugin.name)
log.info("-------------------------------------------")
def test_scheduling_target_dedicated_with_no_dedicated_nodes_raise_error():
with pytest.raises(InvalidModelError, match="Scheduling target cannot be Dedicated if dedicated vm size is 0"):
conf = ClusterConfiguration(
cluster_id="abc",
scheduling_target=SchedulingTarget.Dedicated,
vm_size="standard_a2",
size=0,
size_low_priority=2,
toolkit=Toolkit(software="spark", version="1.6.3"),
)
conf.validate()
def test_cluster_configuration():
data = {
'toolkit': {
'software': 'spark',
'version': '2.3.0',
'environment': 'anaconda'
},
'vm_size':
'standard_a2',
'size':
2,
'size_low_priority':
3,
'subnet_id':
'/subscriptions/21abd678-18c5-4660-9fdd-8c5ba6b6fe1f/resourceGroups/abc/providers/Microsoft.Network/virtualNetworks/prodtest5vnet/subnets/default',
'plugins': [
JupyterPlugin(),
HDFSPlugin(),
],
'user_configuration': {
'username': '******'
}
}
config = ClusterConfiguration.from_dict(data)
assert isinstance(config.toolkit, Toolkit)
assert config.toolkit.software == 'spark'
assert config.toolkit.version == '2.3.0'
assert config.toolkit.environment == 'anaconda'
assert config.size == 2
assert config.size_low_priority == 3
assert config.vm_size == 'standard_a2'
assert config.subnet_id == '/subscriptions/21abd678-18c5-4660-9fdd-8c5ba6b6fe1f/resourceGroups/abc/providers/Microsoft.Network/virtualNetworks/prodtest5vnet/subnets/default'
assert isinstance(config.user_configuration, UserConfiguration)
assert config.user_configuration.username == 'spark'
assert len(config.plugins) == 2
assert config.plugins[0].name == 'jupyter'
assert config.plugins[1].name == 'hdfs'
def test_size_none():
config = ClusterConfiguration(size=None, size_low_priority=2)
assert config.size == 0
assert config.size_low_priority == 2
def test_size_low_priority_none():
config = ClusterConfiguration(size=1, size_low_priority=None)
assert config.size == 1
assert config.size_low_priority == 0
def test_vm_count_deprecated():
with pytest.warns(DeprecationWarning):
config = ClusterConfiguration(vm_count=3)
assert config.size == 3
with pytest.warns(DeprecationWarning):
config = ClusterConfiguration(vm_low_pri_count=10)
assert config.size_low_priority == 10
with pytest.warns(DeprecationWarning):
config = ClusterConfiguration(size=10)
assert config.vm_count == 10
with pytest.warns(DeprecationWarning):
config = ClusterConfiguration(size_low_priority=10)
assert config.vm_low_pri_count == 10
with pytest.warns(DeprecationWarning):
config = ClusterConfiguration(size=10)
config.vm_count = 20
assert config.size == 20
with pytest.warns(DeprecationWarning):
config = ClusterConfiguration(size_low_priority=10)
config.vm_low_pri_count = 20
assert config.size_low_priority == 20