def __validate__(self):
if self.software not in TOOLKIT_MAP:
raise InvalidModelError(
"Toolkit '{0}' is not in the list of allowed toolkits {1}".
format(self.software, list(TOOLKIT_MAP.keys())))
toolkit_def = TOOLKIT_MAP[self.software]
if self.version not in toolkit_def.versions:
raise InvalidModelError(
"Toolkit '{0}' with version '{1}' is not available. Use one of: {2}"
.format(self.software, self.version, toolkit_def.versions))
if self.version == "1.6":
deprecate("0.9.0",
"Spark version 1.6 is being deprecated for Aztk.",
"Please use 2.1 and above.")
if self.environment:
if self.environment not in toolkit_def.environments:
raise InvalidModelError(
"Environment '{0}' for toolkit '{1}' is not available. Use one of: {2}"
.format(self.environment, self.software,
list(toolkit_def.environments.keys())))
env_def = toolkit_def.environments[self.environment]
if self.environment_version and self.environment_version not in env_def.versions:
raise InvalidModelError(
"Environment '{0}' version '{1}' for toolkit '{2}' is not available. Use one of: {3}"
.format(self.environment, self.environment_version,
self.software, env_def.versions))
def validate(self) -> bool:
"""
Validate the config at its current state.
Raises: Error if invalid
"""
if self.toolkit is None:
raise error.InvalidModelError(
"Please supply a toolkit for the cluster")
self.toolkit.validate()
if self.cluster_id is None:
raise error.AztkError(
"Please supply an id for the cluster with a parameter (--id)")
if self.vm_count == 0 and self.vm_low_pri_count == 0:
raise error.AztkError(
"Please supply a valid (greater than 0) size or size_low_pri value either in the cluster.yaml configuration file or with a parameter (--size or --size-low-pri)"
)
if self.vm_size is None:
raise error.AztkError(
"Please supply a vm_size in either the cluster.yaml configuration file or with a parameter (--vm-size)"
)
if self.mixed_mode() and not self.subnet_id:
raise error.AztkError(
"You must configure a VNET to use AZTK in mixed mode (dedicated and low priority nodes). Set the VNET's subnet_id in your cluster.yaml."
)
if self.custom_scripts:
deprecate(
"Custom scripts are DEPRECATED and will be removed in 0.8.0. Use plugins instead See https://aztk.readthedocs.io/en/v0.7.0/15-plugins.html"
)
def _merge_secrets_dict(secrets: SecretsConfiguration, secrets_config):
if 'default' in secrets_config:
deprecate("0.9.0", "default key in secrets.yaml is deprecated.",
"Place all child parameters directly at the root")
secrets_config = dict(**secrets_config,
**secrets_config.pop('default'))
other = SecretsConfiguration.from_dict(secrets_config)
secrets.merge(other)
def execute(args: typing.NamedTuple):
spark_client = aztk.spark.Client(config.load_aztk_secrets())
cluster_conf = ClusterConfiguration()
cluster_conf.spark_configuration = load_aztk_spark_config()
# read cluster.yaml configuration file, overwrite values with args
file_config, wait = config.read_cluster_config()
cluster_conf.merge(file_config)
if args.size_low_pri is not None:
deprecate("0.9.0", "--size-low-pri has been deprecated.", "Please use --size-low-priority.")
args.size_low_priority = args.size_low_pri
cluster_conf.merge(ClusterConfiguration(
cluster_id=args.cluster_id,
size=args.size,
size_low_priority=args.size_low_priority,
vm_size=args.vm_size,
subnet_id=args.subnet_id,
user_configuration=UserConfiguration(
username=args.username,
password=args.password,
)))
if args.docker_repo and cluster_conf.toolkit:
cluster_conf.toolkit.docker_repo = args.docker_repo
wait = wait if args.wait is None else args.wait
user_configuration = cluster_conf.user_configuration
if user_configuration and user_configuration.username:
ssh_key, password = utils.get_ssh_key_or_prompt(spark_client.secrets_config.ssh_pub_key,
user_configuration.username,
user_configuration.password,
spark_client.secrets_config)
cluster_conf.user_configuration = aztk.spark.models.UserConfiguration(
username=user_configuration.username,
password=password,
ssh_key=ssh_key
)
else:
cluster_conf.user_configuration = None
cluster_conf.validate()
utils.print_cluster_conf(cluster_conf, wait)
with utils.Spinner():
# create spark cluster
cluster = spark_client.create_cluster(
cluster_conf,
wait=wait
)
if wait:
log.info("Cluster %s created successfully.", cluster.id)
else:
log.info("Cluster %s is being provisioned.", cluster.id)
def __init__(self, *args, **kwargs):
if 'vm_count' in kwargs:
deprecate("0.9.0", "vm_count is deprecated for ClusterConfiguration.", "Please use size instead.")
kwargs['size'] = kwargs.pop('vm_count')
if 'vm_low_pri_count' in kwargs:
deprecate("vm_low_pri_count is deprecated for ClusterConfiguration.", "Please use size_low_priority instead.")
kwargs['size_low_priority'] = kwargs.pop('vm_low_pri_count')
super().__init__(*args, **kwargs)
def __init__(self, secrets_configuration: models.SecretsConfiguration = None, **kwargs):
self.secrets_configuration = None
context = None
if kwargs.get("secrets_config"):
deprecate(
version="0.10.0",
message="secrets_config key is deprecated in secrets.yaml",
advice="Please use secrets_configuration key instead.")
context = self._get_context(kwargs.get("secrets_config"))
else:
context = self._get_context(secrets_configuration)
self.cluster = ClusterOperations(context)
self.job = JobOperations(context)
def __validate__(self) -> bool:
if self.size == 0 and self.size_low_priority == 0:
raise error.InvalidModelError(
"Please supply a valid (greater than 0) size or size_low_priority value either in the cluster.yaml configuration file or with a parameter (--size or --size-low-pri)"
)
if self.vm_size is None:
raise error.InvalidModelError(
"Please supply a vm_size in either the cluster.yaml configuration file or with a parameter (--vm-size)"
)
if self.mixed_mode() and not self.subnet_id:
raise error.InvalidModelError(
"You must configure a VNET to use AZTK in mixed mode (dedicated and low priority nodes). Set the VNET's subnet_id in your cluster.yaml."
)
if self.custom_scripts:
deprecate("0.9.0", "Custom scripts are DEPRECATED.", "Use plugins instead. See https://aztk.readthedocs.io/en/v0.7.0/15-plugins.html.")
if self.scheduling_target == SchedulingTarget.Dedicated and self.size == 0:
raise error.InvalidModelError("Scheduling target cannot be Dedicated if dedicated vm size is 0")