def get_ssh_key_or_prompt(ssh_key, username, password, secrets_config):
ssh_key = get_ssh_key.get_user_public_key(ssh_key, secrets_config)
if username is not None and password is None and ssh_key is None:
password = getpass.getpass("Please input a password for user '{0}': ".format(username))
confirm_password = getpass.getpass("Please confirm your password for user '{0}': ".format(username))
if password != confirm_password:
raise error.AztkError("Password confirmation did not match, please try again.")
if not password:
raise error.AztkError(
"Password is empty, cannot add user to cluster. Provide a ssh public key in .aztk/secrets.yaml. Or provide an ssh-key or password with commnad line parameters (--ssh-key or --password).")
return ssh_key, password
def wait_until_application_done(self, cluster_id: str, task_id: str):
try:
helpers.wait_for_task_to_complete(job_id=cluster_id,
task_id=task_id,
batch_client=self.batch_client)
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
def create_cluster(self,
cluster_conf: models.ClusterConfiguration,
wait: bool = False):
try:
zip_resource_files = upload_node_scripts.zip_scripts(
self.blob_client, cluster_conf.custom_scripts,
cluster_conf.spark_configuration)
start_task = create_cluster_helper.generate_cluster_start_task(
self, zip_resource_files, cluster_conf.docker_repo)
software_metadata_key = "spark"
vm_image = models.VmImage(publisher='Canonical',
offer='UbuntuServer',
sku='16.04')
cluster = self.__create_pool_and_job(cluster_conf,
software_metadata_key,
start_task, vm_image)
# Wait for the master to be ready
if wait:
util.wait_for_master_to_be_ready(self, cluster.id)
cluster = self.get_cluster(cluster.id)
return cluster
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
def list_clusters(self):
try:
return [
models.Cluster(pool) for pool in self.__list_clusters(
aztk_sdk.models.Software.spark)
]
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
def wait_until_cluster_is_ready(self, cluster_id: str):
try:
util.wait_for_master_to_be_ready(self, cluster_id)
pool = self.batch_client.pool.get(cluster_id)
nodes = self.batch_client.compute_node.list(pool_id=cluster_id)
return models.Cluster(pool, nodes)
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
def submit(self,
cluster_id: str,
application: models.AppModel,
wait: bool = False):
try:
submit_helper.submit_application(self, cluster_id, application,
wait)
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
def get_application_log(self,
cluster_id: str,
application_name: str,
tail=False,
current_bytes: int = 0):
try:
return get_log_helper.get_log(self.batch_client, cluster_id,
application_name, tail,
current_bytes)
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
def create_user(self,
cluster_id: str,
username: str,
password: str = None,
ssh_key: str = None) -> str:
try:
cluster = self.get_cluster(cluster_id)
master_node_id = cluster.master_node_id
self.__create_user(cluster.id, master_node_id, username, password,
ssh_key)
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
def create_pool_if_not_exist(pool, batch_client):
"""
Creates the specified pool if it doesn't already exist
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param pool: The pool to create.
:type pool: `batchserviceclient.models.PoolAddParameter`
"""
try:
batch_client.pool.add(pool)
except batch_models.BatchErrorException as e:
if e.error.code == "PoolExists":
raise error.AztkError(
"A cluster with the same id already exists. Use a different id or delete the existing cluster"
)
else:
raise
return True
def get_log(batch_client,
cluster_id: str,
application_name: str,
tail=False,
current_bytes: int = 0):
job_id = cluster_id
task_id = application_name
task = __wait_for_app_to_be_running(batch_client, cluster_id,
application_name)
if not __check_task_node_exist(batch_client, cluster_id, task):
raise error.AztkError("The node the app ran on doesn't exist anymore!")
file = __get_output_file_properties(batch_client, cluster_id,
application_name)
target_bytes = file.content_length
if target_bytes != current_bytes:
ocp_range = None
if tail:
ocp_range = "bytes={0}-{1}".format(current_bytes, target_bytes - 1)
stream = batch_client.file.get_from_task(
job_id, task_id, output_file,
batch_models.FileGetFromTaskOptions(ocp_range=ocp_range))
content = helpers.read_stream_as_string(stream)
return models.AppLogsModel(name=application_name,
cluster_id=cluster_id,
application_state=task.state._value_,
log=content,
total_bytes=target_bytes)
else:
return models.AppLogsModel(name=application_name,
cluster_id=cluster_id,
application_state=task.state._value_,
log='',
total_bytes=target_bytes)
def get_remote_login_settings(self, cluster_id: str, node_id: str):
try:
return self.__get_remote_login_settings(cluster_id, node_id)
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
def get_cluster(self, cluster_id: str):
try:
pool, nodes = self.__get_pool_details(cluster_id)
return models.Cluster(pool, nodes)
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
def delete_cluster(self, cluster_id: str):
try:
return self.__delete_pool_and_job(cluster_id)
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))
def get_application_status(self, cluster_id: str, app_name: str):
try:
task = self.batch_client.task.get(cluster_id, app_name)
return task.state._value_
except batch_error.BatchErrorException as e:
raise error.AztkError(helpers.format_batch_exception(e))