def execute(args: typing.NamedTuple):
spark_client = aztk.spark.Client(config.load_aztk_secrets())
cluster_ids = args.cluster_ids
for cluster_id in cluster_ids:
if not args.force:
if not args.keep_logs:
log.warning(
"All logs persisted for this cluster will be deleted.")
confirmation_cluster_id = input(
"Please confirm the id of the cluster you wish to delete [{}]: "
.format(cluster_id))
if confirmation_cluster_id != cluster_id:
log.error(
"Confirmation cluster id does not match. Please try again."
)
return
if spark_client.cluster.delete(id=cluster_id,
keep_logs=args.keep_logs):
log.info("Deleting cluster %s", cluster_id)
else:
log.error(
"Cluster with id '%s' doesn't exist or was already deleted.",
cluster_id)
def main():
parser = argparse.ArgumentParser(prog=constants.CLI_EXE)
setup_common_args(parser)
subparsers = parser.add_subparsers(title="Available Softwares",
dest="software",
metavar="<software>")
subparsers.required = True
spark_parser = subparsers.add_parser("spark",
help="Commands to run spark jobs")
plugins_parser = subparsers.add_parser(
"plugins", help="Commands to list and view plugins")
toolkit_parser = subparsers.add_parser(
"toolkit",
help="List current toolkit information and browse available ones")
spark.setup_parser(spark_parser)
plugins.setup_parser(plugins_parser)
toolkit.setup_parser(toolkit_parser)
args = parser.parse_args()
parse_common_args(args)
try:
run_software(args)
except batch_error.BatchErrorException as e:
utils.print_batch_exception(e)
except aztk.error.AztkError as e:
log.error(str(e))
def execute(args: typing.NamedTuple):
spark_client = aztk.spark.Client(config.load_aztk_secrets())
if spark_client.stop_job_app(args.job_id, args.app_name):
log.info("Stopped app {0}".format(args.app_name))
else:
log.error("App with name {0} does not exist or was already deleted")
def execute(args: typing.NamedTuple):
spark_client = aztk.spark.Client(config.load_aztk_secrets())
if spark_client.job.stop_application(args.job_id, args.app_name):
log.info("Stopped app %s", args.app_name)
else:
log.error("App with name %s does not exist or was already deleted",
args.app_name)
def execute(args: typing.NamedTuple):
spark_client = aztk.spark.Client(config.load_aztk_secrets())
cluster_id = args.cluster_id
if not args.force:
confirmation_cluster_id = input("Please confirm the id of the cluster you wish to delete: ")
if confirmation_cluster_id != cluster_id:
log.error("Confirmation cluster id does not match. Please try again.")
return
if spark_client.delete_cluster(cluster_id):
log.info("Deleting cluster %s", cluster_id)
else:
log.error("Cluster with id '%s' doesn't exist or was already deleted.", cluster_id)
def execute(args: typing.NamedTuple):
spark_client = aztk.spark.Client(config.load_aztk_secrets())
job_id = args.job_id
if not args.force:
# check if job exists before prompting for confirmation
spark_client.get_job(job_id)
confirmation_cluster_id = input("Please confirm the id of the cluster you wish to delete: ")
if confirmation_cluster_id != job_id:
log.error("Confirmation cluster id does not match. Please try again.")
return
if spark_client.delete_job(job_id):
log.info("Deleting Job %s", job_id)
else:
log.error("Job with id '%s' doesn't exist or was already deleted.", job_id)
def validate_software(software: str):
if software not in TOOLKIT_MAP:
log.error("Software '%s' is not supported.", software)
print_available_softwares()
return False
return True
