def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_screts()) log.info('-------------------------------------------') log.info('spark cluster id: {}'.format(args.cluster_id)) log.info('username: {}'.format(args.username)) log.info('-------------------------------------------') if args.ssh_key: ssh_key = args.ssh_key else: ssh_key = spark_client.secrets_config.ssh_pub_key ssh_key, password = utils.get_ssh_key_or_prompt(ssh_key, args.username, args.password, spark_client.secrets_config) spark_client.create_user( cluster_id=args.cluster_id, username=args.username, password=password, ssh_key=ssh_key ) if password: log.info('password: %s', '*' * len(password)) elif ssh_key: log.info('ssh public key: %s', ssh_key) log.info('-------------------------------------------')
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_screts()) cluster_conf = ClusterConfiguration() cluster_conf.spark_configuration = load_aztk_spark_config() # read cluster.yaml configuartion file, overwrite values with args file_config, wait = config.read_cluster_config() cluster_conf.merge(file_config) cluster_conf.merge( ClusterConfiguration(cluster_id=args.cluster_id, vm_count=args.size, vm_low_pri_count=args.size_low_pri, vm_size=args.vm_size, subnet_id=args.subnet_id, user_configuration=UserConfiguration( username=args.username, password=args.password, ), docker_repo=args.docker_repo)) wait = wait if args.wait is None else args.wait user_configuration = cluster_conf.user_configuration if user_configuration and user_configuration.username: ssh_key, password = utils.get_ssh_key_or_prompt( spark_client.secrets_config.ssh_pub_key, user_configuration.username, user_configuration.password, spark_client.secrets_config) cluster_conf.user_configuration = aztk.spark.models.UserConfiguration( username=user_configuration.username, password=password, ssh_key=ssh_key) else: cluster_conf.user_configuration = None print_cluster_conf(cluster_conf, wait) spinner = utils.Spinner() spinner.start() # create spark cluster cluster = spark_client.create_cluster(cluster_conf, wait=wait) spinner.stop() if wait: log.info("Cluster %s created successfully.", cluster.id) else: log.info("Cluster %s is being provisioned.", cluster.id)
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_screts()) # read cluster.yaml configuartion file, overwrite values with args cluster_conf = ClusterConfig() cluster_conf.merge(uid=args.cluster_id, size=args.size, size_low_pri=args.size_low_pri, vm_size=args.vm_size, subnet_id=args.subnet_id, wait=args.wait, username=args.username, password=args.password, docker_repo=args.docker_repo) if cluster_conf.custom_scripts: custom_scripts = [] for custom_script in cluster_conf.custom_scripts: custom_scripts.append( aztk.spark.models.CustomScript(script=custom_script['script'], run_on=custom_script['runOn'])) else: custom_scripts = None if cluster_conf.file_shares: file_shares = [] for file_share in cluster_conf.file_shares: file_shares.append( aztk.spark.models.FileShare( storage_account_name=file_share['storage_account_name'], storage_account_key=file_share['storage_account_key'], file_share_path=file_share['file_share_path'], mount_path=file_share['mount_path'])) else: file_shares = None if cluster_conf.username: ssh_key, password = utils.get_ssh_key_or_prompt( spark_client.secrets_config.ssh_pub_key, cluster_conf.username, cluster_conf.password, spark_client.secrets_config) user_conf = aztk.spark.models.UserConfiguration( username=cluster_conf.username, password=password, ssh_key=ssh_key) else: user_conf = None print_cluster_conf(cluster_conf) spinner = utils.Spinner() spinner.start() # create spark cluster cluster = spark_client.create_cluster( aztk.spark.models.ClusterConfiguration( cluster_id=cluster_conf.uid, vm_count=cluster_conf.size, vm_low_pri_count=cluster_conf.size_low_pri, vm_size=cluster_conf.vm_size, subnet_id=cluster_conf.subnet_id, custom_scripts=custom_scripts, file_shares=file_shares, docker_repo=cluster_conf.docker_repo, spark_configuration=load_aztk_spark_config(), user_configuration=user_conf), wait=cluster_conf.wait) spinner.stop() if cluster_conf.wait: log.info("Cluster %s created successfully.", cluster.id) else: log.info("Cluster %s is being provisioned.", cluster.id)
def execute(args: typing.NamedTuple): spark_client = load_spark_client() # read cluster.yaml configuartion file, overwrite values with args cluster_conf = ClusterConfig() cluster_conf.merge(uid=args.cluster_id, size=args.size, size_low_pri=args.size_low_pri, vm_size=args.vm_size, wait=args.wait, username=args.username, password=args.password, docker_repo=args.docker_repo) print_cluster_conf(cluster_conf) if cluster_conf.custom_scripts: custom_scripts = [] for custom_script in cluster_conf.custom_scripts: custom_scripts.append( aztk.spark.models.CustomScript(script=custom_script['script'], run_on=custom_script['runOn'])) else: custom_scripts = None if cluster_conf.file_shares: file_shares = [] for file_share in cluster_conf.file_shares: file_shares.append( aztk.spark.models.FileShare( storage_account_name=file_share['storage_account_name'], storage_account_key=file_share['storage_account_key'], file_share_path=file_share['file_share_path'], mount_path=file_share['mount_path'])) else: file_shares = None jars_src = aztk.utils.constants.DEFAULT_SPARK_JARS_SOURCE # create spark cluster cluster = spark_client.create_cluster( aztk.spark.models.ClusterConfiguration( cluster_id=cluster_conf.uid, vm_count=cluster_conf.size, vm_low_pri_count=cluster_conf.size_low_pri, vm_size=cluster_conf.vm_size, custom_scripts=custom_scripts, file_shares=file_shares, docker_repo=cluster_conf.docker_repo, spark_configuration=aztk.spark.models.SparkConfiguration( spark_defaults_conf=os.path.join( aztk.utils.constants.DEFAULT_SPARK_CONF_SOURCE, 'spark-defaults.conf'), spark_env_sh=os.path.join( aztk.utils.constants.DEFAULT_SPARK_CONF_SOURCE, 'spark-env.sh'), core_site_xml=os.path.join( aztk.utils.constants.DEFAULT_SPARK_CONF_SOURCE, 'core-site.xml'), jars=[ os.path.join(jars_src, path) for path in os.listdir(jars_src) ])), wait=cluster_conf.wait) if cluster_conf.username: ssh_key = spark_client.secrets_config.ssh_pub_key ssh_key, password = utils.get_ssh_key_or_prompt( ssh_key, cluster_conf.username, cluster_conf.password, spark_client.secrets_config) spark_client.create_user(cluster_id=cluster_conf.uid, username=cluster_conf.username, password=password, ssh_key=ssh_key) if cluster_conf.wait: log.info("Cluster %s created successfully.", cluster.id) else: log.info("Cluster %s is being provisioned.", cluster.id)