def create_directory(dest_path: str, environment: str): config_src_path = constants.INIT_DIRECTORY_SOURCE config_dest_path = dest_path if os.path.isdir(config_dest_path): log.warning("This directory has already been initialized.") return copy_tree(config_src_path, config_dest_path, update=1) secrets_template_path = os.path.join(dest_path, "secrets.yaml.template") secrets_path = os.path.join(dest_path, "secrets.yaml") if os.path.isfile(secrets_path): os.remove(secrets_template_path) if os.path.isfile( secrets_template_path) and not os.path.isfile(secrets_path): os.rename(secrets_template_path, secrets_path) cluster_path = os.path.join(dest_path, "cluster.yaml") if os.path.isfile(cluster_path): with open(cluster_path, "r", encoding="UTF-8") as stream: cluster_yaml = stream.read() cluster_yaml = cluster_yaml.replace("{environment}", "{}\n".format(environment)) with open(cluster_path, "w", encoding="UTF-8") as file: file.write(cluster_yaml)
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) cluster_ids = args.cluster_ids for cluster_id in cluster_ids: if not args.force: if not args.keep_logs: log.warning( "All logs persisted for this cluster will be deleted.") confirmation_cluster_id = input( "Please confirm the id of the cluster you wish to delete [{}]: " .format(cluster_id)) if confirmation_cluster_id != cluster_id: log.error( "Confirmation cluster id does not match. Please try again." ) return if spark_client.cluster.delete(id=cluster_id, keep_logs=args.keep_logs): log.info("Deleting cluster %s", cluster_id) else: log.error( "Cluster with id '%s' doesn't exist or was already deleted.", cluster_id)
def native_python_ssh_into_master(spark_client, cluster, cluster_configuration, ssh_conf, password): if not ssh_conf.connect: log.warning("No ssh client found, using pure python connection.") return plugin_ports = [] if cluster_configuration and cluster_configuration.plugins: ports = [ PortForwardingSpecification(port.internal, port.public_port) for plugin in cluster_configuration.plugins for port in plugin.ports if port.expose_publicly ] plugin_ports.extend(ports) print("Press ctrl+c to exit...") spark_client.cluster.ssh_into_master( cluster.id, ssh_conf.username, ssh_key=None, password=password, port_forward_list=[ PortForwardingSpecification(remote_port=8080, local_port=8080), # web ui PortForwardingSpecification(remote_port=4040, local_port=4040), # job ui PortForwardingSpecification(remote_port=18080, local_port=18080), # job history ui ] + plugin_ports, internal=ssh_conf.internal, )
def execute(args: typing.NamedTuple): spark_client = aztk.spark.Client(config.load_aztk_secrets()) job_id = args.job_id if not args.force: # check if job exists before prompting for confirmation spark_client.get_job(job_id) if not args.keep_logs: log.warning("All logs persisted for this job will be deleted.") confirmation_cluster_id = input("Please confirm the id of the cluster you wish to delete: ") if confirmation_cluster_id != job_id: log.error("Confirmation cluster id does not match. Please try again.") return if spark_client.delete_job(job_id, args.keep_logs): log.info("Deleting Job %s", job_id) else: log.error("Job with id '%s' doesn't exist or was already deleted.", job_id)
def _show_warn(message, *_args): log.warning(str(message))
def _merge_secrets_dict(secrets: SecretsConfiguration, secrets_config): service_principal_config = secrets_config.get('service_principal') if service_principal_config: secrets.service_principal = ServicePrincipalConfiguration( tenant_id=service_principal_config.get('tenant_id'), client_id=service_principal_config.get('client_id'), credential=service_principal_config.get('credential'), batch_account_resource_id=service_principal_config.get( 'batch_account_resource_id'), storage_account_resource_id=service_principal_config.get( 'storage_account_resource_id'), ) shared_key_config = secrets_config.get('shared_key') batch = secrets_config.get('batch') storage = secrets_config.get('storage') if shared_key_config and (batch or storage): raise aztk.error.AztkError( "Shared keys must be configured either under 'sharedKey:' or under 'batch:' and 'storage:', not both." ) if shared_key_config: secrets.shared_key = SharedKeyConfiguration( batch_account_name=shared_key_config.get('batch_account_name'), batch_account_key=shared_key_config.get('batch_account_key'), batch_service_url=shared_key_config.get('batch_service_url'), storage_account_name=shared_key_config.get('storage_account_name'), storage_account_key=shared_key_config.get('storage_account_key'), storage_account_suffix=shared_key_config.get( 'storage_account_suffix'), ) elif batch or storage: secrets.shared_key = SharedKeyConfiguration() if batch: log.warning( "Your secrets.yaml format is deprecated. To use shared key authentication use the shared_key key. See config/secrets.yaml.template" ) secrets.shared_key.batch_account_name = batch.get( 'batchaccountname') secrets.shared_key.batch_account_key = batch.get('batchaccountkey') secrets.shared_key.batch_service_url = batch.get('batchserviceurl') if storage: secrets.shared_key.storage_account_name = storage.get( 'storageaccountname') secrets.shared_key.storage_account_key = storage.get( 'storageaccountkey') secrets.shared_key.storage_account_suffix = storage.get( 'storageaccountsuffix') docker_config = secrets_config.get('docker') if docker_config: secrets.docker = DockerConfiguration( endpoint=docker_config.get('endpoint'), username=docker_config.get('username'), password=docker_config.get('password'), ) default_config = secrets_config.get('default') # Check for ssh keys if they are provided if default_config: secrets.ssh_priv_key = default_config.get('ssh_priv_key') secrets.ssh_pub_key = default_config.get('ssh_pub_key')