def _k8s_launcher(config: Config) -> JobLauncher:
from feast.pyspark.launchers import k8s
return k8s.KubernetesJobLauncher(
namespace=config.get(opt.SPARK_K8S_NAMESPACE),
resource_template_path=config.get(opt.SPARK_K8S_JOB_TEMPLATE_PATH, None),
staging_location=config.get(opt.SPARK_STAGING_LOCATION),
incluster=config.getboolean(opt.SPARK_K8S_USE_INCLUSTER_CONFIG),
)
def _k8s_launcher(config: Config) -> JobLauncher:
from feast.pyspark.launchers import k8s
staging_location = config.get(opt.SPARK_STAGING_LOCATION)
staging_uri = urlparse(staging_location)
return k8s.KubernetesJobLauncher(
namespace=config.get(opt.SPARK_K8S_NAMESPACE),
resource_template_path=config.get(opt.SPARK_K8S_JOB_TEMPLATE_PATH, None),
staging_location=staging_location,
incluster=config.getboolean(opt.SPARK_K8S_USE_INCLUSTER_CONFIG),
staging_client=get_staging_client(staging_uri.scheme, config),
# azure-related arguments are None if not using Azure blob storage
azure_account_name=config.get(opt.AZURE_BLOB_ACCOUNT_NAME, None),
azure_account_key=config.get(opt.AZURE_BLOB_ACCOUNT_ACCESS_KEY, None),
)