def _k8s_launcher(config: Config) -> JobLauncher: from feast.pyspark.launchers import k8s return k8s.KubernetesJobLauncher( namespace=config.get(opt.SPARK_K8S_NAMESPACE), resource_template_path=config.get(opt.SPARK_K8S_JOB_TEMPLATE_PATH, None), staging_location=config.get(opt.SPARK_STAGING_LOCATION), incluster=config.getboolean(opt.SPARK_K8S_USE_INCLUSTER_CONFIG), )
def _k8s_launcher(config: Config) -> JobLauncher: from feast.pyspark.launchers import k8s staging_location = config.get(opt.SPARK_STAGING_LOCATION) staging_uri = urlparse(staging_location) return k8s.KubernetesJobLauncher( namespace=config.get(opt.SPARK_K8S_NAMESPACE), resource_template_path=config.get(opt.SPARK_K8S_JOB_TEMPLATE_PATH, None), staging_location=staging_location, incluster=config.getboolean(opt.SPARK_K8S_USE_INCLUSTER_CONFIG), staging_client=get_staging_client(staging_uri.scheme, config), # azure-related arguments are None if not using Azure blob storage azure_account_name=config.get(opt.AZURE_BLOB_ACCOUNT_NAME, None), azure_account_key=config.get(opt.AZURE_BLOB_ACCOUNT_ACCESS_KEY, None), )