def _get_api(cluster):
""" Get custom objects api associated with a cluster specifier.
"""
if cluster is None:
load_incluster_config()
api = CustomObjectsApi()
elif cluster == 'remote_transcode':
host = os.getenv('REMOTE_TRANSCODE_HOST')
port = os.getenv('REMOTE_TRANSCODE_PORT')
token = os.getenv('REMOTE_TRANSCODE_TOKEN')
cert = os.getenv('REMOTE_TRANSCODE_CERT')
conf = Configuration()
conf.api_key['authorization'] = token
conf.host = f'https://{host}:{port}'
conf.verify_ssl = True
conf.ssl_ca_cert = cert
api_client = ApiClient(conf)
api = CustomObjectsApi(api_client)
else:
cluster_obj = JobCluster.objects.get(pk=cluster)
host = cluster_obj.host
port = cluster_obj.port
token = cluster_obj.token
fd, cert = tempfile.mkstemp(text=True)
with open(fd, 'w') as f:
f.write(cluster_obj.cert)
conf = Configuration()
conf.api_key['authorization'] = token
conf.host = f'https://{host}:{port}'
conf.verify_ssl = True
conf.ssl_ca_cert = cert
api_client = ApiClient(conf)
api = CustomObjectsApi(api_client)
return api
def connect():
config_file = None
if os.environ.get('RD_CONFIG_ENV') == 'incluster':
config.load_incluster_config()
return
if os.environ.get('RD_CONFIG_CONFIG_FILE'):
config_file = os.environ.get('RD_CONFIG_CONFIG_FILE')
url = None
if os.environ.get('RD_CONFIG_URL'):
url = os.environ.get('RD_CONFIG_URL')
verify_ssl = None
if os.environ.get('RD_CONFIG_VERIFY_SSL'):
verify_ssl = os.environ.get('RD_CONFIG_VERIFY_SSL')
ssl_ca_cert = None
if os.environ.get('RD_CONFIG_SSL_CA_CERT'):
ssl_ca_cert = os.environ.get('RD_CONFIG_SSL_CA_CERT')
token = None
if os.environ.get('RD_CONFIG_TOKEN'):
token = os.environ.get('RD_CONFIG_TOKEN')
log.debug("config file")
log.debug(config_file)
log.debug("-------------------")
if config_file:
log.debug("getting settings from file %s" % config_file)
config.load_kube_config(config_file=config_file)
else:
if url:
log.debug("getting settings from pluing configuration")
configuration = Configuration()
configuration.host = url
if verify_ssl == 'true':
configuration.verify_ssl = verify_ssl
else:
configuration.verify_ssl = None
if ssl_ca_cert:
configuration.ssl_ca_cert = ssl_ca_cert
configuration.api_key['authorization'] = token
configuration.api_key_prefix['authorization'] = 'Bearer'
client.Configuration.set_default(configuration)
else:
log.debug("getting from default config file")
config.load_kube_config()
c = Configuration()
c.assert_hostname = False
Configuration.set_default(c)
def __init__(self, alg):
""" Intializes the connection. If algorithm object includes
a remote cluster, use that. Otherwise, use this cluster.
"""
if alg.cluster:
host = alg.cluster.host
port = alg.cluster.port
token = alg.cluster.token
fd, cert = tempfile.mkstemp(text=True)
with open(fd, 'w') as f:
f.write(alg.cluster.cert)
conf = Configuration()
conf.api_key['authorization'] = token
conf.host = f'{PROTO}{host}:{port}'
conf.verify_ssl = True
conf.ssl_ca_cert = cert
api_client = ApiClient(conf)
self.corev1 = CoreV1Api(api_client)
self.custom = CustomObjectsApi(api_client)
else:
load_incluster_config()
self.corev1 = CoreV1Api()
self.custom = CustomObjectsApi()
# Read in the manifest.
if alg.manifest:
self.manifest = yaml.safe_load(alg.manifest.open(mode='r'))
# Save off the algorithm.
self.alg = alg
def build_k8s_config(config_path):
cluster_path = os.path.join(config_path, "cluster.yaml")
if not os.path.isfile(cluster_path):
cluster_path = os.path.join(config_path, STATUS_YAML)
with open(cluster_path) as f:
cluster_config = yaml.full_load(f)
config = Configuration()
infra_host = find_infra_node_name(cluster_config["machines"])
if os.path.isfile(cluster_path):
config.host = "https://%s.%s:1443" % (
infra_host, cluster_config["network"]["domain"])
basic_auth = cluster_config["basic_auth"]
else:
config.host = cluster_config["machines"][infra_host]["fqdns"]
with open(os.path.join(config_path, "clusterID",
"k8s_basic_auth.yml")) as auf:
basic_auth = yaml.safe_load(auf)["basic_auth"]
config.username = basic_auth.split(",")[1]
config.password = basic_auth.split(",")[0]
bearer = "%s:%s" % (config.username, config.password)
encoded = base64.b64encode(bearer.encode("utf-8")).decode("utf-8")
config.api_key["authorization"] = "Basic " + encoded
config.ssl_ca_cert = os.path.join(config_path, "ssl/apiserver/ca.pem")
return config
def __init__(self):
""" Intializes the connection. If environment variables for
remote transcode are defined, connect to that cluster.
"""
host = os.getenv('REMOTE_TRANSCODE_HOST')
port = os.getenv('REMOTE_TRANSCODE_PORT')
token = os.getenv('REMOTE_TRANSCODE_TOKEN')
cert = os.getenv('REMOTE_TRANSCODE_CERT')
self.remote = host is not None
if self.remote:
conf = Configuration()
conf.api_key['authorization'] = token
conf.host = f'https://{host}:{port}'
conf.verify_ssl = True
conf.ssl_ca_cert = cert
api_client = ApiClient(conf)
self.corev1 = CoreV1Api(api_client)
self.custom = CustomObjectsApi(api_client)
else:
load_incluster_config()
self.corev1 = CoreV1Api()
self.custom = CustomObjectsApi()
self.setup_common_steps()
def _set_config(self):
configuration = Configuration()
configuration.host = CniService.config.host
configuration.ssl_ca_cert = CniService.ssl_ca_cert
configuration.api_key['authorization'] = CniService.config.api_key[
'authorization']
Configuration.set_default(configuration)
def mock_kube_get_kubernetes_config(obj):
config = Configuration()
config.host = FAKE_API_ENDPOINT
config.ssl_ca_cert = self.ssl_ca_file.name
config.cert_file = self.cert_file.name
config.key_file = self.key_file.name
return config
def _refresh_oidc(self, provider):
config = Configuration()
if 'idp-certificate-authority-data' in provider['config']:
ca_cert = tempfile.NamedTemporaryFile(delete=True)
if PY3:
cert = base64.b64decode(
provider['config']['idp-certificate-authority-data']
).decode('utf-8')
else:
cert = base64.b64decode(
provider['config']['idp-certificate-authority-data'] + "=="
)
with open(ca_cert.name, 'w') as fh:
fh.write(cert)
config.ssl_ca_cert = ca_cert.name
else:
config.verify_ssl = False
client = ApiClient(configuration=config)
response = client.request(
method="GET",
url="%s/.well-known/openid-configuration"
% provider['config']['idp-issuer-url']
)
if response.status != 200:
return
response = json.loads(response.data)
request = OAuth2Session(
client_id=provider['config']['client-id'],
token=provider['config']['refresh-token'],
auto_refresh_kwargs={
'client_id': provider['config']['client-id'],
'client_secret': provider['config']['client-secret']
},
auto_refresh_url=response['token_endpoint']
)
try:
refresh = request.refresh_token(
token_url=response['token_endpoint'],
refresh_token=provider['config']['refresh-token'],
auth=(provider['config']['client-id'],
provider['config']['client-secret']),
verify=config.ssl_ca_cert if config.verify_ssl else None
)
except oauthlib.oauth2.rfc6749.errors.InvalidClientIdError:
return
provider['config'].value['id-token'] = refresh['id_token']
provider['config'].value['refresh-token'] = refresh['refresh_token']
def connect():
config_file = None
if os.environ.get('RD_CONFIG_ENV') == 'incluster':
config.load_incluster_config()
return
if os.environ.get('RD_CONFIG_CONFIG_FILE'):
config_file = os.environ.get('RD_CONFIG_CONFIG_FILE')
elif os.environ.get('RD_NODE_KUBERNETES_CONFIG_FILE'):
config_file = os.environ.get('RD_NODE_KUBERNETES_CONFIG_FILE')
verify_ssl = os.environ.get('RD_CONFIG_VERIFY_SSL')
ssl_ca_cert = os.environ.get('RD_CONFIG_SSL_CA_CERT')
url = os.environ.get('RD_CONFIG_URL')
token = os.environ.get('RD_CONFIG_TOKEN')
if not token:
token = os.environ.get('RD_CONFIG_TOKEN_STORAGE_PATH')
log.debug("config file")
log.debug(config_file)
log.debug("-------------------")
if config_file:
log.debug("getting settings from file %s", config_file)
config.load_kube_config(config_file=config_file)
else:
if url and token:
log.debug("getting settings from plugin configuration")
configuration = Configuration()
configuration.host = url
if verify_ssl == 'true':
configuration.verify_ssl = verify_ssl
else:
configuration.verify_ssl = None
configuration.assert_hostname = False
if ssl_ca_cert:
configuration.ssl_ca_cert = ssl_ca_cert
configuration.api_key['authorization'] = token
configuration.api_key_prefix['authorization'] = 'Bearer'
client.Configuration.set_default(configuration)
else:
log.debug(
"Either URL or Token is not defined. Fall back to getting settings from default config file [$home/.kube/config]"
)
config.load_kube_config()
def _set_config(self, refresh_token):
configuration = Configuration()
configuration.host = self.host
configuration.ssl_ca_cert = self.ssl_ca_cert
configuration.api_key['authorization'] = "bearer " + self.token
Configuration.set_default(configuration)
if not refresh_token:
return
def wrap(f):
in_cluster_config = self
def wrapped(self, identifier):
if identifier == 'authorization' and identifier in self.api_key and in_cluster_config.token_expires_at <= datetime.datetime.now():
in_cluster_config._read_token_file()
self.api_key[identifier] = "bearer " + in_cluster_config.token
return f(self, identifier)
return wrapped
Configuration.get_api_key_with_prefix = wrap(Configuration.get_api_key_with_prefix)
def __init__(self, alg):
""" Intializes the connection. If algorithm object includes
a remote cluster, use that. Otherwise, use this cluster.
"""
if alg.cluster:
host = alg.cluster.host
port = alg.cluster.port
token = alg.cluster.token
fd, cert = tempfile.mkstemp(text=True)
with open(fd, 'w') as f:
f.write(alg.cluster.cert)
conf = Configuration()
conf.api_key['authorization'] = token
conf.host = f'{PROTO}{host}:{port}'
conf.verify_ssl = True
conf.ssl_ca_cert = cert
api_client = ApiClient(conf)
self.corev1 = CoreV1Api(api_client)
self.custom = CustomObjectsApi(api_client)
else:
load_incluster_config()
self.corev1 = CoreV1Api()
self.custom = CustomObjectsApi()
# Read in the manifest.
if alg.manifest:
self.manifest = yaml.safe_load(alg.manifest.open(mode='r'))
if 'volumeClaimTemplates' in self.manifest['spec']:
for claim in self.manifest['spec']['volumeClaimTemplates']:
storage_class_name = claim['spec'].get(
'storageClassName', None)
if storage_class_name is None:
claim['storageClassName'] = os.getenv(
'WORKFLOW_STORAGE_CLASS')
logger.warning(
f"Implicitly sc to pvc of Algo:{alg.pk}")
# Save off the algorithm.
self.alg = alg
def _set_config(self):
configuration = Configuration()
configuration.host = self.host
configuration.ssl_ca_cert = self.ssl_ca_cert
configuration.api_key['authorization'] = "bearer " + self.token
Configuration.set_default(configuration)
def _set_config(self):
configuration = Configuration()
configuration.host = self.host
configuration.ssl_ca_cert = self.ssl_ca_cert
configuration.api_key['authorization'] = "bearer " + self.token
Configuration.set_default(configuration)
