def test_cluster_node_count(admin_mc, remove_resource,
raw_remove_custom_resource):
"""Test that the cluster node count gets updated as nodes are added"""
client = admin_mc.client
cluster = client.create_cluster(
name=random_str(), rancherKubernetesEngineConfig={"accessKey": "junk"})
remove_resource(cluster)
def _check_node_count(cluster, nodes):
c = client.reload(cluster)
return c.nodeCount == nodes
def _node_count_fail(cluster, nodes):
c = client.reload(cluster)
s = "cluster {} failed to have proper node count, expected: {} has: {}"
return s.format(c.id, nodes, c.nodeCount)
node_count = 0
wait_for(lambda: _check_node_count(cluster, node_count),
fail_handler=lambda: _node_count_fail(cluster, node_count))
# Nodes have to be created manually through k8s client to attach to a
# pending cluster
k8s_dynamic_client = CustomObjectsApi(admin_mc.k8s_client)
body = {
"metadata": {
"name": random_str(),
"namespace": cluster.id,
},
"kind": "Node",
"apiVersion": "management.cattle.io/v3",
}
dynamic_nt = k8s_dynamic_client.create_namespaced_custom_object(
"management.cattle.io", "v3", cluster.id, 'nodes', body)
raw_remove_custom_resource(dynamic_nt)
node_count = 1
wait_for(lambda: _check_node_count(cluster, node_count),
fail_handler=lambda: _node_count_fail(cluster, node_count))
# Create node number 2
body['metadata']['name'] = random_str()
dynamic_nt1 = k8s_dynamic_client.create_namespaced_custom_object(
"management.cattle.io", "v3", cluster.id, 'nodes', body)
raw_remove_custom_resource(dynamic_nt1)
node_count = 2
wait_for(lambda: _check_node_count(cluster, node_count),
fail_handler=lambda: _node_count_fail(cluster, node_count))
# Delete a node
k8s_dynamic_client.delete_namespaced_custom_object(
"management.cattle.io", "v3", cluster.id, 'nodes',
dynamic_nt1['metadata']['name'], {})
node_count = 1
wait_for(lambda: _check_node_count(cluster, node_count),
fail_handler=lambda: _node_count_fail(cluster, node_count))
def create_resource_from_yaml(custom_objects: CustomObjectsApi, yaml_manifest,
namespace, plural) -> dict:
"""
Create a Resource based on yaml file.
:param custom_objects: CustomObjectsApi
:param yaml_manifest: an absolute path to file
:param namespace:
:param plural: the plural of the resource
:return: a dictionary representing the resource
"""
with open(yaml_manifest) as f:
body = yaml.safe_load(f)
try:
print("Create a Custom Resource: " + body["kind"])
group, version = body["apiVersion"].split("/")
custom_objects.create_namespaced_custom_object(group, version,
namespace, plural, body)
print(
f"Custom resource {body['kind']} created with name '{body['metadata']['name']}'"
)
return body
except ApiException as ex:
logging.exception(
f"Exception: {ex} occurred while creating {body['kind']}: {body['metadata']['name']}"
)
raise
def create_dos_protected_from_yaml(custom_objects: CustomObjectsApi,
yaml_manifest, namespace,
ing_namespace) -> str:
"""
Create a protected resource for Dos based on yaml file.
:param custom_objects: CustomObjectsApi
:param yaml_manifest: an absolute path to file
:param namespace:
:return: str
"""
print("Create Dos Protected:")
with open(yaml_manifest) as f:
dep = yaml.safe_load(f)
dep['spec']['dosSecurityLog']['apDosLogConf'] = dep['spec'][
'dosSecurityLog']['apDosLogConf'].replace("<NAMESPACE>", namespace)
dep['spec']['dosSecurityLog']['dosLogDest'] = dep['spec'][
'dosSecurityLog']['dosLogDest'].replace("<NAMESPACE>",
ing_namespace)
dep['spec']['apDosPolicy'] = dep['spec']['apDosPolicy'].replace(
"<NAMESPACE>", namespace)
custom_objects.create_namespaced_custom_object("appprotectdos.f5.com",
"v1beta1", namespace,
"dosprotectedresources",
dep)
print(
f"DOS Protected resource created with name '{namespace}/{dep['metadata']['name']}'"
)
return dep["metadata"]["name"]
def create_virtual_server_from_yaml(
custom_objects: CustomObjectsApi, yaml_manifest, namespace
) -> str:
"""
Create a VirtualServer based on yaml file.
:param custom_objects: CustomObjectsApi
:param yaml_manifest: an absolute path to file
:param namespace:
:return: str
"""
print("Create a VirtualServer:")
with open(yaml_manifest) as f:
dep = yaml.safe_load(f)
try:
custom_objects.create_namespaced_custom_object(
"k8s.nginx.org", "v1", namespace, "virtualservers", dep
)
print(f"VirtualServer created with name '{dep['metadata']['name']}'")
return dep["metadata"]["name"]
except ApiException as ex:
logging.exception(
f"Exception: {ex} occurred while creating VirtualServer: {dep['metadata']['name']}"
)
raise
def create_ap_multilog_waf_policy_from_yaml(
custom_objects: CustomObjectsApi,
yaml_manifest,
namespace,
ap_namespace,
waf_enable,
log_enable,
appolicy,
aplogconfs,
logdests,
) -> str:
"""
Create a Policy based on yaml file.
:param custom_objects: CustomObjectsApi
:param yaml_manifest: an absolute path to file
:param namespace: namespace for test resources
:param ap_namespace: namespace for AppProtect resources
:param waf_enable: true/false
:param log_enable: true/false
:param appolicy: AppProtect policy name
:param aplogconfs: List of Logconf names
:param logdests: List of AP log destinations (syslog)
:return: str
"""
with open(yaml_manifest) as f:
dep = yaml.safe_load(f)
try:
dep["spec"]["waf"]["enable"] = waf_enable
dep["spec"]["waf"]["apPolicy"] = f"{ap_namespace}/{appolicy}"
seclogs = []
try:
for i in range(len(aplogconfs)):
seclogs.append({
"enable": True,
"apLogConf": f"{ap_namespace}/{aplogconfs[i]}",
"logDest": f"{logdests[i]}"
})
dep["spec"]["waf"]["securityLogs"] = seclogs
except KeyError:
logging.exception(
f"Exception occurred while creating Policy: {dep['metadata']['name']}"
)
raise
del dep["spec"]["waf"]["securityLog"]
custom_objects.create_namespaced_custom_object("k8s.nginx.org", "v1",
namespace, "policies",
dep)
print(f"Policy created: {dep}")
return dep["metadata"]["name"]
except ApiException:
logging.exception(
f"Exception occurred while creating Policy: {dep['metadata']['name']}"
)
raise
def create_ap_policy_from_yaml(custom_objects: CustomObjectsApi, yaml_manifest, namespace) -> str:
"""
Create a policy for AppProtect based on yaml file.
:param custom_objects: CustomObjectsApi
:param yaml_manifest: an absolute path to file
:param namespace:
:return: str
"""
print("Create AP Policy:")
with open(yaml_manifest) as f:
dep = yaml.safe_load(f)
custom_objects.create_namespaced_custom_object(
"appprotect.f5.com", "v1beta1", namespace, "appolicies", dep
)
print(f"AP Policy created with name '{dep['metadata']['name']}'")
return dep["metadata"]["name"]
def create_v_s_route(custom_objects: CustomObjectsApi, vsr, namespace) -> str:
"""
Create a VirtualServerRoute.
:param custom_objects: CustomObjectsApi
:param vsr: a VirtualServerRoute
:param namespace:
:return: str
"""
print("Create a VirtualServerRoute:")
custom_objects.create_namespaced_custom_object("k8s.nginx.org", "v1",
namespace,
"virtualserverroutes", vsr)
print(
f"VirtualServerRoute created with a name '{vsr['metadata']['name']}'")
return vsr["metadata"]["name"]
def create_dos_logconf_from_yaml(custom_objects: CustomObjectsApi,
yaml_manifest, namespace) -> str:
"""
Create a logconf for Dos, based on yaml file.
:param custom_objects: CustomObjectsApi
:param yaml_manifest: an absolute path to file
:param namespace:
:return: str
"""
print("Create DOS logconf:")
with open(yaml_manifest) as f:
dep = yaml.safe_load(f)
custom_objects.create_namespaced_custom_object("appprotectdos.f5.com",
"v1beta1", namespace,
"apdoslogconfs", dep)
print(f"DOS logconf created with name '{dep['metadata']['name']}'")
return dep["metadata"]["name"]
def ensure_custom_object(api: client.CustomObjectsApi, custom_object, group,
plural, version, namespace, name):
if len(
api.list_namespaced_custom_object(
namespace=namespace,
field_selector=f'metadata.name={name}',
group=group,
plural=plural,
version=version)['items']) == 0:
logger.info(f'creating custom object: {namespace}/{name}')
api.create_namespaced_custom_object(body=custom_object,
namespace=namespace,
group=group,
plural=plural,
version=version)
else:
logger.info(f'custom object exists: {namespace}/{name}')
def create_ap_waf_policy_from_yaml(
custom_objects: CustomObjectsApi,
yaml_manifest,
namespace,
ap_namespace,
waf_enable,
log_enable,
appolicy,
aplogconf,
logdest,
) -> str:
"""
Create a Policy based on yaml file.
:param custom_objects: CustomObjectsApi
:param yaml_manifest: an absolute path to file
:param namespace: namespace for test resources
:param ap_namespace: namespace for AppProtect resources
:param waf_enable: true/false
:param log_enable: true/false
:param appolicy: AppProtect policy name
:param aplogconf: Logconf name
:param logdest: AP log destination (syslog)
:return: str
"""
with open(yaml_manifest) as f:
dep = yaml.safe_load(f)
try:
dep["spec"]["waf"]["enable"] = waf_enable
dep["spec"]["waf"]["apPolicy"] = f"{ap_namespace}/{appolicy}"
dep["spec"]["waf"]["securityLog"]["enable"] = log_enable
dep["spec"]["waf"]["securityLog"][
"apLogConf"] = f"{ap_namespace}/{aplogconf}"
dep["spec"]["waf"]["securityLog"]["logDest"] = f"{logdest}"
custom_objects.create_namespaced_custom_object("k8s.nginx.org", "v1",
namespace, "policies",
dep)
print(f"Policy created: {dep}")
return dep["metadata"]["name"]
except ApiException:
logging.exception(
f"Exception occurred while creating Policy: {dep['metadata']['name']}"
)
raise
def create_v_s_route_from_yaml(custom_objects: CustomObjectsApi, yaml_manifest, namespace) -> str:
"""
Create a VirtualServerRoute based on a yaml file.
:param custom_objects: CustomObjectsApi
:param yaml_manifest: an absolute path to a file
:param namespace:
:return: str
"""
print("Create a VirtualServerRoute:")
with open(yaml_manifest) as f:
dep = yaml.safe_load(f)
custom_objects.create_namespaced_custom_object(
"k8s.nginx.org", "v1", namespace, "virtualserverroutes", dep
)
print(f"VirtualServerRoute created with a name '{dep['metadata']['name']}'")
return dep["metadata"]["name"]
def test_legacy_template_migrate_and_delete(admin_mc, admin_cc,
remove_resource, user_mc,
raw_remove_custom_resource):
"""Asserts that any node template not in cattle-global-nt namespace is
duplicated into cattle-global-nt, then deleted"""
admin_client = admin_mc.client
admin_cc_client = admin_cc.client
user_client = user_mc.client
k8s_dynamic_client = CustomObjectsApi(admin_mc.k8s_client)
ns = admin_cc_client.create_namespace(name="ns-" + random_str(),
clusterId=admin_cc.cluster.id)
remove_resource(ns)
node_template_name = "nt-" + random_str()
body = {
"metadata": {
"name": node_template_name,
"annotations": {
"field.cattle.io/creatorId": user_mc.user.id
}
},
"kind": "NodeTemplate",
"apiVersion": "management.cattle.io/v3",
"azureConfig": {
"customData": "asdfsadfsd"
}
}
dynamic_nt = k8s_dynamic_client.create_namespaced_custom_object(
"management.cattle.io", "v3", ns.name, 'nodetemplates', body)
raw_remove_custom_resource(dynamic_nt)
def migrated_template_exists(id):
try:
nt = user_client.by_id_node_template(id=id)
remove_resource(nt)
return nt
except ApiError as e:
assert e.error.status == 403
return False
id = "cattle-global-nt:nt-" + ns.id + "-" + dynamic_nt["metadata"]["name"]
nt = wait_for(
lambda: migrated_template_exists(id),
fail_handler=lambda: "failed waiting for node template to migrate")
# assert that config has not been removed from node template
assert nt.azureConfig["customData"] ==\
dynamic_nt["azureConfig"]["customData"]
wait_for(
lambda: admin_client.by_id_node_template(id=ns.name + ":" +
node_template_name) is None,
fail_handler=lambda: "failed waiting for old node template to delete")
def create_policy_from_yaml(custom_objects: CustomObjectsApi, yaml_manifest, namespace) -> str:
"""
Create a Policy based on yaml file.
:param custom_objects: CustomObjectsApi
:param yaml_manifest: an absolute path to file
:param namespace:
:return: str
"""
print("Create a Policy:")
with open(yaml_manifest) as f:
dep = yaml.safe_load(f)
try:
custom_objects.create_namespaced_custom_object(
"k8s.nginx.org", "v1alpha1", namespace, "policies", dep
)
print(f"Policy created with name '{dep['metadata']['name']}'")
return dep["metadata"]["name"]
except ApiException:
logging.exception(f"Exception occured while creating Policy: {dep['metadata']['name']}")
raise
def create_virtual_server(custom_objects: CustomObjectsApi, vs,
namespace) -> str:
"""
Create a VirtualServer.
:param custom_objects: CustomObjectsApi
:param vs: a VirtualServer
:param namespace:
:return: str
"""
print("Create a VirtualServer:")
try:
custom_objects.create_namespaced_custom_object("k8s.nginx.org", "v1",
namespace,
"virtualservers", vs)
print(f"VirtualServer created with name '{vs['metadata']['name']}'")
return vs["metadata"]["name"]
except ApiException as ex:
logging.exception(
f"Exception: {ex} occurred while creating VirtualServer: {vs['metadata']['name']}"
)
raise
class InfraEnv(BaseCustomResource):
"""
InfraEnv is used to generate cluster iso.
Image is automatically generated on CRD deployment, after InfraEnv is
reconciled. Image download url will be exposed in the status.
"""
_plural = 'infraenvs'
def __init__(
self,
kube_api_client: ApiClient,
name: str,
namespace: str = env_variables['namespace'],
):
super().__init__(name, namespace)
self.crd_api = CustomObjectsApi(kube_api_client)
def create_from_yaml(self, yaml_data: dict) -> None:
self.crd_api.create_namespaced_custom_object(
group=CRD_API_GROUP,
version=CRD_API_VERSION,
plural=self._plural,
body=yaml_data,
namespace=self.ref.namespace,
)
logger.info('created infraEnv %s: %s', self.ref, pformat(yaml_data))
def create(
self,
cluster_deployment: ClusterDeployment,
secret: Secret,
proxy: Optional[Proxy] = None,
label_selector: Optional[Dict[str, str]] = None,
ignition_config_override: Optional[str] = None,
nmstate_label: Optional[str] = None,
**kwargs,
) -> None:
body = {
'apiVersion': f'{CRD_API_GROUP}/{CRD_API_VERSION}',
'kind': 'InfraEnv',
'metadata': self.ref.as_dict(),
'spec': {
'clusterRef': cluster_deployment.ref.as_dict(),
'pullSecretRef': secret.ref.as_dict(),
'nmStateConfigLabelSelector': {
'matchLabels': {
f'{CRD_API_GROUP}/selector-nmstate-config-name':
nmstate_label or ''
}
},
'agentLabelSelector': {
'matchLabels': label_selector or {}
},
'ignitionConfigOverride': ignition_config_override or '',
}
}
spec = body['spec']
if proxy:
spec['proxy'] = proxy.as_dict()
spec.update(kwargs)
self.crd_api.create_namespaced_custom_object(
group=CRD_API_GROUP,
version=CRD_API_VERSION,
plural=self._plural,
body=body,
namespace=self.ref.namespace,
)
logger.info('created infraEnv %s: %s', self.ref, pformat(body))
def patch(
self,
cluster_deployment: Optional[ClusterDeployment],
secret: Optional[Secret],
proxy: Optional[Proxy] = None,
label_selector: Optional[Dict[str, str]] = None,
ignition_config_override: Optional[str] = None,
nmstate_label: Optional[str] = None,
**kwargs,
) -> None:
body = {'spec': kwargs}
spec = body['spec']
if cluster_deployment:
spec['clusterRef'] = cluster_deployment.ref.as_dict()
if secret:
spec['pullSecretRef'] = secret.ref.as_dict()
if proxy:
spec['proxy'] = proxy.as_dict()
if label_selector:
spec['agentLabelSelector'] = {'matchLabels': label_selector}
if nmstate_label:
spec['nmStateConfigLabelSelector'] = {
'matchLabels': {
f'{CRD_API_GROUP}/selector-nmstate-config-name':
nmstate_label,
}
}
if ignition_config_override:
spec['ignitionConfigOverride'] = ignition_config_override
self.crd_api.patch_namespaced_custom_object(
group=CRD_API_GROUP,
version=CRD_API_VERSION,
plural=self._plural,
name=self.ref.name,
namespace=self.ref.namespace,
body=body,
)
logger.info('patching infraEnv %s: %s', self.ref, pformat(body))
def get(self) -> dict:
return self.crd_api.get_namespaced_custom_object(
group=CRD_API_GROUP,
version=CRD_API_VERSION,
plural=self._plural,
name=self.ref.name,
namespace=self.ref.namespace,
)
def delete(self) -> None:
self.crd_api.delete_namespaced_custom_object(
group=CRD_API_GROUP,
version=CRD_API_VERSION,
plural=self._plural,
name=self.ref.name,
namespace=self.ref.namespace,
)
logger.info('deleted infraEnv %s', self.ref)
def status(
self,
timeout: Union[int, float] = DEFAULT_WAIT_FOR_CRD_STATUS_TIMEOUT,
) -> dict:
"""
Status is a section in the CRD that is created after registration to
assisted service and it defines the observed state of InfraEnv.
Since the status key is created only after resource is processed by the
controller in the service, it might take a few seconds before appears.
"""
def _attempt_to_get_status() -> dict:
return self.get()['status']
return waiting.wait(
_attempt_to_get_status,
sleep_seconds=0.5,
timeout_seconds=timeout,
waiting_for=f'infraEnv {self.ref} status',
expected_exceptions=KeyError,
)
def get_iso_download_url(
self,
timeout: Union[int, float] = DEFAULT_WAIT_FOR_ISO_URL_TIMEOUT,
):
def _attempt_to_get_image_url() -> str:
return self.get()['status']['isoDownloadURL']
return waiting.wait(
_attempt_to_get_image_url,
sleep_seconds=3,
timeout_seconds=timeout,
waiting_for='image to be created',
expected_exceptions=KeyError,
)
def get_cluster_id(self):
iso_download_url = self.get_iso_download_url()
return ISO_URL_PATTERN.match(iso_download_url).group('cluster_id')
class TatorAlgorithm(JobManagerMixin):
""" Interface to kubernetes REST API for starting algorithms.
"""
def __init__(self, alg):
""" Intializes the connection. If algorithm object includes
a remote cluster, use that. Otherwise, use this cluster.
"""
if alg.cluster:
host = alg.cluster.host
port = alg.cluster.port
token = alg.cluster.token
fd, cert = tempfile.mkstemp(text=True)
with open(fd, 'w') as f:
f.write(alg.cluster.cert)
conf = Configuration()
conf.api_key['authorization'] = token
conf.host = f'{PROTO}{host}:{port}'
conf.verify_ssl = True
conf.ssl_ca_cert = cert
api_client = ApiClient(conf)
self.corev1 = CoreV1Api(api_client)
self.custom = CustomObjectsApi(api_client)
else:
load_incluster_config()
self.corev1 = CoreV1Api()
self.custom = CustomObjectsApi()
# Read in the manifest.
if alg.manifest:
self.manifest = yaml.safe_load(alg.manifest.open(mode='r'))
# Save off the algorithm.
self.alg = alg
def start_algorithm(self, media_ids, sections, gid, uid, token, project, user,
extra_params: list=[]):
""" Starts an algorithm job, substituting in parameters in the
workflow spec.
"""
# Make a copy of the manifest from the database.
manifest = copy.deepcopy(self.manifest)
# Update the storage class of the spec if executing locally.
if self.alg.cluster is None:
if 'volumeClaimTemplates' in manifest['spec']:
for claim in manifest['spec']['volumeClaimTemplates']:
claim['spec']['storageClassName'] = _select_storage_class()
logger.warning(f"Implicitly sc to pvc of Algo:{self.alg.pk}")
# Add in workflow parameters.
manifest['spec']['arguments'] = {'parameters': [
{
'name': 'name',
'value': self.alg.name,
}, {
'name': 'media_ids',
'value': media_ids,
}, {
'name': 'sections',
'value': sections,
}, {
'name': 'gid',
'value': gid,
}, {
'name': 'uid',
'value': uid,
}, {
'name': 'host',
'value': f'{PROTO}{os.getenv("MAIN_HOST")}',
}, {
'name': 'rest_url',
'value': f'{PROTO}{os.getenv("MAIN_HOST")}/rest',
}, {
'name': 'rest_token',
'value': str(token),
}, {
'name': 'tus_url',
'value': f'{PROTO}{os.getenv("MAIN_HOST")}/files/',
}, {
'name': 'project_id',
'value': str(project),
},
]}
# Add the non-standard extra parameters if provided
# Expected format of extra_params: list of dictionaries with 'name' and 'value' entries
# for each of the parameters. e.g. {{'name': 'hello_param', 'value': [1]}}
manifest['spec']['arguments']['parameters'].extend(extra_params)
# Set labels and annotations for job management
if 'labels' not in manifest['metadata']:
manifest['metadata']['labels'] = {}
if 'annotations' not in manifest['metadata']:
manifest['metadata']['annotations'] = {}
manifest['metadata']['labels'] = {
**manifest['metadata']['labels'],
'job_type': 'algorithm',
'project': str(project),
'gid': gid,
'uid': uid,
'user': str(user),
}
manifest['metadata']['annotations'] = {
**manifest['metadata']['annotations'],
'name': self.alg.name,
'sections': sections,
'media_ids': media_ids,
}
for num_retries in range(MAX_SUBMIT_RETRIES):
try:
response = self.custom.create_namespaced_custom_object(
group='argoproj.io',
version='v1alpha1',
namespace='default',
plural='workflows',
body=manifest,
)
break
except ApiException:
logger.info(f"Failed to submit workflow:")
logger.info(f"{manifest}")
time.sleep(SUBMIT_RETRY_BACKOFF)
if num_retries == (MAX_SUBMIT_RETRIES - 1):
raise Exception(f"Failed to submit workflow {MAX_SUBMIT_RETRIES} times!")
# Cache the job for cancellation/authentication.
TatorCache().set_job({'uid': uid,
'gid': gid,
'user': user,
'project': project,
'algorithm': self.alg.pk,
'datetime': datetime.datetime.utcnow().isoformat() + 'Z'})
return response
class TatorTranscode(JobManagerMixin):
""" Interface to kubernetes REST API for starting transcodes.
"""
def __init__(self):
""" Intializes the connection. If environment variables for
remote transcode are defined, connect to that cluster.
"""
host = os.getenv('REMOTE_TRANSCODE_HOST')
port = os.getenv('REMOTE_TRANSCODE_PORT')
token = os.getenv('REMOTE_TRANSCODE_TOKEN')
cert = os.getenv('REMOTE_TRANSCODE_CERT')
self.remote = host is not None
if self.remote:
conf = Configuration()
conf.api_key['authorization'] = token
conf.host = f'https://{host}:{port}'
conf.verify_ssl = True
conf.ssl_ca_cert = cert
api_client = ApiClient(conf)
self.corev1 = CoreV1Api(api_client)
self.custom = CustomObjectsApi(api_client)
else:
load_incluster_config()
self.corev1 = CoreV1Api()
self.custom = CustomObjectsApi()
self.setup_common_steps()
def setup_common_steps(self):
""" Sets up the basic steps for a transcode pipeline.
"""
def spell_out_params(params):
yaml_params = [{"name": x} for x in params]
return yaml_params
# Define each task in the pipeline.
# Deletes the remote TUS file
self.delete_task = {
'name': 'delete',
'metadata': {
'labels': {'app': 'transcoder'},
},
'inputs': {'parameters' : spell_out_params(['url'])},
'nodeSelector' : {'cpuWorker' : 'yes'},
'container': {
'image': '{{workflow.parameters.client_image}}',
'imagePullPolicy': 'IfNotPresent',
'command': ['curl',],
'args': ['-X', 'DELETE', '{{inputs.parameters.url}}'],
'resources': {
'limits': {
'memory': '1Gi',
'cpu': '250m',
},
},
},
}
# Unpacks a tarball and sets up the work products for follow up
# dags or steps
unpack_params = [{'name': f'videos-{x}',
'valueFrom': {'path': f'/work/videos_{x}.json'}} for x in range(NUM_WORK_PACKETS)]
# TODO: Don't make work packets for localizations / states
unpack_params.extend([{'name': f'localizations-{x}',
'valueFrom': {'path': f'/work/localizations_{x}.json'}} for x in range(NUM_WORK_PACKETS)])
unpack_params.extend([{'name': f'states-{x}',
'valueFrom': {'path': f'/work/states_{x}.json'}} for x in range(NUM_WORK_PACKETS)])
self.unpack_task = {
'name': 'unpack',
'metadata': {
'labels': {'app': 'transcoder'},
},
'inputs': {'parameters' : spell_out_params(['original'])},
'outputs': {'parameters' : unpack_params},
'nodeSelector' : {'cpuWorker' : 'yes'},
'container': {
'image': '{{workflow.parameters.client_image}}',
'imagePullPolicy': 'IfNotPresent',
'command': ['bash',],
'args': ['unpack.sh', '{{inputs.parameters.original}}', '/work'],
'volumeMounts': [{
'name': 'transcode-scratch',
'mountPath': '/work',
}],
'resources': {
'limits': {
'memory': '4Gi',
'cpu': '1000m',
},
},
},
}
self.data_import = {
'name': 'data-import',
'inputs': {'parameters' : spell_out_params(['md5', 'file', 'mode'])},
'nodeSelector' : {'cpuWorker' : 'yes'},
'container': {
'image': '{{workflow.parameters.client_image}}',
'imagePullPolicy': 'IfNotPresent',
'command': ['python3',],
'args': ['importDataFromCsv.py',
'--host', '{{workflow.parameters.host}}',
'--token', '{{workflow.parameters.token}}',
'--project', '{{workflow.parameters.project}}',
'--mode', '{{inputs.parameters.mode}}',
'--media-md5', '{{inputs.parameters.md5}}',
'{{inputs.parameters.file}}'],
'volumeMounts': [{
'name': 'transcode-scratch',
'mountPath': '/work',
}],
'resources': {
'limits': {
'memory': '4Gi',
'cpu': '1000m',
},
},
},
}
self.prepare_task = {
'name': 'prepare',
'metadata': {
'labels': {'app': 'transcoder'},
},
'retryStrategy': {
'retryPolicy': 'Always',
'limit': 3,
'backoff': {
'duration': '5s',
'factor': 2
},
},
'nodeSelector' : {'cpuWorker' : 'yes'},
'container': {
'image': '{{workflow.parameters.client_image}}',
'imagePullPolicy': 'IfNotPresent',
'command': ['python3',],
'args': ['-m', 'tator.transcode.prepare',
'--url', '{{workflow.parameters.url}}',
'--work_dir', '/work',
'--host', '{{workflow.parameters.host}}',
'--token', '{{workflow.parameters.token}}',
'--project', '{{workflow.parameters.project}}',
'--type', '{{workflow.parameters.type}}',
'--name', '{{workflow.parameters.upload_name}}',
'--section', '{{workflow.parameters.section}}',
'--gid', '{{workflow.parameters.gid}}',
'--uid', '{{workflow.parameters.uid}}',
'--attributes', '{{workflow.parameters.attributes}}',
'--media_id', '{{workflow.parameters.media_id}}',
],
'workingDir': '/scripts',
'volumeMounts': [{
'name': 'scratch-prepare',
'mountPath': '/work',
}],
'resources': {
'limits': {
'memory': os.getenv('TRANSCODER_MEMORY_LIMIT'),
'cpu': os.getenv('TRANSCODER_CPU_LIMIT'),
},
},
},
'outputs': {
'parameters': [{
'name': 'workloads',
'valueFrom': {'path': '/work/workloads.json'},
}, {
'name': 'media_id',
'valueFrom': {'path': '/work/media_id.txt'},
}],
},
}
self.transcode_task = {
'name': 'transcode',
'metadata': {
'labels': {'app': 'transcoder'},
},
'retryStrategy': {
'retryPolicy': 'Always',
'limit': 3,
'backoff': {
'duration': '5s',
'factor': 2
},
},
'nodeSelector' : {'cpuWorker' : 'yes'},
'inputs': {'parameters' : spell_out_params(['original', 'transcoded', 'media',
'category', 'raw_width', 'raw_height',
'configs', 'id'])},
'container': {
'image': '{{workflow.parameters.client_image}}',
'imagePullPolicy': 'IfNotPresent',
'command': ['python3',],
'args': ['-m', 'tator.transcode.transcode',
'--url', '{{workflow.parameters.url}}',
'--work_dir', '/work',
'--host', '{{workflow.parameters.host}}',
'--token', '{{workflow.parameters.token}}',
'--media', '{{inputs.parameters.media}}',
'--category', '{{inputs.parameters.category}}',
'--raw_width', '{{inputs.parameters.raw_width}}',
'--raw_height', '{{inputs.parameters.raw_height}}',
'--configs', '{{inputs.parameters.configs}}'],
'workingDir': '/scripts',
'volumeMounts': [{
'name': 'scratch-{{inputs.parameters.id}}',
'mountPath': '/work',
}],
'resources': {
'limits': {
'memory': os.getenv('TRANSCODER_MEMORY_LIMIT'),
'cpu': os.getenv('TRANSCODER_CPU_LIMIT'),
},
},
},
}
self.image_upload_task = {
'name': 'image-upload',
'metadata': {
'labels': {'app': 'transcoder'},
},
'retryStrategy': {
'retryPolicy': 'Always',
'limit': 3,
'backoff': {
'duration': '5s',
'factor': 2
},
},
'nodeSelector' : {'cpuWorker' : 'yes'},
'container': {
'image': '{{workflow.parameters.client_image}}',
'imagePullPolicy': 'IfNotPresent',
'command': ['python3',],
'args': [
'imageLoop.py',
'--host', '{{workflow.parameters.host}}',
'--token', '{{workflow.parameters.token}}',
'--project', '{{workflow.parameters.project}}',
'--gid', '{{workflow.parameters.gid}}',
'--uid', '{{workflow.parameters.uid}}',
# TODO: If we made section a DAG argument, we could
# conceviably import a tar across multiple sections
'--section', '{{workflow.parameters.section}}',
'--progressName', '{{workflow.parameters.upload_name}}',
],
'workingDir': '/scripts',
'volumeMounts': [{
'name': 'transcode-scratch',
'mountPath': '/work',
}],
'resources': {
'limits': {
'memory': '1Gi',
'cpu': '250m',
},
},
},
}
def get_download_task(self, headers=[]):
# Download task exports the human readable filename a
# workflow global to support the onExit handler
return {
'name': 'download',
'metadata': {
'labels': {'app': 'transcoder'},
},
'retryStrategy': {
'retryPolicy': 'Always',
'limit': 3,
'backoff': {
'duration': '5s',
'factor': 2
},
},
'inputs': {'parameters' : [{'name': 'original'},
{'name': 'url'}]},
'nodeSelector' : {'cpuWorker' : 'yes'},
'container': {
'image': '{{workflow.parameters.client_image}}',
'imagePullPolicy': 'IfNotPresent',
'command': ['wget',],
'args': ['-O', '{{inputs.parameters.original}}'] + headers + \
['{{inputs.parameters.url}}'],
'volumeMounts': [{
'name': 'transcode-scratch',
'mountPath': '/work',
}],
'resources': {
'limits': {
'memory': '4Gi',
'cpu': '1000m',
},
},
},
}
def get_unpack_and_transcode_tasks(self, paths, url):
""" Generate a task object describing the dependencies of a transcode from tar"""
# Generate an args structure for the DAG
args = [{'name': 'url', 'value': url}]
for key in paths:
args.append({'name': key, 'value': paths[key]})
parameters = {"parameters" : args}
def make_item_arg(name):
return {'name': name,
'value': f'{{{{item.{name}}}}}'}
instance_args = ['entity_type',
'name',
'md5']
item_parameters = {"parameters" : [make_item_arg(x) for x in instance_args]}
# unpack work list
item_parameters["parameters"].append({"name": "url",
"value": "None"})
item_parameters["parameters"].append({"name": "original",
"value": "{{item.dirname}}/{{item.name}}"})
item_parameters["parameters"].append({"name": "transcoded",
"value": "{{item.dirname}}/{{item.base}}_transcoded"})
item_parameters["parameters"].append({"name": "thumbnail",
"value": "{{item.dirname}}/{{item.base}}_thumbnail.jpg"})
item_parameters["parameters"].append({"name": "thumbnail_gif",
"value": "{{item.dirname}}/{{item.base}}_thumbnail_gif.gif"})
item_parameters["parameters"].append({"name": "segments",
"value": "{{item.dirname}}/{{item.base}}_segments.json"})
state_import_parameters = {"parameters" : [make_item_arg(x) for x in ["md5", "file"]]}
localization_import_parameters = {"parameters" : [make_item_arg(x) for x in ["md5", "file"]]}
state_import_parameters["parameters"].append({"name": "mode", "value": "state"})
localization_import_parameters["parameters"].append({"name": "mode", "value": "localizations"})
unpack_task = {
'name': 'unpack-pipeline',
'metadata': {
'labels': {'app': 'transcoder'},
},
'dag': {
# First download, unpack and delete archive. Then Iterate over each video and upload
# Lastly iterate over all localization and state files.
'tasks' : [{'name': 'download-task',
'template': 'download',
'arguments': parameters},
{'name': 'unpack-task',
'template': 'unpack',
'arguments': parameters,
'dependencies' : ['download-task']},
{'name': 'delete-task',
'template': 'delete',
'arguments': parameters,
'dependencies' : ['unpack-task']}
]
}
} # end of dag
unpack_task['dag']['tasks'].extend([{'name': f'transcode-task-{x}',
'template': 'transcode-pipeline',
'arguments' : item_parameters,
'withParam' : f'{{{{tasks.unpack-task.outputs.parameters.videos-{x}}}}}',
'dependencies' : ['unpack-task']} for x in range(NUM_WORK_PACKETS)])
unpack_task['dag']['tasks'].append({'name': f'image-upload-task',
'template': 'image-upload',
'dependencies' : ['unpack-task']})
deps = [f'transcode-task-{x}' for x in range(NUM_WORK_PACKETS)]
deps.append('image-upload-task')
unpack_task['dag']['tasks'].extend([{'name': f'state-import-task-{x}',
'template': 'data-import',
'arguments' : state_import_parameters,
'dependencies' : deps,
'withParam': f'{{{{tasks.unpack-task.outputs.parameters.states-{x}}}}}'} for x in range(NUM_WORK_PACKETS)])
unpack_task['dag']['tasks'].extend([{'name': f'localization-import-task-{x}',
'template': 'data-import',
'arguments' : localization_import_parameters,
'dependencies' : deps,
'withParam': f'{{{{tasks.unpack-task.outputs.parameters.localizations-{x}}}}}'} for x in range(NUM_WORK_PACKETS)])
return unpack_task
def get_transcode_dag(self, media_id=None):
""" Return the DAG that describes transcoding a single media file """
def make_passthrough_arg(name):
return {'name': name,
'value': f'{{{{inputs.parameters.{name}}}}}'}
instance_args = ['url',
'original',
'transcoded',
'thumbnail',
'thumbnail_gif',
'segments',
'entity_type',
'name',
'md5']
passthrough_parameters = {"parameters" : [make_passthrough_arg(x) for x in instance_args]}
pipeline_task = {
'name': 'transcode-pipeline',
'metadata': {
'labels': {'app': 'transcoder'},
},
'inputs': passthrough_parameters,
'dag': {
'tasks': [{
'name': 'prepare-task',
'template': 'prepare',
}, {
'name': 'transcode-task',
'template': 'transcode',
'arguments': {
'parameters': passthrough_parameters['parameters'] + [{
'name': 'category',
'value': '{{item.category}}',
}, {
'name': 'raw_width',
'value': '{{item.raw_width}}',
}, {
'name': 'raw_height',
'value': '{{item.raw_height}}',
}, {
'name': 'configs',
'value': '{{item.configs}}',
}, {
'name': 'id',
'value': '{{item.id}}',
}, {
'name': 'media',
'value': '{{tasks.prepare-task.outputs.parameters.media_id}}' \
if media_id is None else str(media_id),
}],
},
'dependencies': ['prepare-task'],
'withParam': '{{tasks.prepare-task.outputs.parameters.workloads}}',
}],
},
}
return pipeline_task
def get_transcode_task(self, item, url):
""" Generate a task object describing the dependencies of a transcode """
# Generate an args structure for the DAG
args = [{'name': 'url', 'value': url}]
for key in item:
args.append({'name': key, 'value': item[key]})
parameters = {"parameters" : args}
pipeline = {
'name': 'single-file-pipeline',
'dag': {
# First download, unpack and delete archive. Then Iterate over each video and upload
# Lastly iterate over all localization and state files.
'tasks' : [{'name': 'transcode-task',
'template': 'transcode-pipeline',
'arguments' : parameters}]
}
}
return pipeline
def start_tar_import(self,
project,
entity_type,
token,
url,
name,
section,
md5,
gid,
uid,
user,
upload_size,
attributes):
""" Initiate a transcode based on the contents on an archive """
comps = name.split('.')
base = comps[0]
ext = '.'.join(comps[1:])
if entity_type != -1:
raise Exception("entity type is not -1!")
pvc_size = os.getenv('TRANSCODER_PVC_SIZE')
if upload_size:
pvc_size = bytes_to_mi_str(upload_size * 4)
args = {'original': '/work/' + name,
'name': name}
docker_registry = os.getenv('SYSTEM_IMAGES_REGISTRY')
host = f'{PROTO}{os.getenv("MAIN_HOST")}'
global_args = {'upload_name': name,
'url': url,
'host': host,
'rest_url': f'{host}/rest',
'tus_url' : f'{host}/files/',
'project' : str(project),
'type': '-1',
'token' : str(token),
'section' : section,
'gid': gid,
'uid': uid,
'user': str(user),
'client_image' : get_client_image_name(),
'attributes' : json.dumps(attributes),
'media_id': '-1'}
global_parameters=[{"name": x, "value": global_args[x]} for x in global_args]
pipeline_task = self.get_unpack_and_transcode_tasks(args, url)
# Define the workflow spec.
manifest = {
'apiVersion': 'argoproj.io/v1alpha1',
'kind': 'Workflow',
'metadata': {
'generateName': 'transcode-workflow-',
'labels': {
'job_type': 'upload',
'project': str(project),
'gid': gid,
'uid': uid,
'user': str(user),
},
'annotations': {
'name': name,
'section': section,
},
},
'spec': {
'entrypoint': 'unpack-pipeline',
'podGC': {'strategy': 'OnPodCompletion'},
'arguments': {'parameters' : global_parameters},
'ttlStrategy': {'secondsAfterSuccess': 300,
'secondsAfterFailure': 86400},
'volumeClaimTemplates': [{
'metadata': {
'name': 'transcode-scratch',
},
'spec': {
'storageClassName': _select_storage_class(),
'accessModes': [ 'ReadWriteOnce' ],
'resources': {
'requests': {
'storage': pvc_size,
}
}
}
}],
'parallelism': 4,
'templates': [
self.prepare_task,
self.get_download_task(),
self.delete_task,
self.transcode_task,
self.image_upload_task,
self.unpack_task,
self.get_transcode_dag(),
pipeline_task,
self.data_import
],
},
}
# Create the workflow
for num_retries in range(MAX_SUBMIT_RETRIES):
try:
response = self.custom.create_namespaced_custom_object(
group='argoproj.io',
version='v1alpha1',
namespace='default',
plural='workflows',
body=manifest,
)
break
except ApiException:
logger.info(f"Failed to submit workflow:")
logger.info(f"{manifest}")
time.sleep(SUBMIT_RETRY_BACKOFF)
if num_retries == (MAX_SUBMIT_RETRIES - 1):
raise Exception(f"Failed to submit workflow {MAX_SUBMIT_RETRIES} times!")
def start_transcode(self, project,
entity_type, token, url, name,
section, md5, gid, uid,
user, upload_size,
attributes, media_id):
MAX_WORKLOADS = 7 # 5 resolutions + audio + archival
""" Creates an argo workflow for performing a transcode.
"""
# Define paths for transcode outputs.
base, _ = os.path.splitext(name)
args = {
'original': '/work/' + name,
'transcoded': '/work/' + base + '_transcoded',
'thumbnail': '/work/' + base + '_thumbnail.jpg',
'thumbnail_gif': '/work/' + base + '_thumbnail_gif.gif',
'segments': '/work/' + base + '_segments.json',
'entity_type': str(entity_type),
'md5' : md5,
'name': name
}
pvc_size = os.getenv('TRANSCODER_PVC_SIZE')
if upload_size:
pvc_size = bytes_to_mi_str(upload_size * 4)
docker_registry = os.getenv('SYSTEM_IMAGES_REGISTRY')
host = f'{PROTO}{os.getenv("MAIN_HOST")}'
global_args = {'upload_name': name,
'url': url,
'host': host,
'rest_url': f'{host}/rest',
'tus_url' : f'{host}/files/',
'token' : str(token),
'project' : str(project),
'type': str(entity_type),
'section' : section,
'gid': gid,
'uid': uid,
'user': str(user),
'client_image' : get_client_image_name(),
'attributes' : json.dumps(attributes),
'media_id': '-1' if media_id is None else str(media_id)}
global_parameters=[{"name": x, "value": global_args[x]} for x in global_args]
pipeline_task = self.get_transcode_task(args, url)
# Define the workflow spec.
manifest = {
'apiVersion': 'argoproj.io/v1alpha1',
'kind': 'Workflow',
'metadata': {
'generateName': 'transcode-workflow-',
'labels': {
'job_type': 'upload',
'project': str(project),
'gid': gid,
'uid': uid,
'user': str(user),
},
'annotations': {
'name': name,
'section': section,
},
},
'spec': {
'entrypoint': 'single-file-pipeline',
'podGC': {'strategy': 'OnPodCompletion'},
'arguments': {'parameters' : global_parameters},
'ttlStrategy': {'secondsAfterSuccess': 300,
'secondsAfterFailure': 86400},
'volumeClaimTemplates': [{
'metadata': {
'name': f'scratch-{workload}',
},
'spec': {
'storageClassName': os.getenv('SCRATCH_STORAGE_CLASS'),
'accessModes': [ 'ReadWriteOnce' ],
'resources': {
'requests': {
'storage': pvc_size,
}
}
}
} for workload in ['prepare'] + list(range(MAX_WORKLOADS))],
'templates': [
self.prepare_task,
self.transcode_task,
self.image_upload_task,
self.get_transcode_dag(media_id),
pipeline_task,
],
},
}
# Create the workflow
response = self.custom.create_namespaced_custom_object(
group='argoproj.io',
version='v1alpha1',
namespace='default',
plural='workflows',
body=manifest,
)
# Cache the job for cancellation/authentication.
TatorCache().set_job({'uid': uid,
'gid': gid,
'user': user,
'project': project,
'algorithm': -1,
'datetime': datetime.datetime.utcnow().isoformat() + 'Z'})
def test_legacy_template_migrate_and_delete(admin_mc, admin_cc,
remove_resource, user_mc,
raw_remove_custom_resource):
"""Asserts that any node template not in cattle-global-nt namespace is
duplicated into cattle-global-nt, then deleted. Also, asserts that
operations on legacy node templates are forwarded to corresponding
migrated node templates"""
admin_client = admin_mc.client
admin_cc_client = admin_cc.client
user_client = user_mc.client
k8s_dynamic_client = CustomObjectsApi(admin_mc.k8s_client)
ns = admin_cc_client.create_namespace(name="ns-" + random_str(),
clusterId=admin_cc.cluster.id)
remove_resource(ns)
node_template_name = "nt-" + random_str()
body = {
"metadata": {
"name": node_template_name,
"annotations": {
"field.cattle.io/creatorId": user_mc.user.id
}
},
"kind": "NodeTemplate",
"apiVersion": "management.cattle.io/v3",
"azureConfig": {
"customData": "asdfsadfsd"
}
}
# create a node template that will be recognized as legacy
dynamic_nt = k8s_dynamic_client.create_namespaced_custom_object(
"management.cattle.io", "v3", ns.name, 'nodetemplates', body)
raw_remove_custom_resource(dynamic_nt)
def migrated_template_exists(id):
try:
nt = user_client.by_id_node_template(id=id)
remove_resource(nt)
return nt
except ApiError as e:
assert e.error.status == 403
return False
id = "cattle-global-nt:nt-" + ns.id + "-" + dynamic_nt["metadata"]["name"]
legacy_id = dynamic_nt["metadata"]["name"]
legacy_ns = dynamic_nt["metadata"]["namespace"]
full_legacy_id = legacy_ns + ":" + legacy_id
# wait for node template to be migrated
nt = wait_for(
lambda: migrated_template_exists(id),
fail_handler=lambda: "failed waiting for node template to migrate")
# assert that config has not been removed from node template
assert nt.azureConfig["customData"] ==\
dynamic_nt["azureConfig"]["customData"]
def legacy_template_deleted():
try:
k8s_dynamic_client.get_namespaced_custom_object(
"management.cattle.io", "v3", ns.name, 'nodetemplates',
legacy_id)
return False
except ApiException as e:
return e.status == 404
# wait for legacy node template to be deleted
wait_for(
lambda: legacy_template_deleted(),
fail_handler=lambda: "failed waiting for old node template to delete")
# retrieve node template via legacy id
nt = admin_client.by_id_node_template(id=full_legacy_id)
# retrieve node template via migrated id
migrated_nt = admin_client.by_id_node_template(id=id)
def compare(d1, d2):
if d1 == d2:
return True
if d1.keys() != d2.keys():
return False
for key in d1.keys():
if key in ["id", "namespace", "links", "annotations"]:
continue
if d1[key] == d2[key]:
continue
if callable(d1[key]):
continue
if isinstance(d1[key], RestObject):
if compare(d1[key], d1[key]):
continue
return False
return True
# ensure templates returned are identical aside from fields containing
# id/ns
if not compare(nt, migrated_nt):
raise Exception("forwarded does not match migrated nodetemplate")
nt.azureConfig.customData = "asdfasdf"
new_config = nt.azureConfig
new_config.customData = "adsfasdfadsf"
# update node template via legacy id
nt = admin_client.update_by_id_node_template(id=full_legacy_id,
azureConfig=new_config)
# assert node template is being updated
assert nt.azureConfig.customData == new_config.customData
nt2 = admin_client.by_id_node_template(id=id)
# assert node template being updated is migrated node template
assert nt2.azureConfig.customData == new_config.customData
# delete node template via legacy id
admin_client.delete(nt)
wait_for(lambda: admin_client.by_id_node_template(id) is None,
fail_handler=lambda:
"failed waiting for migrate node template to delete")
class InfraEnv(BaseCustomResource):
"""
InfraEnv is used to generate cluster iso.
Image is automatically generated on CRD deployment, after InfraEnv is
reconciled. Image download url will be exposed in the status.
"""
_plural = "infraenvs"
def __init__(
self,
kube_api_client: ApiClient,
name: str,
namespace: str = env_variables["namespace"],
):
super().__init__(name, namespace)
self.crd_api = CustomObjectsApi(kube_api_client)
def create_from_yaml(self, yaml_data: dict) -> None:
self.crd_api.create_namespaced_custom_object(
group=CRD_API_GROUP,
version=CRD_API_VERSION,
plural=self._plural,
body=yaml_data,
namespace=self.ref.namespace,
)
logger.info("created infraEnv %s: %s", self.ref, pformat(yaml_data))
def create(
self,
cluster_deployment: ClusterDeployment,
secret: Secret,
proxy: Optional[Proxy] = None,
label_selector: Optional[Dict[str, str]] = None,
ignition_config_override: Optional[str] = None,
nmstate_label: Optional[str] = None,
ssh_pub_key: Optional[str] = None,
**kwargs,
) -> None:
body = {
"apiVersion": f"{CRD_API_GROUP}/{CRD_API_VERSION}",
"kind": "InfraEnv",
"metadata": self.ref.as_dict(),
"spec": {
"clusterRef": cluster_deployment.ref.as_dict(),
"pullSecretRef": secret.ref.as_dict(),
"nmStateConfigLabelSelector": {
"matchLabels": {
f"{CRD_API_GROUP}/selector-nmstate-config-name":
nmstate_label or ""
}
},
"agentLabelSelector": {
"matchLabels": label_selector or {}
},
"ignitionConfigOverride": ignition_config_override or "",
},
}
spec = body["spec"]
if proxy:
spec["proxy"] = proxy.as_dict()
if ssh_pub_key:
spec["sshAuthorizedKey"] = ssh_pub_key
spec.update(kwargs)
self.crd_api.create_namespaced_custom_object(
group=CRD_API_GROUP,
version=CRD_API_VERSION,
plural=self._plural,
body=body,
namespace=self.ref.namespace,
)
logger.info("created infraEnv %s: %s", self.ref, pformat(body))
def patch(
self,
cluster_deployment: Optional[ClusterDeployment],
secret: Optional[Secret],
proxy: Optional[Proxy] = None,
label_selector: Optional[Dict[str, str]] = None,
ignition_config_override: Optional[str] = None,
nmstate_label: Optional[str] = None,
ssh_pub_key: Optional[str] = None,
**kwargs,
) -> None:
body = {"spec": kwargs}
spec = body["spec"]
if cluster_deployment:
spec["clusterRef"] = cluster_deployment.ref.as_dict()
if secret:
spec["pullSecretRef"] = secret.ref.as_dict()
if proxy:
spec["proxy"] = proxy.as_dict()
if label_selector:
spec["agentLabelSelector"] = {"matchLabels": label_selector}
if nmstate_label:
spec["nmStateConfigLabelSelector"] = {
"matchLabels": {
f"{CRD_API_GROUP}/selector-nmstate-config-name":
nmstate_label,
}
}
if ignition_config_override:
spec["ignitionConfigOverride"] = ignition_config_override
if ssh_pub_key:
spec["sshAuthorizedKey"] = ssh_pub_key
self.crd_api.patch_namespaced_custom_object(
group=CRD_API_GROUP,
version=CRD_API_VERSION,
plural=self._plural,
name=self.ref.name,
namespace=self.ref.namespace,
body=body,
)
logger.info("patching infraEnv %s: %s", self.ref, pformat(body))
def get(self) -> dict:
return self.crd_api.get_namespaced_custom_object(
group=CRD_API_GROUP,
version=CRD_API_VERSION,
plural=self._plural,
name=self.ref.name,
namespace=self.ref.namespace,
)
def delete(self) -> None:
self.crd_api.delete_namespaced_custom_object(
group=CRD_API_GROUP,
version=CRD_API_VERSION,
plural=self._plural,
name=self.ref.name,
namespace=self.ref.namespace,
)
logger.info("deleted infraEnv %s", self.ref)
def status(
self,
timeout: Union[int, float] = DEFAULT_WAIT_FOR_CRD_STATUS_TIMEOUT,
) -> dict:
"""
Status is a section in the CRD that is created after registration to
assisted service and it defines the observed state of InfraEnv.
Since the status key is created only after resource is processed by the
controller in the service, it might take a few seconds before appears.
"""
def _attempt_to_get_status() -> dict:
return self.get()["status"]
return waiting.wait(
_attempt_to_get_status,
sleep_seconds=0.5,
timeout_seconds=timeout,
waiting_for=f"infraEnv {self.ref} status",
expected_exceptions=KeyError,
)
def get_iso_download_url(
self,
timeout: Union[int, float] = DEFAULT_WAIT_FOR_ISO_URL_TIMEOUT,
):
def _attempt_to_get_image_url() -> str:
return self.get()["status"]["isoDownloadURL"]
return waiting.wait(
_attempt_to_get_image_url,
sleep_seconds=3,
timeout_seconds=timeout,
waiting_for="image to be created",
expected_exceptions=KeyError,
)
def get_cluster_id(self):
iso_download_url = self.get_iso_download_url()
return ISO_URL_PATTERN.match(iso_download_url).group("cluster_id")
@classmethod
def deploy_default_infraenv(
cls,
kube_api_client: ApiClient,
name: str,
ignore_conflict: bool = True,
cluster_deployment: Optional[ClusterDeployment] = None,
secret: Optional[Secret] = None,
proxy: Optional[Proxy] = None,
label_selector: Optional[Dict[str, str]] = None,
ignition_config_override: Optional[str] = None,
**kwargs,
) -> "InfraEnv":
infra_env = InfraEnv(kube_api_client, name)
try:
if "filepath" in kwargs:
infra_env._create_infraenv_from_yaml_file(
filepath=kwargs["filepath"], )
else:
infra_env._create_infraenv_from_attrs(
kube_api_client=kube_api_client,
name=name,
ignore_conflict=ignore_conflict,
cluster_deployment=cluster_deployment,
secret=secret,
proxy=proxy,
label_selector=label_selector,
ignition_config_override=ignition_config_override,
**kwargs,
)
except ApiException as e:
if not (e.reason == "Conflict" and ignore_conflict):
raise
# wait until install-env will have status (i.e until resource will be
# processed in assisted-service).
infra_env.status()
return infra_env
def _create_infraenv_from_yaml_file(
self,
filepath: str,
) -> None:
with open(filepath) as fp:
yaml_data = yaml.safe_load(fp)
self.create_from_yaml(yaml_data)
def _create_infraenv_from_attrs(
self,
kube_api_client: ApiClient,
cluster_deployment: ClusterDeployment,
secret: Optional[Secret] = None,
proxy: Optional[Proxy] = None,
label_selector: Optional[Dict[str, str]] = None,
ignition_config_override: Optional[str] = None,
**kwargs,
) -> None:
if not secret:
secret = deploy_default_secret(
kube_api_client=kube_api_client,
name=cluster_deployment.ref.name,
)
self.create(
cluster_deployment=cluster_deployment,
secret=secret,
proxy=proxy,
label_selector=label_selector,
ignition_config_override=ignition_config_override,
**kwargs,
)
class TraefikMiddlewareReconciler(KubernetesObjectReconciler[TraefikMiddleware]
):
"""Kubernetes Traefik Middleware Reconciler"""
def __init__(self, controller: "KubernetesController") -> None:
super().__init__(controller)
self.api_ex = ApiextensionsV1Api(controller.client)
self.api = CustomObjectsApi(controller.client)
def _crd_exists(self) -> bool:
"""Check if the traefik middleware exists"""
return bool(
len(
self.api_ex.list_custom_resource_definition(
field_selector=f"metadata.name={CRD_NAME}").items))
def reconcile(self, current: TraefikMiddleware,
reference: TraefikMiddleware):
super().reconcile(current, reference)
if current.spec.forwardAuth.address != reference.spec.forwardAuth.address:
raise NeedsUpdate()
def get_reference_object(self) -> TraefikMiddleware:
"""Get deployment object for outpost"""
if not ProxyProvider.objects.filter(
outpost__in=[self.controller.outpost],
forward_auth_mode=True,
).exists():
self.logger.debug("No providers with forward auth enabled.")
raise Disabled()
if not self._crd_exists():
self.logger.debug("CRD doesn't exist")
raise Disabled()
return TraefikMiddleware(
apiVersion=f"{CRD_GROUP}/{CRD_VERSION}",
kind="Middleware",
metadata=TraefikMiddlewareMetadata(
name=self.name,
namespace=self.namespace,
labels=self.get_object_meta().labels,
),
spec=
TraefikMiddlewareSpec(forwardAuth=TraefikMiddlewareSpecForwardAuth(
address=
f"http://{self.name}.{self.namespace}:4180/akprox/auth?traefik",
authResponseHeaders=[
"Set-Cookie",
"X-Auth-Username",
"X-Forwarded-Email",
"X-Forwarded-Preferred-Username",
"X-Forwarded-User",
],
trustForwardHeader=True,
)),
)
def create(self, reference: TraefikMiddleware):
return self.api.create_namespaced_custom_object(
group=CRD_GROUP,
version=CRD_VERSION,
plural=CRD_PLURAL,
namespace=self.namespace,
body=asdict(reference),
field_manager=FIELD_MANAGER,
)
def delete(self, reference: TraefikMiddleware):
return self.api.delete_namespaced_custom_object(
group=CRD_GROUP,
version=CRD_VERSION,
namespace=self.namespace,
plural=CRD_PLURAL,
name=self.name,
)
def retrieve(self) -> TraefikMiddleware:
return from_dict(
TraefikMiddleware,
self.api.get_namespaced_custom_object(
group=CRD_GROUP,
version=CRD_VERSION,
namespace=self.namespace,
plural=CRD_PLURAL,
name=self.name,
),
)
def update(self, current: TraefikMiddleware, reference: TraefikMiddleware):
return self.api.patch_namespaced_custom_object(
group=CRD_GROUP,
version=CRD_VERSION,
namespace=self.namespace,
plural=CRD_PLURAL,
name=self.name,
body=asdict(reference),
field_manager=FIELD_MANAGER,
)
class ClusterDeployment(BaseCustomResource):
"""
A CRD that represents cluster in assisted-service.
On creation the cluster will be registered to the service.
On deletion it will be unregistered from the service.
When has sufficient data installation will start automatically.
"""
_hive_api_group = 'hive.openshift.io'
_plural = 'clusterdeployments'
def __init__(self,
kube_api_client: ApiClient,
name: str,
namespace: str = env_variables['namespace']):
BaseCustomResource.__init__(self, name, namespace)
self.crd_api = CustomObjectsApi(kube_api_client)
self._assigned_secret = None
@property
def secret(self) -> Secret:
return self._assigned_secret
def create_from_yaml(self, yaml_data: dict) -> None:
self.crd_api.create_namespaced_custom_object(
group=self._hive_api_group,
version='v1',
plural=self._plural,
body=yaml_data,
namespace=self.ref.namespace)
secret_ref = yaml_data['spec']['pullSecretRef']
self._assigned_secret = Secret(
kube_api_client=self.crd_api.api_client,
name=secret_ref['name'],
)
logger.info('created cluster deployment %s: %s', self.ref,
pformat(yaml_data))
def create(self,
platform: Platform,
install_strategy: InstallStrategy,
secret: Secret,
base_domain: str = env_variables['base_domain'],
**kwargs) -> None:
body = {
'apiVersion': f'{self._hive_api_group}/v1',
'kind': 'ClusterDeployment',
'metadata': self.ref.as_dict(),
'spec': {
'clusterName': self.ref.name,
'baseDomain': base_domain,
'platform': platform.as_dict(),
'provisioning': {
'installStrategy': install_strategy.as_dict()
},
'pullSecretRef': secret.ref.as_dict(),
}
}
body['spec'].update(kwargs)
self.crd_api.create_namespaced_custom_object(
group=self._hive_api_group,
version='v1',
plural=self._plural,
body=body,
namespace=self.ref.namespace)
self._assigned_secret = secret
logger.info('created cluster deployment %s: %s', self.ref,
pformat(body))
def patch(self,
platform: Optional[Platform] = None,
install_strategy: Optional[InstallStrategy] = None,
secret: Optional[Secret] = None,
**kwargs) -> None:
body = {'spec': kwargs}
spec = body['spec']
if platform:
spec['platform'] = platform.as_dict()
if install_strategy:
spec['provisioning'] = {
'installStrategy': install_strategy.as_dict()
}
if secret:
spec['pullSecretRef'] = secret.ref.as_dict()
self.crd_api.patch_namespaced_custom_object(
group=self._hive_api_group,
version='v1',
plural=self._plural,
name=self.ref.name,
namespace=self.ref.namespace,
body=body)
logger.info('patching cluster deployment %s: %s', self.ref,
pformat(body))
def get(self) -> dict:
return self.crd_api.get_namespaced_custom_object(
group=self._hive_api_group,
version='v1',
plural=self._plural,
name=self.ref.name,
namespace=self.ref.namespace)
def delete(self) -> None:
self.crd_api.delete_namespaced_custom_object(
group=self._hive_api_group,
version='v1',
plural=self._plural,
name=self.ref.name,
namespace=self.ref.namespace)
logger.info('deleted cluster deployment %s', self.ref)
def status(
self,
timeout: Union[int,
float] = DEFAULT_WAIT_FOR_CRD_STATUS_TIMEOUT) -> dict:
"""
Status is a section in the CRD that is created after registration to
assisted service and it defines the observed state of ClusterDeployment.
Since the status key is created only after resource is processed by the
controller in the service, it might take a few seconds before appears.
"""
def _attempt_to_get_status() -> dict:
return self.get()['status']
return waiting.wait(_attempt_to_get_status,
sleep_seconds=0.5,
timeout_seconds=timeout,
waiting_for=f'cluster {self.ref} status',
expected_exceptions=KeyError)
def state(
self,
timeout: Union[int, float] = DEFAULT_WAIT_FOR_CRD_STATE_TIMEOUT
) -> Tuple[str, str]:
state, state_info = None, None
for condition in self.status(timeout).get('conditions', []):
reason = condition.get('reason')
if reason == 'AgentPlatformState':
state = condition.get('message')
elif reason == 'AgentPlatformStateInfo':
state_info = condition.get('message')
if state and state_info:
break
return state, state_info
def wait_for_state(
self,
required_state: str,
timeout: Union[int,
float] = DEFAULT_WAIT_FOR_CRD_STATE_TIMEOUT) -> None:
required_state = required_state.lower()
def _has_required_state() -> bool:
state, _ = self.state(timeout=0.5)
return state.lower() == required_state
waiting.wait(
_has_required_state,
timeout_seconds=timeout,
waiting_for=f'cluster {self.ref} state to be {required_state}',
expected_exceptions=waiting.exceptions.TimeoutExpired)
class TatorTranscode(JobManagerMixin):
""" Interface to kubernetes REST API for starting transcodes.
"""
def __init__(self):
""" Intializes the connection. If environment variables for
remote transcode are defined, connect to that cluster.
"""
host = os.getenv('REMOTE_TRANSCODE_HOST')
port = os.getenv('REMOTE_TRANSCODE_PORT')
token = os.getenv('REMOTE_TRANSCODE_TOKEN')
cert = os.getenv('REMOTE_TRANSCODE_CERT')
if host:
conf = Configuration()
conf.api_key['authorization'] = token
conf.host = f'https://{host}:{port}'
conf.verify_ssl = True
conf.ssl_ca_cert = cert
api_client = ApiClient(conf)
self.corev1 = CoreV1Api(api_client)
self.custom = CustomObjectsApi(api_client)
else:
load_incluster_config()
self.corev1 = CoreV1Api()
self.custom = CustomObjectsApi()
def setup_common_steps(self, project, token, section, gid, uid, user):
""" Sets up the basic steps for a transcode pipeline.
TODO: Would be nice if this was just in a yaml file.
"""
docker_registry = os.getenv('SYSTEM_IMAGES_REGISTRY')
transcoder_image = f"{docker_registry}/tator_transcoder:{Git.sha}"
# Setup common pipeline steps
# Define persistent volume claim.
self.pvc = {
'metadata': {
'name': 'transcode-scratch',
},
'spec': {
'storageClassName': 'nfs-client',
'accessModes': ['ReadWriteOnce'],
'resources': {
'requests': {
'storage': '10Gi',
}
}
}
}
def spell_out_params(params):
yaml_params = [{"name": x} for x in params]
return yaml_params
# Define each task in the pipeline.
# Download task exports the human readable filename a
# workflow global to support the onExit handler
self.download_task = {
'name': 'download',
'retryStrategy': {
'limit': 3,
'retryOn': "Always",
'backoff': {
'duration': 1,
'factor': 2,
'maxDuration': "1m",
},
},
'inputs': {
'parameters': spell_out_params(['original', 'url', 'name'])
},
'outputs': {
'parameters': [{
'name': 'name',
'value': '{{inputs.parameters.name}}',
'globalName': 'upload_name'
}]
},
'container': {
'image':
transcoder_image,
'imagePullPolicy':
'IfNotPresent',
'command': [
'curl',
],
'args': [
'-o', '{{inputs.parameters.original}}',
'{{inputs.parameters.url}}'
],
'volumeMounts': [{
'name': 'transcode-scratch',
'mountPath': '/work',
}],
'resources': {
'limits': {
'memory': '128Mi',
'cpu': '500m',
},
},
},
}
# Deletes the remote TUS file
self.delete_task = {
'name': 'delete',
'inputs': {
'parameters': spell_out_params(['url'])
},
'container': {
'image': transcoder_image,
'imagePullPolicy': 'IfNotPresent',
'command': [
'curl',
],
'args': ['-X', 'DELETE', '{{inputs.parameters.url}}'],
'resources': {
'limits': {
'memory': '128Mi',
'cpu': '500m',
},
},
},
}
# Unpacks a tarball and sets up the work products for follow up
# dags or steps
self.unpack_task = {
'name': 'unpack',
'inputs': {
'parameters': spell_out_params(['original'])
},
'outputs': {
'parameters': [
{
'name': 'videos',
'valueFrom': {
'path': '/work/videos.json'
}
},
{
'name': 'images',
'valueFrom': {
'path': '/work/images.json'
}
},
{
'name': 'localizations',
'valueFrom': {
'path': '/work/localizations.json'
}
},
{
'name': 'states',
'valueFrom': {
'path': '/work/states.json'
}
},
]
},
'container': {
'image':
transcoder_image,
'imagePullPolicy':
'IfNotPresent',
'command': [
'bash',
],
'args':
['unpack.sh', '{{inputs.parameters.original}}', '/work'],
'volumeMounts': [{
'name': 'transcode-scratch',
'mountPath': '/work',
}],
'resources': {
'limits': {
'memory': '512Mi',
'cpu': '1000m',
},
},
},
}
self.data_import = {
'name': 'data-import',
'inputs': {
'parameters': spell_out_params(['md5', 'file', 'mode'])
},
'container': {
'image':
transcoder_image,
'imagePullPolicy':
'IfNotPresent',
'command': [
'python3',
],
'args': [
'importDataFromCsv.py', '--url',
f'https://{os.getenv("MAIN_HOST")}/rest', '--token',
str(token), '--project',
str(project), '--mode', '{{inputs.parameters.mode}}',
'--media-md5', '{{inputs.parameters.md5}}',
'{{inputs.parameters.file}}'
],
'volumeMounts': [{
'name': 'transcode-scratch',
'mountPath': '/work',
}],
'resources': {
'limits': {
'memory': '512Mi',
'cpu': '1000m',
},
},
},
}
self.transcode_task = {
'name': 'transcode',
'inputs': {
'parameters': spell_out_params(['original', 'transcoded'])
},
'container': {
'image':
transcoder_image,
'imagePullPolicy':
'IfNotPresent',
'command': [
'python3',
],
'args': [
'transcode.py', '--output',
'{{inputs.parameters.transcoded}}',
'{{inputs.parameters.original}}'
],
'workingDir':
'/scripts',
'volumeMounts': [{
'name': 'transcode-scratch',
'mountPath': '/work',
}],
'resources': {
'limits': {
'memory': '2Gi',
'cpu': '4000m',
},
},
},
}
self.thumbnail_task = {
'name': 'thumbnail',
'inputs': {
'parameters':
spell_out_params(['original', 'thumbnail', 'thumbnail_gif'])
},
'container': {
'image':
transcoder_image,
'imagePullPolicy':
'IfNotPresent',
'command': [
'python3',
],
'args': [
'makeThumbnails.py',
'--output',
'{{inputs.parameters.thumbnail}}',
'--gif',
'{{inputs.parameters.thumbnail_gif}}',
'{{inputs.parameters.original}}',
],
'workingDir':
'/scripts',
'volumeMounts': [{
'name': 'transcode-scratch',
'mountPath': '/work',
}],
'resources': {
'limits': {
'memory': '500Mi',
'cpu': '1000m',
},
},
},
}
self.segments_task = {
'name': 'segments',
'inputs': {
'parameters': spell_out_params(['transcoded', 'segments'])
},
'container': {
'image':
transcoder_image,
'imagePullPolicy':
'IfNotPresent',
'command': [
'python3',
],
'args': [
'makeFragmentInfo.py',
'--output',
'{{inputs.parameters.segments}}',
'{{inputs.parameters.transcoded}}',
],
'workingDir':
'/scripts',
'volumeMounts': [{
'name': 'transcode-scratch',
'mountPath': '/work',
}],
'resources': {
'limits': {
'memory': '500Mi',
'cpu': '1000m',
},
},
},
}
self.upload_task = {
'name': 'upload',
'inputs': {
'parameters':
spell_out_params([
'url', 'original', 'transcoded', 'thumbnail',
'thumbnail_gif', 'segments', 'entity_type', 'name', 'md5'
])
},
'container': {
'image':
transcoder_image,
'imagePullPolicy':
'IfNotPresent',
'command': [
'python3',
],
'args': [
'uploadTranscodedVideo.py',
'--original_path',
'{{inputs.parameters.original}}',
'--original_url',
'{{inputs.parameters.url}}',
'--transcoded_path',
'{{inputs.parameters.transcoded}}',
'--thumbnail_path',
'{{inputs.parameters.thumbnail}}',
'--thumbnail_gif_path',
'{{inputs.parameters.thumbnail_gif}}',
'--segments_path',
'{{inputs.parameters.segments}}',
'--tus_url',
f'https://{os.getenv("MAIN_HOST")}/files/',
'--url',
f'https://{os.getenv("MAIN_HOST")}/rest',
'--token',
str(token),
'--project',
str(project),
'--type',
'{{inputs.parameters.entity_type}}',
'--gid',
gid,
'--uid',
uid,
# TODO: If we made section a DAG argument, we could
# conceviably import a tar across multiple sections
'--section',
section,
'--name',
'{{inputs.parameters.name}}',
'--md5',
'{{inputs.parameters.md5}}',
'--progressName',
'{{workflow.outputs.parameters.upload_name}}',
],
'workingDir':
'/scripts',
'volumeMounts': [{
'name': 'transcode-scratch',
'mountPath': '/work',
}],
'resources': {
'limits': {
'memory': '500Mi',
'cpu': '1000m',
},
},
},
}
# Define task to send progress message in case of failure.
self.progress_task = {
'name': 'progress',
'inputs': {
'parameters':
spell_out_params(['state', 'message', 'progress'])
},
'container': {
'image':
get_marshal_image_name(),
'imagePullPolicy':
'IfNotPresent',
'command': [
'python3',
],
'args': [
'sendProgress.py',
'--url',
f'https://{os.getenv("MAIN_HOST")}/rest',
'--token',
str(token),
'--project',
str(project),
'--job_type',
'upload',
'--gid',
gid,
'--uid',
uid,
'--state',
'{{inputs.parameters.state}}',
'--message',
'{{inputs.parameters.message}}',
'--progress',
'{{inputs.parameters.progress}}',
# Pull the name from the upload parameter
'--name',
'{{workflow.outputs.parameters.upload_name}}',
'--section',
section,
],
'workingDir':
'/',
'resources': {
'limits': {
'memory': '32Mi',
'cpu': '100m',
},
},
},
}
# Define a exit handler.
self.exit_handler = {
'name':
'exit-handler',
'steps': [[{
'name': 'send-fail',
'template': 'progress',
'when': '{{workflow.status}} != Succeeded',
'arguments': {
'parameters': [
{
'name': 'state',
'value': 'failed'
},
{
'name': 'message',
'value': 'Media Import Failed'
},
{
'name': 'progress',
'value': '0'
},
]
}
}, {
'name': 'send-success',
'template': 'progress',
'when': '{{workflow.status}} == Succeeded',
'arguments': {
'parameters': [
{
'name': 'state',
'value': 'finished'
},
{
'name': 'message',
'value': 'Media Import Complete'
},
{
'name': 'progress',
'value': '100'
},
]
}
}]],
}
def get_unpack_and_transcode_tasks(self, paths, url):
""" Generate a task object describing the dependencies of a transcode from tar"""
# Generate an args structure for the DAG
args = [{'name': 'url', 'value': url}]
for key in paths:
args.append({'name': key, 'value': paths[key]})
parameters = {"parameters": args}
def make_item_arg(name):
return {'name': name, 'value': f'{{{{item.{name}}}}}'}
def make_passthrough_arg(name):
return {'name': name, 'value': f'{{{{inputs.parameters.{name}}}}}'}
all_args = [
'url', 'original', 'transcoded', 'thumbnail', 'thumbnail_gif',
'segments', 'entity_type', 'name', 'md5'
]
item_parameters = {"parameters": [make_item_arg(x) for x in all_args]}
state_import_parameters = {
"parameters": [make_item_arg(x) for x in ["md5", "file"]]
}
localization_import_parameters = {
"parameters": [make_item_arg(x) for x in ["md5", "file"]]
}
passthrough_parameters = {
"parameters": [make_passthrough_arg(x) for x in all_args]
}
state_import_parameters["parameters"].append({
"name": "mode",
"value": "state"
})
localization_import_parameters["parameters"].append({
"name":
"mode",
"value":
"localizations"
})
logger.info(f"item_params = {item_parameters}")
unpack_task = {
'name': 'unpack-pipeline',
'dag': {
# First download, unpack and delete archive. Then Iterate over each video and upload
# Lastly iterate over all localization and state files.
'tasks': [
{
'name': 'download-task',
'template': 'download',
'arguments': parameters
},
{
'name': 'unpack-task',
'template': 'unpack',
'arguments': parameters,
'dependencies': ['download-task']
},
{
'name': 'delete-task',
'template': 'delete',
'arguments': parameters,
'dependencies': ['unpack-task']
},
# Loop over unpacked archive
{
'name': 'transcode-task',
'template': 'transcode-pipeline',
'arguments': item_parameters,
'withParam':
'{{tasks.unpack-task.outputs.parameters.videos}}',
'dependencies': ['unpack-task']
},
{
'name':
'state-import-task',
'template':
'data-import',
'arguments':
state_import_parameters,
'dependencies': ['transcode-task'],
'withParam':
'{{tasks.unpack-task.outputs.parameters.states}}'
},
{
'name':
'localization-import-task',
'template':
'data-import',
'arguments':
localization_import_parameters,
'dependencies': ['transcode-task'],
'withParam':
'{{tasks.unpack-task.outputs.parameters.localizations}}'
}
]
} # end of dag
}
transcode_task = self.get_transcode_dag(False)
transcode_task['inputs'] = passthrough_parameters
# pass through the arguments
for task in transcode_task['dag']['tasks']:
task['arguments'] = passthrough_parameters
return [unpack_task, transcode_task]
def get_transcode_dag(self, include_download=True):
if include_download == True:
pipeline_task = {
'name': 'transcode-pipeline',
'dag': {
'tasks': [{
'name': 'download-task',
'template': 'download',
}, {
'name': 'transcode-task',
'template': 'transcode',
'dependencies': [
'download-task',
],
}, {
'name': 'thumbnail-task',
'template': 'thumbnail',
'dependencies': [
'download-task',
],
}, {
'name': 'segments-task',
'template': 'segments',
'dependencies': [
'transcode-task',
],
}, {
'name':
'upload-task',
'template':
'upload',
'dependencies':
['transcode-task', 'thumbnail-task', 'segments-task'],
}],
},
}
else:
pipeline_task = {
'name': 'transcode-pipeline',
'dag': {
'tasks': [{
'name': 'transcode-task',
'template': 'transcode',
}, {
'name': 'thumbnail-task',
'template': 'thumbnail',
}, {
'name': 'segments-task',
'template': 'segments',
'dependencies': [
'transcode-task',
],
}, {
'name':
'upload-task',
'template':
'upload',
'dependencies':
['transcode-task', 'thumbnail-task', 'segments-task'],
}],
},
}
return pipeline_task
def get_transcode_task(self, item, url):
""" Generate a task object describing the dependencies of a transcode """
# Generate an args structure for the DAG
args = [{'name': 'url', 'value': url}]
for key in item:
args.append({'name': key, 'value': item[key]})
parameters = {"parameters": args}
pipeline = self.get_transcode_dag()
for task in pipeline['dag']['tasks']:
task['arguments'] = parameters
return pipeline
def _get_progress_aux(self, job):
return {'section': job['metadata']['annotations']['section']}
def _cancel_message(self):
return 'Transcode aborted!'
def _job_type(self):
return 'upload'
def start_tar_import(self, project, entity_type, token, url, name, section,
md5, gid, uid, user):
""" Initiate a transcode based on the contents on an archive """
comps = name.split('.')
base = comps[0]
ext = '.'.join(comps[1:])
if entity_type != -1:
raise Exception("entity type is not -1!")
self.setup_common_steps(project, token, section, gid, uid, user)
args = {'original': '/work/' + name, 'name': name}
pipeline_tasks = self.get_unpack_and_transcode_tasks(args, url)
# Define the workflow spec.
manifest = {
'apiVersion': 'argoproj.io/v1alpha1',
'kind': 'Workflow',
'metadata': {
'generateName': 'transcode-workflow-',
'labels': {
'job_type': 'upload',
'project': str(project),
'gid': gid,
'uid': uid,
'user': str(user),
},
'annotations': {
'name': name,
'section': section,
},
},
'spec': {
'entrypoint':
'unpack-pipeline',
'onExit':
'exit-handler',
'ttlSecondsAfterFinished':
300,
'volumeClaimTemplates': [self.pvc],
'templates': [
self.download_task, self.delete_task, self.transcode_task,
self.thumbnail_task, self.segments_task, self.upload_task,
self.unpack_task, *pipeline_tasks, self.progress_task,
self.exit_handler, self.data_import
],
},
}
# Create the workflow
response = self.custom.create_namespaced_custom_object(
group='argoproj.io',
version='v1alpha1',
namespace='default',
plural='workflows',
body=manifest,
)
def start_transcode(self, project, entity_type, token, url, name, section,
md5, gid, uid, user):
""" Creates an argo workflow for performing a transcode.
"""
# Define paths for transcode outputs.
base, _ = os.path.splitext(name)
args = {
'original': '/work/' + name,
'transcoded': '/work/' + base + '_transcoded.mp4',
'thumbnail': '/work/' + base + '_thumbnail.jpg',
'thumbnail_gif': '/work/' + base + '_thumbnail_gif.gif',
'segments': '/work/' + base + '_segments.json',
'entity_type': str(entity_type),
'md5': md5,
'name': name
}
self.setup_common_steps(project, token, section, gid, uid, user)
pipeline_task = self.get_transcode_task(args, url)
# Define the workflow spec.
manifest = {
'apiVersion': 'argoproj.io/v1alpha1',
'kind': 'Workflow',
'metadata': {
'generateName': 'transcode-workflow-',
'labels': {
'job_type': 'upload',
'project': str(project),
'gid': gid,
'uid': uid,
'user': str(user),
},
'annotations': {
'name': name,
'section': section,
},
},
'spec': {
'entrypoint':
'transcode-pipeline',
'onExit':
'exit-handler',
'ttlSecondsAfterFinished':
300,
'volumeClaimTemplates': [self.pvc],
'templates': [
self.download_task,
self.transcode_task,
self.thumbnail_task,
self.segments_task,
self.upload_task,
pipeline_task,
self.progress_task,
self.exit_handler,
],
},
}
# Create the workflow
response = self.custom.create_namespaced_custom_object(
group='argoproj.io',
version='v1alpha1',
namespace='default',
plural='workflows',
body=manifest,
)
class TatorAlgorithm(JobManagerMixin):
""" Interface to kubernetes REST API for starting algorithms.
"""
def __init__(self, alg):
""" Intializes the connection. If algorithm object includes
a remote cluster, use that. Otherwise, use this cluster.
"""
if alg.cluster:
host = alg.cluster.host
port = alg.cluster.port
token = alg.cluster.token
fd, cert = tempfile.mkstemp(text=True)
with open(fd, 'w') as f:
f.write(alg.cluster.cert)
conf = Configuration()
conf.api_key['authorization'] = token
conf.host = f'https://{host}:{port}'
conf.verify_ssl = True
conf.ssl_ca_cert = cert
api_client = ApiClient(conf)
self.corev1 = CoreV1Api(api_client)
self.custom = CustomObjectsApi(api_client)
else:
load_incluster_config()
self.corev1 = CoreV1Api()
self.custom = CustomObjectsApi()
# Read in the mainfest.
if alg.manifest:
self.manifest = yaml.safe_load(alg.manifest.open(mode='r'))
# Save off the algorithm name.
self.name = alg.name
def _get_progress_aux(self, job):
return {
'sections': job['metadata']['annotations']['sections'],
'media_ids': job['metadata']['annotations']['media_ids'],
}
def _cancel_message(self):
return 'Algorithm aborted!'
def _job_type(self):
return 'algorithm'
def start_algorithm(self, media_ids, sections, gid, uid, token, project,
user):
""" Starts an algorithm job, substituting in parameters in the
workflow spec.
"""
# Make a copy of the manifest from the database.
manifest = copy.deepcopy(self.manifest)
# Add in workflow parameters.
manifest['spec']['arguments'] = {
'parameters': [
{
'name': 'name',
'value': self.name,
},
{
'name': 'media_ids',
'value': media_ids,
},
{
'name': 'sections',
'value': sections,
},
{
'name': 'gid',
'value': gid,
},
{
'name': 'uid',
'value': uid,
},
{
'name': 'rest_url',
'value': f'https://{os.getenv("MAIN_HOST")}/rest',
},
{
'name': 'rest_token',
'value': str(token),
},
{
'name': 'tus_url',
'value': f'https://{os.getenv("MAIN_HOST")}/files/',
},
{
'name': 'project_id',
'value': str(project),
},
]
}
# If no exit process is defined, add one to close progress.
if 'onExit' not in manifest['spec']:
failed_task = {
'name': 'tator-failed',
'container': {
'image':
get_marshal_image_name(),
'imagePullPolicy':
'Always',
'command': [
'python3',
],
'args': [
'sendProgress.py',
'--url',
f'https://{os.getenv("MAIN_HOST")}/rest',
'--token',
str(token),
'--project',
str(project),
'--job_type',
'algorithm',
'--gid',
gid,
'--uid',
uid,
'--state',
'failed',
'--message',
'Algorithm failed!',
'--progress',
'0',
'--name',
self.name,
'--sections',
sections,
'--media_ids',
media_ids,
],
'resources': {
'limits': {
'memory': '32Mi',
'cpu': '100m',
},
},
},
}
succeeded_task = {
'name': 'tator-succeeded',
'container': {
'image':
get_marshal_image_name(),
'imagePullPolicy':
'Always',
'command': [
'python3',
],
'args': [
'sendProgress.py',
'--url',
f'https://{os.getenv("MAIN_HOST")}/rest',
'--token',
str(token),
'--project',
str(project),
'--job_type',
'algorithm',
'--gid',
gid,
'--uid',
uid,
'--state',
'finished',
'--message',
'Algorithm complete!',
'--progress',
'100',
'--name',
self.name,
'--sections',
sections,
'--media_ids',
media_ids,
],
'resources': {
'limits': {
'memory': '32Mi',
'cpu': '100m',
},
},
},
}
exit_handler = {
'name':
'tator-exit-handler',
'steps': [[{
'name': 'send-fail',
'template': 'tator-failed',
'when': '{{workflow.status}} != Succeeded',
}, {
'name': 'send-succeed',
'template': 'tator-succeeded',
'when': '{{workflow.status}} == Succeeded',
}]],
}
manifest['spec']['onExit'] = 'tator-exit-handler'
manifest['spec']['templates'] += [
failed_task, succeeded_task, exit_handler
]
# Set labels and annotations for job management
if 'labels' not in manifest['metadata']:
manifest['metadata']['labels'] = {}
if 'annotations' not in manifest['metadata']:
manifest['metadata']['annotations'] = {}
manifest['metadata']['labels'] = {
**manifest['metadata']['labels'],
'job_type': 'algorithm',
'project': str(project),
'gid': gid,
'uid': uid,
'user': str(user),
}
manifest['metadata']['annotations'] = {
**manifest['metadata']['annotations'],
'name': self.name,
'sections': sections,
'media_ids': media_ids,
}
response = self.custom.create_namespaced_custom_object(
group='argoproj.io',
version='v1alpha1',
namespace='default',
plural='workflows',
body=manifest,
)
return response
class ClusterDeployment(BaseCustomResource):
"""
A CRD that represents cluster in assisted-service.
On creation the cluster will be registered to the service.
On deletion it will be unregistered from the service.
When has sufficient data installation will start automatically.
"""
_plural = 'clusterdeployments'
def __init__(
self,
kube_api_client: ApiClient,
name: str,
namespace: str = env_variables['namespace'],
):
super().__init__(name, namespace)
self.crd_api = CustomObjectsApi(kube_api_client)
def create_from_yaml(self, yaml_data: dict) -> None:
self.crd_api.create_namespaced_custom_object(
group=HIVE_API_GROUP,
version=HIVE_API_VERSION,
plural=self._plural,
body=yaml_data,
namespace=self.ref.namespace,
)
logger.info('created cluster deployment %s: %s', self.ref,
pformat(yaml_data))
def create(
self,
platform: Platform,
install_strategy: InstallStrategy,
secret: Secret,
base_domain: str = env_variables['base_domain'],
**kwargs,
):
body = {
'apiVersion': f'{HIVE_API_GROUP}/{HIVE_API_VERSION}',
'kind': 'ClusterDeployment',
'metadata': self.ref.as_dict(),
'spec': {
'clusterName': self.ref.name,
'baseDomain': base_domain,
'platform': platform.as_dict(),
'provisioning': {
'installStrategy': install_strategy.as_dict()
},
'pullSecretRef': secret.ref.as_dict(),
}
}
body['spec'].update(kwargs)
self.crd_api.create_namespaced_custom_object(
group=HIVE_API_GROUP,
version=HIVE_API_VERSION,
plural=self._plural,
body=body,
namespace=self.ref.namespace,
)
logger.info('created cluster deployment %s: %s', self.ref,
pformat(body))
def patch(
self,
platform: Optional[Platform] = None,
install_strategy: Optional[InstallStrategy] = None,
secret: Optional[Secret] = None,
**kwargs,
) -> None:
body = {'spec': kwargs}
spec = body['spec']
if platform:
spec['platform'] = platform.as_dict()
if install_strategy:
spec['provisioning'] = {
'installStrategy': install_strategy.as_dict()
}
if secret:
spec['pullSecretRef'] = secret.ref.as_dict()
self.crd_api.patch_namespaced_custom_object(
group=HIVE_API_GROUP,
version=HIVE_API_VERSION,
plural=self._plural,
name=self.ref.name,
namespace=self.ref.namespace,
body=body,
)
logger.info('patching cluster deployment %s: %s', self.ref,
pformat(body))
def annotate_install_config(self, install_config: str) -> None:
body = {
'metadata': {
'annotations': {
f'{CRD_API_GROUP}/install-config-overrides': install_config
}
}
}
self.crd_api.patch_namespaced_custom_object(
group=HIVE_API_GROUP,
version=HIVE_API_VERSION,
plural=self._plural,
name=self.ref.name,
namespace=self.ref.namespace,
body=body,
)
logger.info('patching cluster install config %s: %s', self.ref,
pformat(body))
def get(self) -> dict:
return self.crd_api.get_namespaced_custom_object(
group=HIVE_API_GROUP,
version=HIVE_API_VERSION,
plural=self._plural,
name=self.ref.name,
namespace=self.ref.namespace,
)
def delete(self) -> None:
self.crd_api.delete_namespaced_custom_object(
group=HIVE_API_GROUP,
version=HIVE_API_VERSION,
plural=self._plural,
name=self.ref.name,
namespace=self.ref.namespace,
)
logger.info('deleted cluster deployment %s', self.ref)
def status(
self,
timeout: Union[int, float] = DEFAULT_WAIT_FOR_CRD_STATUS_TIMEOUT,
) -> dict:
"""
Status is a section in the CRD that is created after registration to
assisted service and it defines the observed state of ClusterDeployment.
Since the status key is created only after resource is processed by the
controller in the service, it might take a few seconds before appears.
"""
def _attempt_to_get_status() -> dict:
return self.get()['status']
return waiting.wait(
_attempt_to_get_status,
sleep_seconds=0.5,
timeout_seconds=timeout,
waiting_for=f'cluster {self.ref} status',
expected_exceptions=KeyError,
)
def state(
self,
timeout: Union[int, float] = DEFAULT_WAIT_FOR_CRD_STATE_TIMEOUT,
) -> Tuple[str, str]:
state, state_info = None, None
for condition in self.status(timeout).get('conditions', []):
reason = condition.get('reason')
if reason == 'AgentPlatformState':
state = condition.get('message')
elif reason == 'AgentPlatformStateInfo':
state_info = condition.get('message')
if state and state_info:
break
return state, state_info
def wait_for_state(
self,
required_state: str,
timeout: Union[int, float] = DEFAULT_WAIT_FOR_CRD_STATE_TIMEOUT,
*,
raise_on_states: Iterable[str] = FAILURE_STATES,
) -> None:
required_state = required_state.lower()
raise_on_states = [x.lower() for x in raise_on_states]
def _has_required_state() -> Optional[bool]:
state, state_info = self.state(timeout=0.5)
state = state.lower() if state else state
if state == required_state:
return True
elif state in raise_on_states:
raise UnexpectedStateError(
f'while waiting for state `{required_state}`, cluster '
f'{self.ref} state changed unexpectedly to `{state}`: '
f'{state_info}')
logger.info("Waiting till cluster will be in %s state", required_state)
waiting.wait(
_has_required_state,
timeout_seconds=timeout,
waiting_for=f'cluster {self.ref} state to be {required_state}',
expected_exceptions=waiting.exceptions.TimeoutExpired,
)
def list_agents(self) -> List[Agent]:
return Agent.list(self.crd_api, self)
def wait_for_agents(
self,
num_agents: int = 1,
timeout: Union[int, float] = DEFAULT_WAIT_FOR_AGENTS_TIMEOUT,
) -> List[Agent]:
def _wait_for_sufficient_agents_number() -> List[Agent]:
agents = self.list_agents()
return agents if len(agents) == num_agents else []
return waiting.wait(
_wait_for_sufficient_agents_number,
sleep_seconds=0.5,
timeout_seconds=timeout,
waiting_for=f'cluster {self.ref} to have {num_agents} agents',
)
def wait_to_be_installed(
self,
timeout: Union[int,
float] = DEFAULT_WAIT_FOR_INSTALLATION_COMPLETE_TIMEOUT,
) -> None:
waiting.wait(
lambda: self.get()['spec'].get('installed') is True,
timeout_seconds=timeout,
waiting_for=f'cluster {self.ref} state installed',
expected_exceptions=waiting.exceptions.TimeoutExpired,
)
def download_kubeconfig(self, kubeconfig_path):
def _get_kubeconfig_secret() -> dict:
return self.get(
)['spec']['clusterMetadata']['adminKubeconfigSecretRef']
secret_ref = waiting.wait(
_get_kubeconfig_secret,
sleep_seconds=1,
timeout_seconds=240,
expected_exceptions=KeyError,
waiting_for=f'kubeconfig secret creation for cluster {self.ref}',
)
kubeconfig_data = Secret(
kube_api_client=self.crd_api.api_client,
**secret_ref,
).get().data['kubeconfig']
with open(kubeconfig_path, 'wt') as kubeconfig_file:
kubeconfig_file.write(b64decode(kubeconfig_data).decode())
class ClusterDeployment(BaseCustomResource):
"""
A CRD that represents cluster in assisted-service.
On creation the cluster will be registered to the service.
On deletion it will be unregistered from the service.
When has sufficient data installation will start automatically.
"""
_plural = "clusterdeployments"
_platform_field = {"platform": {"agentBareMetal": {"agentSelector": {}}}}
def __init__(
self,
kube_api_client: ApiClient,
name: str,
namespace: str = env_variables["namespace"],
):
super().__init__(name, namespace)
self.crd_api = CustomObjectsApi(kube_api_client)
def create_from_yaml(self, yaml_data: dict) -> None:
self.crd_api.create_namespaced_custom_object(
group=HIVE_API_GROUP,
version=HIVE_API_VERSION,
plural=self._plural,
body=yaml_data,
namespace=self.ref.namespace,
)
logger.info("created cluster deployment %s: %s", self.ref, pformat(yaml_data))
def create(
self,
secret: Secret,
base_domain: str = env_variables["base_domain"],
agent_cluster_install_ref: Optional[ObjectReference] = None,
**kwargs,
):
body = {
"apiVersion": f"{HIVE_API_GROUP}/{HIVE_API_VERSION}",
"kind": "ClusterDeployment",
"metadata": self.ref.as_dict(),
"spec": {
"clusterName": self.ref.name,
"baseDomain": base_domain,
"pullSecretRef": secret.ref.as_dict(),
}
}
body["spec"].update(self._platform_field)
if agent_cluster_install_ref:
body["spec"]["clusterInstallRef"] = agent_cluster_install_ref.as_dict()
body["spec"].update(kwargs)
self.crd_api.create_namespaced_custom_object(
group=HIVE_API_GROUP,
version=HIVE_API_VERSION,
plural=self._plural,
body=body,
namespace=self.ref.namespace,
)
logger.info("created cluster deployment %s: %s", self.ref, pformat(body))
def patch(
self,
secret: Optional[Secret] = None,
**kwargs,
) -> None:
body = {"spec": kwargs}
body["spec"]["platform"] = {"agentBareMetal": {}}
spec = body["spec"]
body["spec"].update(self._platform_field)
if secret:
spec["pullSecretRef"] = secret.ref.as_dict()
if "agent_cluster_install_ref" in kwargs:
spec["clusterInstallRef"] = kwargs["agent_cluster_install_ref"].as_dict()
self.crd_api.patch_namespaced_custom_object(
group=HIVE_API_GROUP,
version=HIVE_API_VERSION,
plural=self._plural,
name=self.ref.name,
namespace=self.ref.namespace,
body=body,
)
logger.info("patching cluster deployment %s: %s", self.ref, pformat(body))
def annotate_install_config(self, install_config: str) -> None:
body = {"metadata": {"annotations": {f"{CRD_API_GROUP}/install-config-overrides": install_config}}}
self.crd_api.patch_namespaced_custom_object(
group=HIVE_API_GROUP,
version=HIVE_API_VERSION,
plural=self._plural,
name=self.ref.name,
namespace=self.ref.namespace,
body=body,
)
logger.info("patching cluster install config %s: %s", self.ref, pformat(body))
def get(self) -> dict:
return self.crd_api.get_namespaced_custom_object(
group=HIVE_API_GROUP,
version=HIVE_API_VERSION,
plural=self._plural,
name=self.ref.name,
namespace=self.ref.namespace,
)
def delete(self) -> None:
self.crd_api.delete_namespaced_custom_object(
group=HIVE_API_GROUP,
version=HIVE_API_VERSION,
plural=self._plural,
name=self.ref.name,
namespace=self.ref.namespace,
)
logger.info("deleted cluster deployment %s", self.ref)
def status(
self,
timeout: Union[int, float] = DEFAULT_WAIT_FOR_CRD_STATUS_TIMEOUT,
) -> dict:
"""
Status is a section in the CRD that is created after registration to
assisted service and it defines the observed state of ClusterDeployment.
Since the status key is created only after resource is processed by the
controller in the service, it might take a few seconds before appears.
"""
def _attempt_to_get_status() -> dict:
return self.get()["status"]
return waiting.wait(
_attempt_to_get_status,
sleep_seconds=0.5,
timeout_seconds=timeout,
waiting_for=f"cluster {self.ref} status",
expected_exceptions=KeyError,
)
def condition(
self,
cond_type,
timeout: Union[int, float] = DEFAULT_WAIT_FOR_CRD_STATE_TIMEOUT,
) -> Tuple[Optional[str], Optional[str]]:
for condition in self.status(timeout).get("conditions", []):
if cond_type == condition.get("type"):
return condition.get("status"), condition.get("reason")
return None, None
def wait_for_condition(
self,
cond_type: str,
required_status: str,
required_reason: Optional[str] = None,
timeout: Union[int, float] = DEFAULT_WAIT_FOR_CRD_STATE_TIMEOUT,
) -> None:
def _has_required_condition() -> Optional[bool]:
status, reason = self.condition(cond_type=cond_type, timeout=0.5)
if status == required_status:
if required_reason:
return required_reason == reason
return True
return False
logger.info(
"Waiting till cluster will be in condition %s with status: %s "
"reason: %s", cond_type, required_status, required_reason
)
waiting.wait(
_has_required_condition,
timeout_seconds=timeout,
waiting_for=f"cluster {self.ref} condition {cond_type} to be in {required_status}",
expected_exceptions=waiting.exceptions.TimeoutExpired,
)
def list_agents(self) -> List[Agent]:
return Agent.list(self.crd_api, self)
def wait_for_agents(
self,
num_agents: int = 1,
timeout: Union[int, float] = DEFAULT_WAIT_FOR_AGENTS_TIMEOUT,
) -> List[Agent]:
def _wait_for_sufficient_agents_number() -> List[Agent]:
agents = self.list_agents()
return agents if len(agents) == num_agents else []
return waiting.wait(
_wait_for_sufficient_agents_number,
sleep_seconds=0.5,
timeout_seconds=timeout,
waiting_for=f"cluster {self.ref} to have {num_agents} agents",
)
class TraefikMiddlewareReconciler(KubernetesObjectReconciler[TraefikMiddleware]
):
"""Kubernetes Traefik Middleware Reconciler"""
def __init__(self, controller: "KubernetesController") -> None:
super().__init__(controller)
self.api_ex = ApiextensionsV1Api(controller.client)
self.api = CustomObjectsApi(controller.client)
@property
def noop(self) -> bool:
if not ProxyProvider.objects.filter(
outpost__in=[self.controller.outpost],
mode__in=[ProxyMode.FORWARD_SINGLE, ProxyMode.FORWARD_DOMAIN],
).exists():
self.logger.debug("No providers with forward auth enabled.")
return True
if not self._crd_exists():
self.logger.debug("CRD doesn't exist")
return True
return False
def _crd_exists(self) -> bool:
"""Check if the traefik middleware exists"""
return bool(
len(
self.api_ex.list_custom_resource_definition(
field_selector=f"metadata.name={CRD_NAME}").items))
def reconcile(self, current: TraefikMiddleware,
reference: TraefikMiddleware):
super().reconcile(current, reference)
if current.spec.forwardAuth.address != reference.spec.forwardAuth.address:
raise NeedsUpdate()
if (current.spec.forwardAuth.authResponseHeadersRegex !=
reference.spec.forwardAuth.authResponseHeadersRegex):
raise NeedsUpdate()
# Ensure all of our headers are set, others can be added by the user.
if not set(current.spec.forwardAuth.authResponseHeaders).issubset(
reference.spec.forwardAuth.authResponseHeaders):
raise NeedsUpdate()
def get_reference_object(self) -> TraefikMiddleware:
"""Get deployment object for outpost"""
return TraefikMiddleware(
apiVersion=f"{CRD_GROUP}/{CRD_VERSION}",
kind="Middleware",
metadata=TraefikMiddlewareMetadata(
name=self.name,
namespace=self.namespace,
labels=self.get_object_meta().labels,
),
spec=
TraefikMiddlewareSpec(forwardAuth=TraefikMiddlewareSpecForwardAuth(
address=
f"http://{self.name}.{self.namespace}:9000/akprox/auth/traefik",
authResponseHeaders=[
"X-authentik-username",
"X-authentik-groups",
"X-authentik-email",
"X-authentik-name",
"X-authentik-uid",
"X-authentik-jwt",
"X-authentik-meta-jwks",
"X-authentik-meta-outpost",
"X-authentik-meta-provider",
"X-authentik-meta-app",
"X-authentik-meta-version",
],
authResponseHeadersRegex="",
trustForwardHeader=True,
)),
)
def create(self, reference: TraefikMiddleware):
return self.api.create_namespaced_custom_object(
group=CRD_GROUP,
version=CRD_VERSION,
plural=CRD_PLURAL,
namespace=self.namespace,
body=asdict(reference),
field_manager=FIELD_MANAGER,
)
def delete(self, reference: TraefikMiddleware):
return self.api.delete_namespaced_custom_object(
group=CRD_GROUP,
version=CRD_VERSION,
namespace=self.namespace,
plural=CRD_PLURAL,
name=self.name,
)
def retrieve(self) -> TraefikMiddleware:
return from_dict(
TraefikMiddleware,
self.api.get_namespaced_custom_object(
group=CRD_GROUP,
version=CRD_VERSION,
namespace=self.namespace,
plural=CRD_PLURAL,
name=self.name,
),
)
def update(self, current: TraefikMiddleware, reference: TraefikMiddleware):
return self.api.patch_namespaced_custom_object(
group=CRD_GROUP,
version=CRD_VERSION,
namespace=self.namespace,
plural=CRD_PLURAL,
name=self.name,
body=asdict(reference),
field_manager=FIELD_MANAGER,
)
class AgentClusterInstall(BaseCustomResource):
""" This CRD represents a request to provision an agent based cluster.
In the AgentClusterInstall, the user can specify requirements like
networking, number of control plane and workers nodes and more.
The installation will start automatically if the required number of
hosts is available, the hosts are ready to be installed and the Agents
are approved.
The AgentClusterInstall reflects the ClusterDeployment/Installation
status through Conditions."""
_api_group = "extensions.hive.openshift.io"
_api_version = "v1beta1"
_plural = "agentclusterinstalls"
_kind = "AgentClusterInstall"
_requirements_met_condition_name = "RequirementsMet"
_completed_condition_name = "Completed"
def __init__(
self,
kube_api_client: ApiClient,
name: str,
namespace: str = consts.DEFAULT_NAMESPACE,
):
super().__init__(name, namespace)
self.crd_api = CustomObjectsApi(kube_api_client)
self.ref.kind = self._kind
self.ref.group = self._api_group
self.ref.version = self._api_version
def create(
self,
cluster_deployment_ref: ObjectReference,
cluster_cidr: str,
host_prefix: int,
service_network: str,
control_plane_agents: int,
**kwargs,
) -> None:
body = {
"apiVersion":
f"{self._api_group}/{self._api_version}",
"kind":
self._kind,
"metadata":
self.ref.as_dict(),
"spec":
self._get_spec_dict(
cluster_deployment_ref=cluster_deployment_ref,
cluster_cidr=cluster_cidr,
host_prefix=host_prefix,
service_network=service_network,
control_plane_agents=control_plane_agents,
**kwargs,
),
}
self.crd_api.create_namespaced_custom_object(
group=self._api_group,
version=self._api_version,
plural=self._plural,
body=body,
namespace=self.ref.namespace,
)
logger.info("created agentclusterinstall %s: %s", self.ref,
pformat(body))
def patch(
self,
cluster_deployment_ref: ObjectReference,
cluster_cidr: str,
host_prefix: int,
service_network: str,
control_plane_agents: int,
**kwargs,
) -> None:
body = {
"spec":
self._get_spec_dict(
cluster_deployment_ref=cluster_deployment_ref,
cluster_cidr=cluster_cidr,
host_prefix=host_prefix,
service_network=service_network,
control_plane_agents=control_plane_agents,
**kwargs,
)
}
self.crd_api.patch_namespaced_custom_object(
group=self._api_group,
version=self._api_version,
plural=self._plural,
name=self.ref.name,
namespace=self.ref.namespace,
body=body,
)
logger.info("patching agentclusterinstall %s: %s", self.ref,
pformat(body))
@staticmethod
def _get_spec_dict(
cluster_deployment_ref: ObjectReference,
cluster_cidr: str,
host_prefix: int,
service_network: str,
control_plane_agents: int,
**kwargs,
) -> dict:
spec = {
"clusterDeploymentRef":
cluster_deployment_ref.as_dict(),
"imageSetRef":
kwargs.pop("image_set_ref", ClusterImageSetReference()).as_dict(),
"networking": {
"clusterNetwork": [{
"cidr": cluster_cidr,
"hostPrefix": host_prefix,
}],
"serviceNetwork": [service_network],
},
"provisionRequirements": {
"controlPlaneAgents": control_plane_agents,
"workerAgents": kwargs.pop("worker_agents", 0),
}
}
if "api_vip" in kwargs:
spec["apiVIP"] = kwargs.pop("api_vip")
if "ingress_vip" in kwargs:
spec["ingressVIP"] = kwargs.pop("ingress_vip")
if "ssh_pub_key" in kwargs:
spec["sshPublicKey"] = kwargs.pop("ssh_pub_key")
if "machine_cidr" in kwargs:
spec["networking"]["machineNetwork"] = [{
"cidr":
kwargs.pop("machine_cidr")
}]
spec.update(kwargs)
return spec
def get(self) -> dict:
return self.crd_api.get_namespaced_custom_object(
group=self._api_group,
version=self._api_version,
plural=self._plural,
name=self.ref.name,
namespace=self.ref.namespace,
)
def delete(self) -> None:
self.crd_api.delete_namespaced_custom_object(
group=self._api_group,
version=self._api_version,
plural=self._plural,
name=self.ref.name,
namespace=self.ref.namespace,
)
logger.info("deleted agentclusterinstall %s", self.ref)
def status(
self,
timeout: Union[int] = DEFAULT_WAIT_FOR_CRD_STATE_TIMEOUT) -> dict:
def _attempt_to_get_status() -> dict:
return self.get()["status"]
return waiting.wait(
_attempt_to_get_status,
sleep_seconds=0.5,
timeout_seconds=timeout,
waiting_for=f"cluster {self.ref} status",
expected_exceptions=KeyError,
)
def wait_to_be_ready(
self,
ready: bool,
timeout: Union[int, float] = DEFAULT_WAIT_FOR_CRD_STATE_TIMEOUT,
) -> None:
return self.wait_for_condition(
cond_type=self._requirements_met_condition_name,
required_status=str(ready),
timeout=timeout,
)
def wait_to_be_installing(
self,
timeout: Union[int, float] = DEFAULT_WAIT_FOR_CRD_STATE_TIMEOUT,
) -> None:
return self.wait_for_condition(
cond_type=self._requirements_met_condition_name,
required_status="True",
required_reason="ClusterAlreadyInstalling",
timeout=timeout,
)
def wait_to_be_installed(
self,
timeout: Union[int,
float] = DEFAULT_WAIT_FOR_INSTALLATION_COMPLETE_TIMEOUT,
) -> None:
return self.wait_for_condition(
cond_type=self._completed_condition_name,
required_status="True",
required_reason="InstallationCompleted",
timeout=timeout,
)
def wait_for_condition(
self,
cond_type: str,
required_status: str,
required_reason: Optional[str] = None,
timeout: Union[int, float] = DEFAULT_WAIT_FOR_CRD_STATE_TIMEOUT,
) -> None:
logger.info(
"waiting for agentclusterinstall %s condition %s to be in status "
"%s", self.ref, cond_type, required_status)
def _has_required_condition() -> Optional[bool]:
status, reason = self.condition(cond_type=cond_type, timeout=0.5)
logger.info(
f"waiting for condition <{cond_type}> to be in status <{required_status}>. actual status is: {status} {reason}"
)
if status == required_status:
if required_reason:
return required_reason == reason
return True
waiting.wait(
_has_required_condition,
timeout_seconds=timeout,
waiting_for=f"agentclusterinstall {self.ref} condition "
f"{cond_type} to be {required_status}",
sleep_seconds=10,
expected_exceptions=waiting.exceptions.TimeoutExpired,
)
def condition(
self,
cond_type,
timeout: Union[int, float] = DEFAULT_WAIT_FOR_CRD_STATE_TIMEOUT,
) -> Tuple[Optional[str], Optional[str]]:
for condition in self.status(timeout).get("conditions", []):
if cond_type == condition.get("type"):
return condition.get("status"), condition.get("reason")
return None, None
def download_kubeconfig(self, kubeconfig_path):
def _get_kubeconfig_secret() -> dict:
return self.get(
)["spec"]["clusterMetadata"]["adminKubeconfigSecretRef"]
secret_ref = waiting.wait(
_get_kubeconfig_secret,
sleep_seconds=1,
timeout_seconds=DEFAULT_WAIT_FOR_KUBECONFIG_TIMEOUT,
expected_exceptions=KeyError,
waiting_for=f"kubeconfig secret creation for cluster {self.ref}",
)
kubeconfig_data = (Secret(
kube_api_client=self.crd_api.api_client,
namespace=self._reference.namespace,
**secret_ref,
).get().data["kubeconfig"])
with open(kubeconfig_path, "wt") as kubeconfig_file:
kubeconfig_file.write(b64decode(kubeconfig_data).decode())
class NMStateConfig(BaseCustomResource):
"""Configure nmstate (static IP) related settings for agents."""
_plural = 'nmstateconfigs'
def __init__(
self,
kube_api_client: ApiClient,
name: str,
namespace: str = env_variables['namespace'],
):
super().__init__(name, namespace)
self.crd_api = CustomObjectsApi(kube_api_client)
def create_from_yaml(self, yaml_data: dict) -> None:
self.crd_api.create_namespaced_custom_object(
group=CRD_API_GROUP,
version=CRD_API_VERSION,
plural=self._plural,
body=yaml_data,
namespace=self.ref.namespace,
)
logger.info('created nmstate config %s: %s', self.ref,
pformat(yaml_data))
def create(
self,
config: dict,
interfaces: list,
label: Optional[str] = None,
**kwargs,
) -> None:
body = {
'apiVersion': f'{CRD_API_GROUP}/{CRD_API_VERSION}',
'kind': 'NMStateConfig',
'metadata': {
'labels': {
f'{CRD_API_GROUP}/selector-nmstate-config-name': label,
},
**self.ref.as_dict()
},
'spec': {
'config': config,
'interfaces': interfaces,
}
}
body['spec'].update(kwargs)
self.crd_api.create_namespaced_custom_object(
group=CRD_API_GROUP,
version=CRD_API_VERSION,
plural=self._plural,
body=body,
namespace=self.ref.namespace,
)
logger.info('created nmstate config %s: %s', self.ref, pformat(body))
def patch(
self,
config: dict,
interfaces: list,
**kwargs,
) -> None:
body = {'spec': kwargs}
spec = body['spec']
if config:
spec['config'] = config
if interfaces:
spec['interfaces'] = interfaces
self.crd_api.patch_namespaced_custom_object(
group=CRD_API_GROUP,
version=CRD_API_VERSION,
plural=self._plural,
name=self.ref.name,
namespace=self.ref.namespace,
body=body,
)
logger.info('patching nmstate config %s: %s', self.ref, pformat(body))
def get(self) -> dict:
return self.crd_api.get_namespaced_custom_object(
group=CRD_API_GROUP,
version=CRD_API_VERSION,
plural=self._plural,
name=self.ref.name,
namespace=self.ref.namespace,
)
def delete(self) -> None:
self.crd_api.delete_namespaced_custom_object(
group=CRD_API_GROUP,
version=CRD_API_VERSION,
plural=self._plural,
name=self.ref.name,
namespace=self.ref.namespace,
)
logger.info('deleted nmstate config %s', self.ref)
def status(
self,
timeout: Union[int,
float] = DEFAULT_WAIT_FOR_CRD_STATUS_TIMEOUT) -> dict:
"""
Status is a section in the CRD that is created after registration to
assisted service and it defines the observed state of NMStateConfig.
Since the status key is created only after resource is processed by the
controller in the service, it might take a few seconds before appears.
"""
def _attempt_to_get_status() -> dict:
return self.get()['status']
return waiting.wait(_attempt_to_get_status,
sleep_seconds=0.5,
timeout_seconds=timeout,
waiting_for=f'nmstate config {self.ref} status',
expected_exceptions=KeyError)
class NMStateConfig(BaseCustomResource):
"""Configure nmstate (static IP) related settings for agents."""
_plural = "nmstateconfigs"
def __init__(
self,
kube_api_client: ApiClient,
name: str,
namespace: str = consts.DEFAULT_NAMESPACE,
):
super().__init__(name, namespace)
self.crd_api = CustomObjectsApi(kube_api_client)
def create_from_yaml(self, yaml_data: dict) -> None:
self.crd_api.create_namespaced_custom_object(
group=CRD_API_GROUP,
version=CRD_API_VERSION,
plural=self._plural,
body=yaml_data,
namespace=self.ref.namespace,
)
logger.info("created nmstate config %s: %s", self.ref,
pformat(yaml_data))
def create(
self,
config: dict,
interfaces: list,
label: Optional[str] = None,
**kwargs,
) -> None:
body = {
"apiVersion": f"{CRD_API_GROUP}/{CRD_API_VERSION}",
"kind": "NMStateConfig",
"metadata": {
"labels": {
f"{CRD_API_GROUP}/selector-nmstate-config-name": label,
},
**self.ref.as_dict(),
},
"spec": {
"config": config,
"interfaces": interfaces,
},
}
body["spec"].update(kwargs)
self.crd_api.create_namespaced_custom_object(
group=CRD_API_GROUP,
version=CRD_API_VERSION,
plural=self._plural,
body=body,
namespace=self.ref.namespace,
)
logger.info("created nmstate config %s: %s", self.ref, pformat(body))
def patch(
self,
config: dict,
interfaces: list,
**kwargs,
) -> None:
body = {"spec": kwargs}
spec = body["spec"]
if config:
spec["config"] = config
if interfaces:
spec["interfaces"] = interfaces
self.crd_api.patch_namespaced_custom_object(
group=CRD_API_GROUP,
version=CRD_API_VERSION,
plural=self._plural,
name=self.ref.name,
namespace=self.ref.namespace,
body=body,
)
logger.info("patching nmstate config %s: %s", self.ref, pformat(body))
def get(self) -> dict:
return self.crd_api.get_namespaced_custom_object(
group=CRD_API_GROUP,
version=CRD_API_VERSION,
plural=self._plural,
name=self.ref.name,
namespace=self.ref.namespace,
)
def delete(self) -> None:
self.crd_api.delete_namespaced_custom_object(
group=CRD_API_GROUP,
version=CRD_API_VERSION,
plural=self._plural,
name=self.ref.name,
namespace=self.ref.namespace,
)
logger.info("deleted nmstate config %s", self.ref)
def status(
self,
timeout: Union[int,
float] = DEFAULT_WAIT_FOR_CRD_STATUS_TIMEOUT) -> dict:
"""
Status is a section in the CRD that is created after registration to
assisted service and it defines the observed state of NMStateConfig.
Since the status key is created only after resource is processed by the
controller in the service, it might take a few seconds before appears.
"""
def _attempt_to_get_status() -> dict:
return self.get()["status"]
return waiting.wait(
_attempt_to_get_status,
sleep_seconds=0.5,
timeout_seconds=timeout,
waiting_for=f"nmstate config {self.ref} status",
expected_exceptions=KeyError,
)
class PrometheusServiceMonitorReconciler(
KubernetesObjectReconciler[PrometheusServiceMonitor]):
"""Kubernetes Prometheus ServiceMonitor Reconciler"""
def __init__(self, controller: "KubernetesController") -> None:
super().__init__(controller)
self.api_ex = ApiextensionsV1Api(controller.client)
self.api = CustomObjectsApi(controller.client)
@property
def noop(self) -> bool:
return (not self._crd_exists()) or (self.is_embedded)
def _crd_exists(self) -> bool:
"""Check if the Prometheus ServiceMonitor exists"""
return bool(
len(
self.api_ex.list_custom_resource_definition(
field_selector=f"metadata.name={CRD_NAME}").items))
def get_reference_object(self) -> PrometheusServiceMonitor:
"""Get service monitor object for outpost"""
return PrometheusServiceMonitor(
apiVersion=f"{CRD_GROUP}/{CRD_VERSION}",
kind="ServiceMonitor",
metadata=PrometheusServiceMonitorMetadata(
name=self.name,
namespace=self.namespace,
labels=self.get_object_meta().labels,
),
spec=PrometheusServiceMonitorSpec(
endpoints=[
PrometheusServiceMonitorSpecEndpoint(port="http-metrics", )
],
selector=PrometheusServiceMonitorSpecSelector(
matchLabels=self.get_object_meta(name=self.name).labels, ),
),
)
def create(self, reference: PrometheusServiceMonitor):
return self.api.create_namespaced_custom_object(
group=CRD_GROUP,
version=CRD_VERSION,
plural=CRD_PLURAL,
namespace=self.namespace,
body=asdict(reference),
field_manager=FIELD_MANAGER,
)
def delete(self, reference: PrometheusServiceMonitor):
return self.api.delete_namespaced_custom_object(
group=CRD_GROUP,
version=CRD_VERSION,
namespace=self.namespace,
plural=CRD_PLURAL,
name=self.name,
)
def retrieve(self) -> PrometheusServiceMonitor:
return from_dict(
PrometheusServiceMonitor,
self.api.get_namespaced_custom_object(
group=CRD_GROUP,
version=CRD_VERSION,
namespace=self.namespace,
plural=CRD_PLURAL,
name=self.name,
),
)
def update(self, current: PrometheusServiceMonitor,
reference: PrometheusServiceMonitor):
return self.api.patch_namespaced_custom_object(
group=CRD_GROUP,
version=CRD_VERSION,
namespace=self.namespace,
plural=CRD_PLURAL,
name=self.name,
body=asdict(reference),
field_manager=FIELD_MANAGER,
)
