def get_new_containers_log_targets(
containers: list, containers_path: str, cluster_id: str, kube_url=None, strict_labels=None) -> list:
"""
Return list of container log targets. A ``target`` includes:
{
"id": <container_id>,
"kwargs": <template_kwargs>,
"pod_labels": <container's pod labels>
}
:param containers: List of container configs dicts.
:type containers: list
:param containers_path: Path to mounted containers directory.
:type containers_path: str
:param cluster_id: kubernetes cluster ID. If not set, then it will not be added to job/config files.
:type cluster_id: str
:param kube_url: URL to Kube API proxy.
:type kube_url: str
:param strict_labels: List of labels pods need to posses in order to be followed.
:type strict_labels: List
:return: List of existing container log targets.
:rtype: list
"""
containers_log_targets = []
strict_labels = strict_labels or []
for container in containers:
try:
config = container['config']
if kube.is_pause_container(config['Config']):
# We have no interest in Pause containers.
continue
pod_name = get_container_label_value(config, 'pod.name')
container_name = get_container_label_value(config, 'container.name')
pod_namespace = get_container_label_value(config, 'pod.namespace')
try:
pod = kube.get_pod(pod_name, namespace=pod_namespace, kube_url=kube_url)
except kube.PodNotFound:
logger.warning('Cannot find pod "%s" ... skipping container: %s', pod_name, container_name)
continue
metadata = pod.obj['metadata']
pod_labels, pod_annotations = metadata.get('labels', {}), metadata.get('annotations', {})
kwargs = {}
kwargs['container_id'] = container['id']
kwargs['container_path'] = os.path.join(containers_path, container['id'])
kwargs['log_file_name'] = os.path.basename(container['log_file'])
kwargs['log_file_path'] = container['log_file']
kwargs['image'], kwargs['image_version'] = get_container_image_parts(config['Config'])
kwargs['application'] = pod_labels.get(APP_LABEL, '')
kwargs['component'] = pod_labels.get(COMPONENT_LABEL)
kwargs['environment'] = pod_labels.get(ENVIRONMENT_LABEL, CLUSTER_ENVIRONMENT)
kwargs['version'] = pod_labels.get(VERSION_LABEL, '')
kwargs['release'] = pod_labels.get('release', '')
kwargs['cluster_id'] = cluster_id
kwargs['pod_name'] = pod_name
kwargs['namespace'] = pod_namespace
kwargs['container_name'] = container_name
kwargs['node_name'] = CLUSTER_NODE_NAME
kwargs['pod_annotations'] = pod_annotations
if set(strict_labels) - set(pod_labels.keys()):
logger.warning('Labels "%s" are required for container(%s: %s) in pod(%s) ... Skipping!',
','.join(strict_labels), container_name, container['id'], pod_name)
continue
containers_log_targets.append({'id': container['id'], 'kwargs': kwargs, 'pod_labels': pod_labels})
except Exception:
logger.exception('Failed to create log target for container(%s)', container['id'])
return containers_log_targets
def test_pause_container(monkeypatch, config, res):
assert res == is_pause_container(config)
def get_new_containers_log_targets(containers: list,
containers_path: str,
cluster_id: str,
kube_url=None,
strict_labels=False) -> list:
"""
Return list of container log targets. A ``target`` includes:
{
"id": <container_id>,
"kwargs": <template_kwargs>,
"pod_labels": <container's pod labels>
}
:param containers: List of container configs dicts.
:type containers: list
:param containers_path: Path to mounted containers directory.
:type containers_path: str
:param cluster_id: kubernetes cluster ID. If not set, then it will not be added to job/config files.
:type cluster_id: str
:param kube_url: URL to Kube API proxy.
:type kube_url: str
:param strict_labels: Only follow logs from pods with labels "application" and "version" labels set. Default False.
:type strict_labels: bool
:return: List of existing container log targets.
:rtype: list
"""
pod_map = {}
pod_map[kube.DEFAULT_NAMESPACE] = kube.get_pods(kube_url=kube_url)
containers_log_targets = []
for container in containers:
try:
config = container['config']
if kube.is_pause_container(config['Config']):
# We have no interest in Pause containers.
logger.debug('Skipping pause container({})'.format(
container['id']))
continue
pod_name = get_label_value(config, 'pod.name')
container_name = get_label_value(config, 'container.name')
pod_namespace = get_label_value(config, 'pod.namespace')
pods = pod_map.get(pod_namespace)
if not pods:
# We need to get pods in different namespace
logger.debug(
'Retrieving pods in namespace: {}'.format(pod_namespace))
pods = kube.get_pods(kube_url=kube_url,
namespace=pod_namespace)
pod_map[pod_namespace] = pods
pod_labels, pod_annotations = kube.get_pod_labels_annotations(
pods, pod_name)
kwargs = {}
kwargs['container_id'] = container['id']
kwargs['container_path'] = os.path.join(containers_path,
container['id'])
kwargs['log_file_name'] = os.path.basename(container['log_file'])
kwargs['log_file_path'] = container['log_file']
kwargs['image'], kwargs[
'image_version'] = get_container_image_parts(config['Config'])
kwargs['application_id'] = pod_labels.get(APP_LABEL)
kwargs['application_version'] = pod_labels.get(VERSION_LABEL, '')
kwargs['release'] = pod_labels.get('release', '')
kwargs['cluster_id'] = cluster_id
kwargs['pod_name'] = pod_name
kwargs['namespace'] = pod_namespace
kwargs['container_name'] = container_name
kwargs['node_name'] = CLUSTER_NODE_NAME
kwargs['pod_annotations'] = pod_annotations
if not all(
[kwargs['application_id'], kwargs['application_version']]):
if strict_labels:
logger.warning((
'Labels "{}" and "{}" are required for container({}: {}) in pod({}) '
'... Skipping!').format(APP_LABEL, VERSION_LABEL,
container_name,
container['id'], pod_name))
continue
else:
if not kwargs['application_id']:
kwargs['application_id'] = kwargs['pod_name']
containers_log_targets.append({
'id': container['id'],
'kwargs': kwargs,
'pod_labels': pod_labels
})
except Exception:
logger.exception(
'Failed to create log target for container({})'.format(
container['id']))
return containers_log_targets