def _piggyback_serializer(section: PiggyBackSection):
with ConditionalPiggybackSection(section.name):
with SectionWriter("labels") as w:
w.append(json.dumps(section.labels))
for s in section.sections:
new_s = ResultSection(f"{section.service_name}_{s.name}",
s.results)
_result_serializer(new_s)
def agent_mobileiron_main(args: Args) -> None:
"""Fetches and writes selected information formatted as agent output to stdout.
Standard out with sections and piggyback example:
<<<<entityName>>>>
<<<mobileiron_section>>>
{"...": ...}
<<<<entityName>>>>
<<<mobileiron_source_host>>>
{"...": ...}
<<<<>>>>
"""
LOGGER.info("Fetch general device information...")
if args.debug:
LOGGER.debug("Initialize Mobileiron API")
with MobileironAPI(
args.hostname,
args.port,
auth=(args.username, args.password),
verify=not args.no_cert_check,
proxies=_proxy_address(
args.proxy_host,
args.proxy_port,
args.proxy_user,
args.proxy_password,
) if args.proxy_host else None,
) as mobileiron_api:
all_devices = mobileiron_api.get_all_devices(partitions=args.partition)
if args.debug:
LOGGER.debug("Received the following devices: %s", all_devices)
LOGGER.info("Write agent output..")
for device in all_devices:
if "total_count" in all_devices[device]:
with ConditionalPiggybackSection(device), SectionWriter(
"mobileiron_source_host") as writer:
writer.append_json(all_devices[device])
else:
with ConditionalPiggybackSection(device), SectionWriter(
"mobileiron_section") as writer:
writer.append_json(all_devices[device])
def alertmanager_rules_section(
api_client: AlertmanagerAPI,
config: Dict[str, Any],
) -> None:
rule_groups = retrieve_rule_data(api_client)
if not rule_groups.get("groups"):
return
parsed_data = parse_rule_data(rule_groups["groups"], config["ignore_alerts"])
with ConditionalPiggybackSection(config["hostname"]):
with SectionWriter("alertmanager") as writer:
writer.append_json(parsed_data)
def output_nodes_api_sections(api_nodes: Sequence[Node]) -> None:
def output_sections(cluster_node: Node) -> None:
sections = {
"kube_node_container_count_v1": cluster_node.container_count,
"kube_node_kubelet_v1": cluster_node.kubelet,
"kube_pod_resources_with_capacity_v1": cluster_node.pod_resources,
"kube_node_info_v1": cluster_node.info,
}
_write_sections(sections)
for node in api_nodes:
with ConditionalPiggybackSection(f"node_{node.name}"):
output_sections(node)
def output_pods_api_sections(api_pods: Sequence[Pod]) -> None:
def output_sections(cluster_pod: Pod) -> None:
sections = {
"kube_cpu_resources_v1": cluster_pod.cpu_resources,
"k8s_pod_conditions_v1": cluster_pod.conditions,
"kube_pod_containers_v1": cluster_pod.containers_infos,
"kube_start_time_v1": cluster_pod.start_time,
"kube_memory_resources_v1": cluster_pod.memory_resources,
"kube_pod_lifecycle_v1": cluster_pod.lifecycle_phase,
}
_write_sections(sections)
for pod in api_pods:
with ConditionalPiggybackSection(
f"pod_{pod.name(prepend_namespace=True)}"):
output_sections(pod)
def output_deployments_api_sections(
api_deployments: Sequence[Deployment]) -> None:
"""Write the deployment relevant sections based on k8 API information"""
def output_sections(cluster_deployment: Deployment) -> None:
sections = {
"kube_pod_resources_v1": cluster_deployment.pod_resources,
"kube_memory_resources_v1": cluster_deployment.memory_resources,
"kube_deployment_info_v1": cluster_deployment.info,
"kube_deployment_conditions_v1": cluster_deployment.conditions,
}
_write_sections(sections)
for deployment in api_deployments:
with ConditionalPiggybackSection(
f"deployment_{deployment.name(prepend_namespace=True)}"):
output_sections(deployment)
def agent_proxmox_ve_main(args: Args) -> None:
"""Fetches and writes selected information formatted as agent output to stdout"""
with ProxmoxVeAPI(
host=args.hostname,
port=args.port,
credentials={
k: getattr(args, k)
for k in ("username", "password") if getattr(args, k)
},
timeout=args.timeout,
verify_ssl=not args.no_cert_check,
) as session:
LOGGER.info("Fetch general cluster and node information..")
data = session.get_tree({
"cluster": {
"backup": [],
"resources": [],
},
"nodes": [{
"{node}": {
"subscription": {},
# for now just get basic task data - we'll read the logs later
"tasks": [],
"qemu": [{
"{vmid}": {
"snapshot": [],
}
}],
"lxc": [{
"{vmid}": {
"snapshot": [],
}
}],
"version": {},
"time": {},
},
}],
"version": {},
})
LOGGER.info("Fetch and process backup logs..")
logged_backup_data = fetch_backup_data(args, session, data["nodes"])
all_vms = {
str(entry["vmid"]): entry
for entry in data["cluster"]["resources"]
if entry["type"] in ("lxc", "qemu")
}
backup_data = {
# generate list of all VMs IDs - both lxc and qemu
"vmids":
sorted(list(all_vms.keys())),
# look up scheduled backups and extract assigned VMIDs
"scheduled_vmids":
sorted(
list(
set(vmid #
for backup in data["cluster"]["backup"]
if "vmid" in backup and backup["enabled"] == "1"
for vmid in backup["vmid"].split(",")))),
# add data of actually logged VMs
"logged_vmids":
logged_backup_data,
}
node_timezones = {} # Timezones on nodes can be potentially different
snapshot_data = {}
for node in data["nodes"]:
node_timezones[node["node"]] = node["time"]["timezone"]
# only lxc and qemu can have snapshots
for vm in node.get("lxc", []) + node.get("qemu", []):
snapshot_data[str(vm["vmid"])] = {
"snaptimes":
[x["snaptime"] for x in vm["snapshot"] if "snaptime" in x],
}
def date_to_utc(naive_string: str, tz: str) -> str:
"""
Adds timezone information to a date string.
Returns a timezone-aware string
"""
local_tz = pytz.timezone(tz)
timezone_unaware = datetime.strptime(naive_string, "%Y-%m-%d %H:%M:%S")
timezone_aware = local_tz.localize(timezone_unaware)
return timezone_aware.strftime("%Y-%m-%d %H:%M:%S%z")
# overwrite all the start time strings with timezone aware start strings
for vmid in logged_backup_data:
logged_backup_data[vmid]["started_time"] = date_to_utc(
logged_backup_data[vmid]["started_time"],
node_timezones[all_vms[vmid]["node"]])
LOGGER.info("all VMs: %r", backup_data["vmids"])
LOGGER.info("expected backups: %r", backup_data["scheduled_vmids"])
LOGGER.info("actual backups: %r",
sorted(list(logged_backup_data.keys())))
LOGGER.info("snaptimes: %r", snapshot_data)
LOGGER.info("Write agent output..")
for node in data["nodes"]:
assert node["type"] == "node"
piggyback_host = None if args.hostname.startswith(
node["node"] + ".") else node["node"]
with ConditionalPiggybackSection(piggyback_host):
with SectionWriter("proxmox_ve_node_info") as writer:
writer.append_json({
"status":
node["status"],
"lxc": [
vmid for vmid in all_vms
if all_vms[vmid]["type"] == "lxc"
],
"qemu": [
vmid for vmid in all_vms
if all_vms[vmid]["type"] == "qemu"
],
"proxmox_ve_version":
node["version"],
"time_info":
node["time"],
"subscription": {
key: value
for key, value in node["subscription"].items()
if key in {
"status",
"checktime",
"key",
"level",
"nextduedate",
"productname",
"regdate",
}
},
})
with SectionWriter("proxmox_ve_mem_usage") as writer:
writer.append_json({
"mem": node["mem"],
"max_mem": node["maxmem"],
})
with SectionWriter("uptime", separator=None) as writer:
writer.append(node["uptime"])
for vmid, vm in all_vms.items():
with ConditionalPiggybackSection(vm["name"]):
with SectionWriter("proxmox_ve_vm_info") as writer:
writer.append_json({
"vmid": vmid,
"node": vm["node"],
"type": vm["type"],
"status": vm["status"],
"name": vm["name"],
})
if vm["type"] != "qemu":
with SectionWriter("proxmox_ve_disk_usage") as writer:
writer.append_json({
"disk": vm["disk"],
"max_disk": vm["maxdisk"],
})
with SectionWriter("proxmox_ve_mem_usage") as writer:
writer.append_json({
"mem": vm["mem"],
"max_mem": vm["maxmem"],
})
with SectionWriter("proxmox_ve_vm_backup_status") as writer:
writer.append_json({
# todo: info about erroneous backups
"last_backup": logged_backup_data.get(vmid),
})
with SectionWriter("proxmox_ve_vm_snapshot_age") as writer:
writer.append_json(snapshot_data.get(vmid))
def agent_proxmox_ve_main(args: Args) -> None:
"""Fetches and writes selected information formatted as agent output to stdout"""
with ProxmoxVeAPI(
host=args.hostname,
port=args.port,
credentials={
k: getattr(args, k)
for k in {"username", "password"} if getattr(args, k)
},
timeout=args.timeout,
verify_ssl=not args.no_cert_check,
) as session:
LOGGER.info("Fetch general cluster and node information..")
data = session.get_tree({
"cluster": {
"backup": [],
"resources": [],
},
"nodes": [{
"{node}": {
"subscription": {},
# for now just get basic task data - we'll read the logs later
"tasks": [],
"version": {},
},
}],
"version": {},
})
LOGGER.info("Fetch and process backup logs..")
logged_backup_data = fetch_backup_data(args, session, data["nodes"])
all_vms = {
str(entry["vmid"]): entry
for entry in data["cluster"]["resources"]
if entry["type"] in ("lxc", "qemu")
}
backup_data = {
# generate list of all VMs IDs - both lxc and qemu
"vmids":
sorted(list(all_vms.keys())),
# look up scheduled backups and extract assigned VMIDs
"scheduled_vmids":
sorted(
list(
set(vmid #
for backup in data["cluster"]["backup"]
if "vmid" in backup and backup["enabled"] == "1"
for vmid in backup["vmid"].split(",")))),
# add data of actually logged VMs
"logged_vmids":
logged_backup_data,
}
LOGGER.info("all VMs: %r", backup_data["vmids"])
LOGGER.info("expected backups: %r", backup_data["scheduled_vmids"])
LOGGER.info("actual backups: %r",
sorted(list(logged_backup_data.keys())))
LOGGER.info("Write agent output..")
for node in data["nodes"]:
assert node["type"] == "node"
piggyback_host = None if args.hostname.startswith(
node["node"] + ".") else node["node"]
with ConditionalPiggybackSection(piggyback_host):
with SectionWriter("proxmox_ve_node_info") as writer:
writer.append_json({
"status":
node["status"],
"lxc": [
vmid for vmid in all_vms
if all_vms[vmid]["type"] == "lxc"
],
"qemu": [
vmid for vmid in all_vms
if all_vms[vmid]["type"] == "qemu"
],
"proxmox_ve_version":
node["version"],
"subscription": {
key: value
for key, value in node["subscription"].items()
if key in {
"status",
"checktime",
"key",
"level",
"nextduedate",
"productname",
"regdate",
}
},
})
with SectionWriter("proxmox_ve_mem_usage") as writer:
writer.append_json({
"mem": node["mem"],
"max_mem": node["maxmem"],
})
with SectionWriter("uptime", separator=None) as writer:
writer.append(node["uptime"])
for vmid, vm in all_vms.items():
with ConditionalPiggybackSection(vm["name"]):
with SectionWriter("proxmox_ve_vm_info") as writer:
writer.append_json({
"vmid": vmid,
"node": vm["node"],
"type": vm["type"],
"status": vm["status"],
"name": vm["name"],
})
if vm["type"] != "qemu":
with SectionWriter("proxmox_ve_disk_usage") as writer:
writer.append_json({
"disk": vm["disk"],
"max_disk": vm["maxdisk"],
})
with SectionWriter("proxmox_ve_mem_usage") as writer:
writer.append_json({
"mem": vm["mem"],
"max_mem": vm["maxmem"],
})
with SectionWriter("proxmox_ve_vm_backup_status") as writer:
writer.append_json({
# todo: info about erroneous backups
"last_backup": logged_backup_data.get(vmid),
})
def main(args: Optional[List[str]] = None) -> int:
if args is None:
cmk.utils.password_store.replace_passwords()
args = sys.argv[1:]
arguments = parse_arguments(args)
try:
setup_logging(arguments.verbose)
logging.debug("parsed arguments: %s\n", arguments)
with cmk.utils.profile.Profile(enabled=bool(arguments.profile),
profile_file=arguments.profile):
api_client = make_api_client(arguments)
api_server = APIServer.from_kubernetes(api_client)
cluster = Cluster.from_api_server(api_server)
# Sections based on API server
output_cluster_api_sections(cluster)
output_nodes_api_sections(cluster.nodes())
output_deployments_api_sections(cluster.deployments())
output_pods_api_sections(
cluster.pods()) # TODO: make more explicit
# Sections based on cluster agent live data
performance_metrics = collect_metrics_from_cluster_agent(
arguments.cluster_agent_endpoint, arguments.verify_cert)
performance_containers = group_metrics_by_containers(
performance_metrics)
performance_pods = group_containers_by_pods(performance_containers)
uid_piggyback_mappings = map_uid_to_piggyback_host_name(
cluster.pods())
for pod in filter_outdated_pods(list(performance_pods.values()),
uid_piggyback_mappings):
with ConditionalPiggybackSection(
f"pod_{uid_piggyback_mappings[pod.uid]}"):
pod_performance_sections(pod.containers)
for node in cluster.nodes():
with ConditionalPiggybackSection(f"node_{node.name}"):
node_performance_sections([
performance_pods[pod.uid]
for pod in node.pods(phase=api.Phase.RUNNING)
if pod.uid in performance_pods
])
for deployment in cluster.deployments():
with ConditionalPiggybackSection(
f"deployment_{deployment.name(prepend_namespace=True)}"
):
deployment_performance_sections([
performance_pods[pod.uid]
for pod in deployment.pods(phase=api.Phase.RUNNING)
if pod.uid in performance_pods
])
with ConditionalPiggybackSection(
"cluster_kube"): # TODO: make name configurable
cluster_performance_sections([
performance_pods[pod.uid]
for pod in cluster.pods(phase=api.Phase.RUNNING)
if pod.uid in performance_pods
])
# TODO: handle pods with no performance data (pod.uid not in performance pods)
except urllib3.exceptions.MaxRetryError as e:
if arguments.debug:
raise
if isinstance(e.reason, urllib3.exceptions.NewConnectionError):
sys.stderr.write(
"Failed to establish a connection to %s:%s at URL %s" %
(e.pool.host, e.pool.port, e.url))
else:
sys.stderr.write("%s" % e)
return 1
except Exception as e:
if arguments.debug:
raise
sys.stderr.write("%s" % e)
return 1
return 0