async def node_report(pods=None, userid=1000):
"""Print a report of suspicious processes on a single node"""
if pods is None:
pods = pods_by_uid(await in_pool(get_pods))
procs = await in_pool(lambda: get_procs(userid))
print(f"Total processes for {hostname}: {len(procs)}\n", end="")
pods, procs_without_pod = associate_pods_procs(pods, procs)
# inspect all procs in our pods
user_procs = []
for pod in pods.values():
user_procs.extend(pod["minesweeper"]["procs"])
pod["minesweeper"]["procs"] = [
inspect_process(p) for p in pod["minesweeper"]["procs"]
]
print(f"Total user pods for {hostname}: {len(pods)}\n", end="")
print(f"Total user processes for {hostname}: {len(user_procs)}\n", end="")
suspicious_pods = [
pod for pod in pods.values() if inspect_pod(pod)["suspicious"]
]
print(f"Pods of interest for {hostname}: {len(suspicious_pods)}")
# report on all suspicious pods
report_futures = []
for pod in suspicious_pods:
fut = asyncio.ensure_future(report_pod(pod))
report_futures.append(fut)
await asyncio.sleep(0)
# report on suspicious processes with no matching pod
suspicious_procs_without_pod = []
if config["inspect_procs_without_pod"]:
procs_without_pod = [inspect_process(p) for p in procs_without_pod]
suspicious_procs_without_pod = [
p for p in procs_without_pod if p.suspicious
]
if suspicious_procs_without_pod:
print(
f"No pods found for {len(suspicious_procs_without_pod)} suspicious processes on {hostname}:"
)
for proc in suspicious_procs_without_pod:
print(f" {proc.pid}: {proc.cmd}")
if report_futures:
await asyncio.gather(*report_futures)
# finally, terminate pods that meet the immediate termination condition
pods_to_terminate = [
pod for pod in suspicious_pods
if pod["minesweeper"]["should_terminate"]
]
if pods_to_terminate:
terminate_futures = [
in_pool(partial(terminate_pod, pod)) for pod in pods_to_terminate
]
await asyncio.gather(*terminate_futures)
async def node_report(pods=None, userid=1000):
"""Print a report of suspicious processes on a single node"""
if pods is None:
pods = pods_by_uid(await in_pool(get_pods))
procs = await in_pool(lambda: get_procs(userid))
print(f"Total processes for {hostname}: {len(procs)}\n", end="")
pods, procs_without_pod = associate_pods_procs(pods, procs)
# inspect all procs in our pods
user_procs = []
for pod in pods.values():
user_procs.extend(pod["minesweeper"]["procs"])
pod["minesweeper"]["procs"] = [
inspect_process(p) for p in pod["minesweeper"]["procs"]
]
print(f"Total user pods for {hostname}: {len(pods)}\n", end="")
print(f"Total user processes for {hostname}: {len(user_procs)}\n", end="")
suspicious_pods = [
pod for pod in pods.values() if inspect_pod(pod)["suspicious"]
]
print(f"Pods of interest for {hostname}: {len(suspicious_pods)}")
# report on all suspicious pods
report_futures = []
for pod in suspicious_pods:
fut = asyncio.ensure_future(report_pod(pod))
report_futures.append(fut)
await asyncio.sleep(0)
# report on suspicious processes with no matching pod
suspicious_procs_without_pod = []
if config["inspect_procs_without_pod"]:
procs_without_pod = [inspect_process(p) for p in procs_without_pod]
suspicious_procs_without_pod = [
p for p in procs_without_pod if p.suspicious
]
if suspicious_procs_without_pod:
print(
f"No pods found for {len(suspicious_procs_without_pod)} suspicious processes on {hostname}:"
)
for proc in suspicious_procs_without_pod:
print(f" {proc.pid}: {proc.cmd}")
# report on suspicious dind processes
if config["inspect_dind"]:
dind_procs = [inspect_process(p) for p in get_dind_procs()]
print(f"Total dind processes for {hostname}: {len(dind_procs)}")
for proc in dind_procs:
if proc.should_terminate:
print(f"dind process should terminate: {proc}")
try:
os.kill(proc.pid, signal.SIGKILL)
except OSError as e:
print(f"Failed to kill {proc}: {e}")
elif proc.suspicious:
print(f"dind process is suspicious: {proc}")
# TODO: find a way to identity the build repo responsible for suspicious processes in dind
# suspicious_dind_procs_without_pod = [
# p for p in procs_without_pod if p.suspicious
# ]
if report_futures:
await asyncio.gather(*report_futures)
# finally, terminate pods that meet the immediate termination condition
pods_to_terminate = [
pod for pod in suspicious_pods
if pod["minesweeper"]["should_terminate"]
]
if pods_to_terminate:
terminate_futures = [
in_pool(partial(terminate_pod, pod)) for pod in pods_to_terminate
]
await asyncio.gather(*terminate_futures)