def resetScanConfiguration(self):
if not os.path.exists(self.isolation_path):
os.mkdir(self.isolation_path)
whitelist = Kdatabase().get_obj('virus_whitelist')
virus = Kdatabase().get_obj('virus')
self.white_list = whitelist
self.avoid_list.append(self.isolation_path)
if virus['finished']:
self.current_path = ''
self.searched_count = 0
virus['lastScanedPath'] = self.current_path
virus['searchedCount'] = self.searched_count
else:
self.current_path = virus['lastScanedPath']
self.searched_count = virus['searchedCount']
if self.current_path:
self.isNeedIgnore = True
virus['finished'] = 0
virus['lasttime'] = time_op.now()
Kdatabase().dump('virus')
def publish(self, timeline, kind, des, root_cause, suggestion, activities,
icon, introduction, score):
ueba = Kdatabase().get_obj("ueba")
threat_id = Krandom().purely(16)
threat = {
"kind": kind,
"timeline": timeline,
"resolved": False,
"resolved_ts": -1,
"unread": True,
"read_ts": -1,
"description": des,
"root_cause": root_cause,
"suggestion": suggestion,
"activities": activities,
"icon": icon,
"introduction": introduction,
"ts": time_op.now(),
"score": score
}
ueba["lasttime"] = time_op.now()
ueba["storys"][threat_id] = threat
Kdatabase().dump("ueba")
threat["description"] = Klanguage().decode_ts(threat["description"])
threat["root_cause"] = Klanguage().decode_ts(threat["root_cause"])
threat["suggestion"] = Klanguage().decode_ts(threat["suggestion"])
KCybertek().publish_threat(threat_id, threat)
print(threat)
print("KUEBA timeline add one!")
def run(payload, socket):
response = {
"cmd_id" : payload["cmd_id"],
"session_id" : payload["args"]["session_id"],
"error" : ""
}
ueba = Kdatabase().get_obj("ueba")
for key, story in ueba["storys"].items():
if key == payload["args"]["key"]:
response["ueba"] = {
"key" : key,
"description" : Klanguage().decode_ts(story["description"]),
"root_cause" : Klanguage().decode_ts(story["root_cause"]),
"suggestion" : Klanguage().decode_ts(story["suggestion"]),
"timeline" : translate(story["timeline"]),
"unread" : story["unread"],
"resolved" : story["resolved"],
"resolved_ts" : story["resolved_ts"]
}
story["unread"] = False
story["read_ts"] = time_op.now()
Kdatabase().dump("ueba")
break
socket.response(response)
def runVirusScan(self, root_paths):
retry_list = []
self.resetScanConfiguration()
for filepath in self.retrieveFiles(root_paths):
sha256 = file_op.sha256_checksum(filepath)
ret = KCybertek().detect_file_sha256(sha256)
if not ret:
continue
ret = json.loads(ret)
data = ret["data"]
if data:
if data['isVirus']:
self.handleVirus(filepath, data['virusType'],
data['level'], sha256)
if data['isNeedUpload']:
self.uploadFile(filepath)
# when we finish scan,set "finished" and "lastScanedPath"
self.current_path = ''
virus = Kdatabase().get_obj('virus')
virus['finished'] = 1
virus['lastScanedPath'] = self.current_path
virus['searchedCount'] = self.searched_count
virus['lasttime'] = time_op.now()
Kdatabase().dump('virus')
def run(payload, socket):
response = {
"cmd_id": payload["cmd_id"],
"session_id": payload["args"]["session_id"],
"error": ""
}
ueba = Kdatabase().get_obj("ueba")
marked = payload["args"]["marked"]
for key, story in ueba["storys"].items():
if key == payload["args"]["key"]:
if marked:
story["resolved"] = True
story["resolved_ts"] = time_op.now()
print("marked key:" + key)
else:
story["resolved"] = False
story["resolved_ts"] = -1
print("unmarked key:" + key)
response["resolved"] = story["resolved"]
response["resolved_ts"] = story["resolved_ts"]
Kdatabase().dump("ueba")
break
socket.response(response)
def run(payload, socket):
response = {
"cmd_id": payload["cmd_id"],
"session_id": payload["args"]["session_id"],
"error": ""
}
baseline = Kdatabase().get_obj("baseline")
risk_id = payload["args"]["risk_id"]
verified_1 = 0
verified_2 = 0
for i in baseline["risks"]:
if baseline["risks"][i]["stage"] == macro.BASELINE_STAGE["VERIFIED"]:
verified_1 += 1
for i in risk_id:
Kbaseline().verify_all(i)
for i in baseline["risks"]:
if baseline["risks"][i]["stage"] == macro.BASELINE_STAGE["VERIFIED"]:
verified_2 += 1
if verified_2 > verified_1:
pass
else:
response["error"] = "No baseline verified"
Kdatabase().dump('baseline')
socket.response(response)
def run(payload, socket):
if common.is_linux():
vuls = Kdatabase().get_obj("vuls")
while True:
if Kevent().is_terminate():
print("vulscan thread terminate")
break
now = time_op.now()
settings = Kdatabase().get_obj('strategy')
if settings:
if now > vuls["lasttime"] + settings["vuls"]["period"]:
with Kvuls().get_lock():
Kvuls().vulscan()
vuls["lasttime"] = now
if settings["vuls"]["auto_repair"]:
if common.is_program_running(
"apt-get") or common.is_program_running(
"yum"):
pass
else:
if lib.check_root():
for package in list(vuls["items"]):
Kvuls().repair(package)
Kdatabase().dump("vuls")
time.sleep(5)
def clean(items, socket, session_id):
response = {
"cmd_id": "1044",
"session_id": session_id,
"kind": None,
"size": 0,
"id": None
}
total_size = 0
total_failed = 0
if len(items) > 0:
cleaner = Kdatabase().get_obj("cleaner")
for i in items:
for kind, info in cleaner["kinds"].items():
if i in list(info["items"].keys()):
ret = Kcleaner().clean_option(info, i, cleaner["record"])
total_size += ret[0]
total_failed += ret[1]
if ret[1] == 0:
response["kind"] = kind
response["size"] = info["size"]
response["id"] = i
socket.response(response)
break
if total_size > 0:
Kdatabase().dump("cleaner")
return total_failed, total_size
def run(payload, socket):
cleaner = Kdatabase().get_obj("cleaner")
Kcleaner().load_jsons()
while True:
if Kevent().is_terminate():
print("cleaner thread terminate")
break
now = time_op.now()
settings = Kdatabase().get_obj('strategy')
if settings:
if now > cleaner["lasttime"] + settings["garbage"]["period"]:
with Kcleaner().get_lock():
cleaner["kinds"] = Kcleaner().scan()
cleaner["lasttime"] = now
if settings["garbage"]["auto_clean"]:
for kind, info in cleaner["kinds"].items():
if "autoclean" in info and info["autoclean"]:
for i in list(info["items"].keys()):
Kcleaner().clean_option(
info, i, cleaner["record"])
Kdatabase().dump("cleaner")
time.sleep(5)
def clearHistory(self):
virus = Kdatabase().get_obj('virus')
for key in virus['handledList']:
if virus['handledList'][key][-1] == OPERATION_TRUST:
virus['handledList'][key][-1] = RESERVERED_WHEN_CLEARED_HISTORY
else:
virus['handledList'].pop(key)
virus['untrustList'] = {}
Kdatabase().dump('virus')
def synchronizeProcessing(self):
virus = Kdatabase().get_obj('virus')
virus['lasttime'] = time_op.now()
if self.current_path:
virus['lastScanedPath'] = self.current_path
if self.searched_count:
virus['searchedCount'] = self.searched_count
Kdatabase().dump('virus')
def delWhiteList(self, filepath):
md5 = hashlib.md5()
md5.update(filepath.encode('utf-8'))
path = os.path.join(self.isolation_path, md5.hexdigest())
whitelist = Kdatabase().get_obj('virus_whitelist')
opsTime = '{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())
if filepath in whitelist:
whitelist.pop(filepath)
self.white_list = whitelist
Kdatabase().dump('virus_whitelist')
def run(payload, socket):
if common.is_linux():
data = {
"kernel": [],
"authentication": [],
"feature": [],
}
audit = Kdatabase().get_obj('audit')
while True:
if Kevent().is_terminate():
print("security audit thread terminate")
break
# reset data
data['kernel'] = []
data['authentication'] = []
data['feature'] = []
data['statistic'] = {"critical": 0, "warning": 0}
now = time_op.now()
if now > audit["lasttime"] + constant.AUDIT_SCAN_PERIOD:
#check_upgradable_packages(data)
audit_implement.kernel_available_version(data)
audit_implement.enum_kernel_modules(data)
audit_implement.check_magickey_configuration(data)
audit_implement.check_need_reboot(data)
audit_implement.check_coredump_config(data)
#check_kdump_config(data)
#kernel_default_limits(data)
audit_implement.security_info(data)
audit_implement.get_useradd_list(data)
audit_implement.logged_user(data)
audit_implement.check_sudoers_file(data)
audit_implement.login_defs_policy(data)
audit['feature'] = data['feature']
audit['authentication'] = data['authentication']
audit['kernel'] = data['kernel']
audit['statistic'] = data['statistic']
audit["lasttime"] = now
Kdatabase().dump('audit')
time.sleep(5)
def update_record(self, package):
vuls = Kdatabase().get_obj("vuls")
if package in vuls["items"]:
info = vuls["items"][package]
today = time_op.get_last_day()
if len(vuls["record"]) > 0:
last_record = vuls["record"][-1]
lastday = time_op.get_last_day(last_record["time"])
if today == lastday:
last_record["cves"].extend(info["cves"])
else:
vuls["record"].append({
"time": today,
"cves": info["cves"]
})
else:
vuls["record"].append({"time": today, "cves": info["cves"]})
vuls["repaired_packages"].append({
"name": package,
"installed": info["installed"]
})
del vuls["items"][package]
def start(self):
while True:
try:
username = Kdatabase().get_obj("setting")["username"]
host, port = Khttp().get_connection(constant.SERVER_URL,
username)
if host and port:
Kevent().do_unpack()
self.socket = Ksocket(host, port, username)
self.socket.start()
self.socket.loop()
else:
Klogger().info(
"Reconnect to {} after 5s".format(marsnake_server))
time.sleep(5)
continue
except Exception as e:
Klogger().exception()
if self.socket:
self.socket.close()
self.socket = None
time.sleep(10)
def run(payload, socket):
response = {
"cmd_id": payload["cmd_id"],
"session_id": payload["args"]["session_id"],
"cpu": 0,
"memory": 0,
"disk": {
"read": 0,
"write": 0
},
"network": {
"tx": 0,
"rx": 0
},
"error": ""
}
monitor_second = Kdatabase().get_monitor_second()
cpu_usage(response, monitor_second)
ram_usage(response, monitor_second)
disk_usage(response, monitor_second)
network_usage(response, monitor_second)
socket.response(response)
def vulscan(self):
self.reset()
distro, distro_release = lib.detect_distribution()
yum = common.check_programs_installed("yum")
apt = common.check_programs_installed("apt-get")
if yum:
self.redhat_get_installed_packages()
if "centos" in distro.lower():
self.redhat_centos_checkupdate()
self.redhat_centos_vulscan()
else:
self.redhat_common_checkupdate()
self.redhat_common_vulscan()
if apt:
self.debian_update_packages()
self.debian_check_dependencies()
self.debian_get_installed_packages()
self.debian_scan_cveid_from_changelog()
vuls = Kdatabase().get_obj("vuls")
vuls["items"] = self.vuls
def run(payload, socket):
response = {
"cmd_id": payload["cmd_id"],
"session_id": payload["args"]["session_id"],
"check_resolved": payload["args"]["check_resolved"],
"error": ""
}
ueba = Kdatabase().get_obj("ueba")
kind = [{} for x in range(macro.STORY_KIND["MAX"])]
check_resolved = payload["args"]["check_resolved"]
for key, story in ueba["storys"].items():
if check_resolved == story["resolved"]:
resolved_ts = -1
if story["resolved"]:
resolved_ts = story["resolved_ts"]
kind[story["kind"]][key] = {
"distro": common.get_distribution(),
"hostname": platform.node(),
"icon": story["icon"],
"activities": story["activities"],
"introduction": translate(story["introduction"]),
"ts": story["ts"],
"resolved": story["resolved"],
"resolved_ts": resolved_ts
}
response["kind"] = kind
socket.response(response)
def run(payload, socket):
response = {
"cmd_id": payload["cmd_id"],
"session_id": payload["args"]["session_id"],
"error": ""
}
if payload["args"]["user_id"] != Kdatabase().get_obj(
'setting')["username"]:
response["error"] = Klanguage().to_ts(4007)
socket.response(response)
return
with Kcleaner().get_lock():
total_failed, total_size = clean(payload["args"]["items"], socket,
payload["args"]["session_id"])
if total_failed + total_size > 0:
prompt = "{} {}".format(common.size_human_readable(total_size),
Klanguage().to_ts(1830))
'''
if total_failed > 0:
response["error"] = "{}, {}".format(prompt, Klanguage().to_ts(1831))
else:
response["prompt"] = prompt
'''
response["prompt"] = prompt
else:
response["error"] = Klanguage().to_ts(4006)
socket.response(response)
def run(payload, socket):
response = {
'cmd_id': payload['cmd_id'],
"session_id": payload["args"]["session_id"],
"items": None,
"lasttime": 0,
'error': ""
}
vuls = Kdatabase().get_obj("vuls")
last_8week_ts = time_op.get_last_nmonday_ts(8)
history = list(range(8))
for i in range(len(last_8week_ts)):
history[i] = {"time": last_8week_ts[i], "cves": []}
for record in vuls["record"]:
for value in history:
if record["time"] >= value["time"]:
value["cves"].extend(record["cves"])
value["cves"] = list(set(value["cves"]))
break
history = sorted(history, key=lambda t: t["time"])
response["items"] = vuls["items"]
response["records"] = history
response["lasttime"] = vuls["lasttime"]
socket.response(response)
def runScannerCron(self):
time.sleep(30)
while True:
virus = Kdatabase().get_obj('virus')
settings = Kdatabase().get_obj('strategy')
if settings:
if virus['finished']:
if not len(virus['isolateList']):
if time_op.now() >= (virus['lasttime'] +
settings["virus"]["period"]):
self.runDeepScan()
else:
self.runDeepScan()
time.sleep(60 * 60) # wait for 1 hour to ask
def get_history(self):
ueba = Kdatabase().get_obj("ueba")
count = 0
for key, story in ueba["storys"].items():
if story["resolved"]:
count += 1
return count
def get_history(self):
cleaner = Kdatabase().get_obj("cleaner")
record = cleaner["record"]
total_size = 0
for i in record:
total_size += i["size"]
return total_size
def check_loop(self):
while True:
baseline = Kdatabase().get_obj('baseline')
settings = Kdatabase().get_obj('strategy')
now = time_op.now()
if settings:
if now > (baseline['lasttime'] + settings["audit"]["period"]):
items = settings["audit"]["items"]
for i in items:
if items[i] == True and i in self.maps:
self.maps[i].do(None)
baseline["lasttime"] = now
Kdatabase().dump('baseline')
time.sleep(5)
def run(payload, socket):
response = {
"cmd_id": payload["cmd_id"],
"session_id": payload["args"]["session_id"],
"error": ""
}
fingerprint = Kdatabase().get_obj("fingerprint")
change = fingerprint["port"]["change"]
change_id = payload["args"]["change_id"]
if change_id in change:
del change[change_id]
response["change_id"] = change_id
Kdatabase().dump("fingerprint")
else:
response["error"] = "Change ID Not Found"
socket.response(response)
def get_garbage(response):
cleaner = Kdatabase().get_obj("cleaner")
for kind, info in cleaner["kinds"].items():
size = info["size"]
if size > 0:
response["garbage"].append({
"name" : Klanguage().to_ts(info["name"]),
"size" : info["size"]
})
def run(payload, socket):
response = {
"cmd_id": payload["cmd_id"],
"session_id": payload["args"]["session_id"],
"key": None,
"error": ""
}
unlucky = payload["args"]["key"]
ueba = Kdatabase().get_obj("ueba")
for key, story in ueba["storys"].items():
if story["resolved"] and key == unlucky:
del ueba["storys"][unlucky]
response["key"] = unlucky
Kdatabase().dump("ueba")
break
socket.response(response)
def run(payload, socket):
from core.fingerprint import Kfingerprint
fingerprint = Kdatabase().get_obj("fingerprint")
while True:
if Kevent().is_terminate():
print("fingerprint thread terminate")
break
now = time_op.now()
settings = Kdatabase().get_obj('strategy')
if settings:
if now > fingerprint["port"]["lasttime"] + settings["asset"]["port_scan"]:
Kfingerprint().record_listening_port()
if now > fingerprint["account"]["lasttime"] + settings["asset"]["account_scan"]:
Kfingerprint().record_account()
time.sleep(5)
def run(payload, socket):
response = {
"cmd_id": payload["cmd_id"],
"session_id": payload["args"]["session_id"],
"error": ""
}
baseline = Kdatabase().get_obj("baseline")
risk_id = payload["args"]["risk_id"]
if len(risk_id) > 0:
for i in risk_id:
baseline["risks"][i]["stage"] = macro.BASELINE_STAGE["IGNORED"]
baseline["risks"][i]["handle_ts"] = time_op.now()
Kdatabase().dump('baseline')
else:
response["error"] = "Nothing to ignore"
socket.response(response)
def get_warning_and_score(self):
cleaner = Kdatabase().get_obj("cleaner")
kinds_db = cleaner['kinds']
totalsize = 0
for i,item in kinds_db.items():
totalsize += item['size']
garbage_128M = (totalsize >> 27)
lost_score = lost_score_correction(garbage_128M * 5, self.coefficient)
return "cleaner", totalsize, lost_score, cleaner["lasttime"]