def resetScanConfiguration(self): if not os.path.exists(self.isolation_path): os.mkdir(self.isolation_path) whitelist = Kdatabase().get_obj('virus_whitelist') virus = Kdatabase().get_obj('virus') self.white_list = whitelist self.avoid_list.append(self.isolation_path) if virus['finished']: self.current_path = '' self.searched_count = 0 virus['lastScanedPath'] = self.current_path virus['searchedCount'] = self.searched_count else: self.current_path = virus['lastScanedPath'] self.searched_count = virus['searchedCount'] if self.current_path: self.isNeedIgnore = True virus['finished'] = 0 virus['lasttime'] = time_op.now() Kdatabase().dump('virus')
def publish(self, timeline, kind, des, root_cause, suggestion, activities, icon, introduction, score): ueba = Kdatabase().get_obj("ueba") threat_id = Krandom().purely(16) threat = { "kind": kind, "timeline": timeline, "resolved": False, "resolved_ts": -1, "unread": True, "read_ts": -1, "description": des, "root_cause": root_cause, "suggestion": suggestion, "activities": activities, "icon": icon, "introduction": introduction, "ts": time_op.now(), "score": score } ueba["lasttime"] = time_op.now() ueba["storys"][threat_id] = threat Kdatabase().dump("ueba") threat["description"] = Klanguage().decode_ts(threat["description"]) threat["root_cause"] = Klanguage().decode_ts(threat["root_cause"]) threat["suggestion"] = Klanguage().decode_ts(threat["suggestion"]) KCybertek().publish_threat(threat_id, threat) print(threat) print("KUEBA timeline add one!")
def run(payload, socket): response = { "cmd_id" : payload["cmd_id"], "session_id" : payload["args"]["session_id"], "error" : "" } ueba = Kdatabase().get_obj("ueba") for key, story in ueba["storys"].items(): if key == payload["args"]["key"]: response["ueba"] = { "key" : key, "description" : Klanguage().decode_ts(story["description"]), "root_cause" : Klanguage().decode_ts(story["root_cause"]), "suggestion" : Klanguage().decode_ts(story["suggestion"]), "timeline" : translate(story["timeline"]), "unread" : story["unread"], "resolved" : story["resolved"], "resolved_ts" : story["resolved_ts"] } story["unread"] = False story["read_ts"] = time_op.now() Kdatabase().dump("ueba") break socket.response(response)
def runVirusScan(self, root_paths): retry_list = [] self.resetScanConfiguration() for filepath in self.retrieveFiles(root_paths): sha256 = file_op.sha256_checksum(filepath) ret = KCybertek().detect_file_sha256(sha256) if not ret: continue ret = json.loads(ret) data = ret["data"] if data: if data['isVirus']: self.handleVirus(filepath, data['virusType'], data['level'], sha256) if data['isNeedUpload']: self.uploadFile(filepath) # when we finish scan,set "finished" and "lastScanedPath" self.current_path = '' virus = Kdatabase().get_obj('virus') virus['finished'] = 1 virus['lastScanedPath'] = self.current_path virus['searchedCount'] = self.searched_count virus['lasttime'] = time_op.now() Kdatabase().dump('virus')
def run(payload, socket): response = { "cmd_id": payload["cmd_id"], "session_id": payload["args"]["session_id"], "error": "" } ueba = Kdatabase().get_obj("ueba") marked = payload["args"]["marked"] for key, story in ueba["storys"].items(): if key == payload["args"]["key"]: if marked: story["resolved"] = True story["resolved_ts"] = time_op.now() print("marked key:" + key) else: story["resolved"] = False story["resolved_ts"] = -1 print("unmarked key:" + key) response["resolved"] = story["resolved"] response["resolved_ts"] = story["resolved_ts"] Kdatabase().dump("ueba") break socket.response(response)
def run(payload, socket): response = { "cmd_id": payload["cmd_id"], "session_id": payload["args"]["session_id"], "error": "" } baseline = Kdatabase().get_obj("baseline") risk_id = payload["args"]["risk_id"] verified_1 = 0 verified_2 = 0 for i in baseline["risks"]: if baseline["risks"][i]["stage"] == macro.BASELINE_STAGE["VERIFIED"]: verified_1 += 1 for i in risk_id: Kbaseline().verify_all(i) for i in baseline["risks"]: if baseline["risks"][i]["stage"] == macro.BASELINE_STAGE["VERIFIED"]: verified_2 += 1 if verified_2 > verified_1: pass else: response["error"] = "No baseline verified" Kdatabase().dump('baseline') socket.response(response)
def run(payload, socket): if common.is_linux(): vuls = Kdatabase().get_obj("vuls") while True: if Kevent().is_terminate(): print("vulscan thread terminate") break now = time_op.now() settings = Kdatabase().get_obj('strategy') if settings: if now > vuls["lasttime"] + settings["vuls"]["period"]: with Kvuls().get_lock(): Kvuls().vulscan() vuls["lasttime"] = now if settings["vuls"]["auto_repair"]: if common.is_program_running( "apt-get") or common.is_program_running( "yum"): pass else: if lib.check_root(): for package in list(vuls["items"]): Kvuls().repair(package) Kdatabase().dump("vuls") time.sleep(5)
def clean(items, socket, session_id): response = { "cmd_id": "1044", "session_id": session_id, "kind": None, "size": 0, "id": None } total_size = 0 total_failed = 0 if len(items) > 0: cleaner = Kdatabase().get_obj("cleaner") for i in items: for kind, info in cleaner["kinds"].items(): if i in list(info["items"].keys()): ret = Kcleaner().clean_option(info, i, cleaner["record"]) total_size += ret[0] total_failed += ret[1] if ret[1] == 0: response["kind"] = kind response["size"] = info["size"] response["id"] = i socket.response(response) break if total_size > 0: Kdatabase().dump("cleaner") return total_failed, total_size
def run(payload, socket): cleaner = Kdatabase().get_obj("cleaner") Kcleaner().load_jsons() while True: if Kevent().is_terminate(): print("cleaner thread terminate") break now = time_op.now() settings = Kdatabase().get_obj('strategy') if settings: if now > cleaner["lasttime"] + settings["garbage"]["period"]: with Kcleaner().get_lock(): cleaner["kinds"] = Kcleaner().scan() cleaner["lasttime"] = now if settings["garbage"]["auto_clean"]: for kind, info in cleaner["kinds"].items(): if "autoclean" in info and info["autoclean"]: for i in list(info["items"].keys()): Kcleaner().clean_option( info, i, cleaner["record"]) Kdatabase().dump("cleaner") time.sleep(5)
def clearHistory(self): virus = Kdatabase().get_obj('virus') for key in virus['handledList']: if virus['handledList'][key][-1] == OPERATION_TRUST: virus['handledList'][key][-1] = RESERVERED_WHEN_CLEARED_HISTORY else: virus['handledList'].pop(key) virus['untrustList'] = {} Kdatabase().dump('virus')
def synchronizeProcessing(self): virus = Kdatabase().get_obj('virus') virus['lasttime'] = time_op.now() if self.current_path: virus['lastScanedPath'] = self.current_path if self.searched_count: virus['searchedCount'] = self.searched_count Kdatabase().dump('virus')
def delWhiteList(self, filepath): md5 = hashlib.md5() md5.update(filepath.encode('utf-8')) path = os.path.join(self.isolation_path, md5.hexdigest()) whitelist = Kdatabase().get_obj('virus_whitelist') opsTime = '{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) if filepath in whitelist: whitelist.pop(filepath) self.white_list = whitelist Kdatabase().dump('virus_whitelist')
def run(payload, socket): if common.is_linux(): data = { "kernel": [], "authentication": [], "feature": [], } audit = Kdatabase().get_obj('audit') while True: if Kevent().is_terminate(): print("security audit thread terminate") break # reset data data['kernel'] = [] data['authentication'] = [] data['feature'] = [] data['statistic'] = {"critical": 0, "warning": 0} now = time_op.now() if now > audit["lasttime"] + constant.AUDIT_SCAN_PERIOD: #check_upgradable_packages(data) audit_implement.kernel_available_version(data) audit_implement.enum_kernel_modules(data) audit_implement.check_magickey_configuration(data) audit_implement.check_need_reboot(data) audit_implement.check_coredump_config(data) #check_kdump_config(data) #kernel_default_limits(data) audit_implement.security_info(data) audit_implement.get_useradd_list(data) audit_implement.logged_user(data) audit_implement.check_sudoers_file(data) audit_implement.login_defs_policy(data) audit['feature'] = data['feature'] audit['authentication'] = data['authentication'] audit['kernel'] = data['kernel'] audit['statistic'] = data['statistic'] audit["lasttime"] = now Kdatabase().dump('audit') time.sleep(5)
def update_record(self, package): vuls = Kdatabase().get_obj("vuls") if package in vuls["items"]: info = vuls["items"][package] today = time_op.get_last_day() if len(vuls["record"]) > 0: last_record = vuls["record"][-1] lastday = time_op.get_last_day(last_record["time"]) if today == lastday: last_record["cves"].extend(info["cves"]) else: vuls["record"].append({ "time": today, "cves": info["cves"] }) else: vuls["record"].append({"time": today, "cves": info["cves"]}) vuls["repaired_packages"].append({ "name": package, "installed": info["installed"] }) del vuls["items"][package]
def start(self): while True: try: username = Kdatabase().get_obj("setting")["username"] host, port = Khttp().get_connection(constant.SERVER_URL, username) if host and port: Kevent().do_unpack() self.socket = Ksocket(host, port, username) self.socket.start() self.socket.loop() else: Klogger().info( "Reconnect to {} after 5s".format(marsnake_server)) time.sleep(5) continue except Exception as e: Klogger().exception() if self.socket: self.socket.close() self.socket = None time.sleep(10)
def run(payload, socket): response = { "cmd_id": payload["cmd_id"], "session_id": payload["args"]["session_id"], "cpu": 0, "memory": 0, "disk": { "read": 0, "write": 0 }, "network": { "tx": 0, "rx": 0 }, "error": "" } monitor_second = Kdatabase().get_monitor_second() cpu_usage(response, monitor_second) ram_usage(response, monitor_second) disk_usage(response, monitor_second) network_usage(response, monitor_second) socket.response(response)
def vulscan(self): self.reset() distro, distro_release = lib.detect_distribution() yum = common.check_programs_installed("yum") apt = common.check_programs_installed("apt-get") if yum: self.redhat_get_installed_packages() if "centos" in distro.lower(): self.redhat_centos_checkupdate() self.redhat_centos_vulscan() else: self.redhat_common_checkupdate() self.redhat_common_vulscan() if apt: self.debian_update_packages() self.debian_check_dependencies() self.debian_get_installed_packages() self.debian_scan_cveid_from_changelog() vuls = Kdatabase().get_obj("vuls") vuls["items"] = self.vuls
def run(payload, socket): response = { "cmd_id": payload["cmd_id"], "session_id": payload["args"]["session_id"], "check_resolved": payload["args"]["check_resolved"], "error": "" } ueba = Kdatabase().get_obj("ueba") kind = [{} for x in range(macro.STORY_KIND["MAX"])] check_resolved = payload["args"]["check_resolved"] for key, story in ueba["storys"].items(): if check_resolved == story["resolved"]: resolved_ts = -1 if story["resolved"]: resolved_ts = story["resolved_ts"] kind[story["kind"]][key] = { "distro": common.get_distribution(), "hostname": platform.node(), "icon": story["icon"], "activities": story["activities"], "introduction": translate(story["introduction"]), "ts": story["ts"], "resolved": story["resolved"], "resolved_ts": resolved_ts } response["kind"] = kind socket.response(response)
def run(payload, socket): response = { "cmd_id": payload["cmd_id"], "session_id": payload["args"]["session_id"], "error": "" } if payload["args"]["user_id"] != Kdatabase().get_obj( 'setting')["username"]: response["error"] = Klanguage().to_ts(4007) socket.response(response) return with Kcleaner().get_lock(): total_failed, total_size = clean(payload["args"]["items"], socket, payload["args"]["session_id"]) if total_failed + total_size > 0: prompt = "{} {}".format(common.size_human_readable(total_size), Klanguage().to_ts(1830)) ''' if total_failed > 0: response["error"] = "{}, {}".format(prompt, Klanguage().to_ts(1831)) else: response["prompt"] = prompt ''' response["prompt"] = prompt else: response["error"] = Klanguage().to_ts(4006) socket.response(response)
def run(payload, socket): response = { 'cmd_id': payload['cmd_id'], "session_id": payload["args"]["session_id"], "items": None, "lasttime": 0, 'error': "" } vuls = Kdatabase().get_obj("vuls") last_8week_ts = time_op.get_last_nmonday_ts(8) history = list(range(8)) for i in range(len(last_8week_ts)): history[i] = {"time": last_8week_ts[i], "cves": []} for record in vuls["record"]: for value in history: if record["time"] >= value["time"]: value["cves"].extend(record["cves"]) value["cves"] = list(set(value["cves"])) break history = sorted(history, key=lambda t: t["time"]) response["items"] = vuls["items"] response["records"] = history response["lasttime"] = vuls["lasttime"] socket.response(response)
def runScannerCron(self): time.sleep(30) while True: virus = Kdatabase().get_obj('virus') settings = Kdatabase().get_obj('strategy') if settings: if virus['finished']: if not len(virus['isolateList']): if time_op.now() >= (virus['lasttime'] + settings["virus"]["period"]): self.runDeepScan() else: self.runDeepScan() time.sleep(60 * 60) # wait for 1 hour to ask
def get_history(self): ueba = Kdatabase().get_obj("ueba") count = 0 for key, story in ueba["storys"].items(): if story["resolved"]: count += 1 return count
def get_history(self): cleaner = Kdatabase().get_obj("cleaner") record = cleaner["record"] total_size = 0 for i in record: total_size += i["size"] return total_size
def check_loop(self): while True: baseline = Kdatabase().get_obj('baseline') settings = Kdatabase().get_obj('strategy') now = time_op.now() if settings: if now > (baseline['lasttime'] + settings["audit"]["period"]): items = settings["audit"]["items"] for i in items: if items[i] == True and i in self.maps: self.maps[i].do(None) baseline["lasttime"] = now Kdatabase().dump('baseline') time.sleep(5)
def run(payload, socket): response = { "cmd_id": payload["cmd_id"], "session_id": payload["args"]["session_id"], "error": "" } fingerprint = Kdatabase().get_obj("fingerprint") change = fingerprint["port"]["change"] change_id = payload["args"]["change_id"] if change_id in change: del change[change_id] response["change_id"] = change_id Kdatabase().dump("fingerprint") else: response["error"] = "Change ID Not Found" socket.response(response)
def get_garbage(response): cleaner = Kdatabase().get_obj("cleaner") for kind, info in cleaner["kinds"].items(): size = info["size"] if size > 0: response["garbage"].append({ "name" : Klanguage().to_ts(info["name"]), "size" : info["size"] })
def run(payload, socket): response = { "cmd_id": payload["cmd_id"], "session_id": payload["args"]["session_id"], "key": None, "error": "" } unlucky = payload["args"]["key"] ueba = Kdatabase().get_obj("ueba") for key, story in ueba["storys"].items(): if story["resolved"] and key == unlucky: del ueba["storys"][unlucky] response["key"] = unlucky Kdatabase().dump("ueba") break socket.response(response)
def run(payload, socket): from core.fingerprint import Kfingerprint fingerprint = Kdatabase().get_obj("fingerprint") while True: if Kevent().is_terminate(): print("fingerprint thread terminate") break now = time_op.now() settings = Kdatabase().get_obj('strategy') if settings: if now > fingerprint["port"]["lasttime"] + settings["asset"]["port_scan"]: Kfingerprint().record_listening_port() if now > fingerprint["account"]["lasttime"] + settings["asset"]["account_scan"]: Kfingerprint().record_account() time.sleep(5)
def run(payload, socket): response = { "cmd_id": payload["cmd_id"], "session_id": payload["args"]["session_id"], "error": "" } baseline = Kdatabase().get_obj("baseline") risk_id = payload["args"]["risk_id"] if len(risk_id) > 0: for i in risk_id: baseline["risks"][i]["stage"] = macro.BASELINE_STAGE["IGNORED"] baseline["risks"][i]["handle_ts"] = time_op.now() Kdatabase().dump('baseline') else: response["error"] = "Nothing to ignore" socket.response(response)
def get_warning_and_score(self): cleaner = Kdatabase().get_obj("cleaner") kinds_db = cleaner['kinds'] totalsize = 0 for i,item in kinds_db.items(): totalsize += item['size'] garbage_128M = (totalsize >> 27) lost_score = lost_score_correction(garbage_128M * 5, self.coefficient) return "cleaner", totalsize, lost_score, cleaner["lasttime"]