def get_label_for_endpoint(endpoint_id): if endpoint_id == ObjectId("000000000000000000000000"): return 'MonkeyIsland' if Monkey.is_monkey(endpoint_id): return Monkey.get_label_by_id(endpoint_id) else: return NodeService.get_node_label(NodeService.get_node_by_id(endpoint_id))
def test_dispatch_to_relevant_collector(self): self.fail_if_not_testing_env() self.clean_monkey_db() a_monkey = Monkey(guid=str(uuid.uuid4())) a_monkey.save() dispatcher = SystemInfoTelemetryDispatcher() # JSON with results - make sure functions are called instance_id = "i-0bd2c14bd4c7d703f" telem_json = { "data": { "collectors": { "AwsCollector": { "instance_id": instance_id }, } }, "monkey_guid": a_monkey.guid } dispatcher.dispatch_collector_results_to_relevant_processors( telem_json) self.assertEquals( Monkey.get_single_monkey_by_guid(a_monkey.guid).aws_instance_id, instance_id)
def get_displayed_node_by_id(node_id, for_report=False): if ObjectId(node_id) == NodeService.get_monkey_island_pseudo_id(): return NodeService.get_monkey_island_node() new_node = {"id": node_id} node = NodeService.get_node_by_id(node_id) if node is None: monkey = NodeService.get_monkey_by_id(node_id) if monkey is None: return new_node # node is infected new_node = NodeService.monkey_to_net_node(monkey, for_report) for key in monkey: if key not in [ '_id', 'modifytime', 'parent', 'dead', 'description' ]: new_node[key] = monkey[key] else: # node is uninfected new_node = NodeService.node_to_net_node(node, for_report) new_node["ip_addresses"] = node["ip_addresses"] new_node["domain_name"] = node["domain_name"] accessible_from_nodes = [] accessible_from_nodes_hostnames = [] exploits = [] edges = DisplayedEdgeService.get_displayed_edges_by_dst( node_id, for_report) for edge in edges: from_node_id = edge['from'] from_node_label = Monkey.get_label_by_id(from_node_id) from_node_hostname = Monkey.get_hostname_by_id(from_node_id) accessible_from_nodes.append(from_node_label) accessible_from_nodes_hostnames.append(from_node_hostname) for edge_exploit in edge['exploits']: edge_exploit['origin'] = from_node_label exploits.append(edge_exploit) exploits = sorted(exploits, key=lambda exploit: exploit['timestamp']) new_node["exploits"] = exploits new_node["accessible_from_nodes"] = accessible_from_nodes new_node[ "accessible_from_nodes_hostnames"] = accessible_from_nodes_hostnames if len(edges) > 0: new_node["services"] = edges[-1]["services"] else: new_node["services"] = [] new_node[ 'has_log'] = monkey_island.cc.services.log.LogService.log_exists( ObjectId(node_id)) return new_node
def check_tunneling_violation(tunnel_telemetry_json): if tunnel_telemetry_json["data"]["proxy"] is not None: # Monkey is tunneling, create findings tunnel_host_ip = get_tunnel_host_ip_from_proxy_field( tunnel_telemetry_json) current_monkey = Monkey.get_single_monkey_by_guid( tunnel_telemetry_json["monkey_guid"]) tunneling_events = [ Event.create_event( title="Tunneling event", message="Monkey on {hostname} tunneled traffic through {proxy}." .format(hostname=current_monkey.hostname, proxy=tunnel_host_ip), event_type=zero_trust_consts.EVENT_TYPE_MONKEY_NETWORK, timestamp=tunnel_telemetry_json["timestamp"], ) ] MonkeyZTFindingService.create_or_add_to_existing( test=zero_trust_consts.TEST_TUNNELING, status=zero_trust_consts.STATUS_FAILED, events=tunneling_events, ) MonkeyZTFindingService.add_malicious_activity_to_timeline( tunneling_events)
def monkey_to_net_node(monkey, for_report=False): monkey_id = monkey["_id"] label = Monkey.get_hostname_by_id(monkey_id) if for_report else Monkey.get_label_by_id(monkey_id) monkey_group = NodeService.get_monkey_group(monkey) return \ { "id": monkey_id, "label": label, "group": monkey_group, "os": NodeService.get_monkey_os(monkey), # The monkey is running IFF the group contains "_running". Therefore it's dead IFF the group does NOT # contain "_running". This is a small optimisation, to not call "is_dead" twice. "dead": "_running" not in monkey_group, "domain_name": "", "pba_results": monkey["pba_results"] if "pba_results" in monkey else [] }
def _get_infected_island_net_edges(monkey_island_monkey): existing_ids = [ x.src_node_id for x in EdgeService.get_by_dst_node( dst_node_id=monkey_island_monkey["_id"]) ] monkey_ids = [ x.id for x in Monkey.objects() if ("tunnel" not in x) and (x.id not in existing_ids) and ( x.id != monkey_island_monkey["_id"]) ] edges = [] count = 0 for monkey_id in monkey_ids: count += 1 # generating fake ID, because front end requires unique ID's for each edge. Collision # improbable fake_id = ObjectId(hex(count)[2:].zfill(24)) src_label = NodeService.get_label_for_endpoint(monkey_id) dst_label = NodeService.get_label_for_endpoint( monkey_island_monkey["_id"]) edge = DisplayedEdgeService.generate_pseudo_edge( edge_id=fake_id, src_node_id=monkey_id, dst_node_id=monkey_island_monkey["_id"], src_label=src_label, dst_label=dst_label, ) edges.append(edge) return edges
def test_antivirus_existence(process_list_json, monkey_guid): current_monkey = Monkey.get_single_monkey_by_guid(monkey_guid) process_list_event = Event.create_event( title="Process list", message="Monkey on {} scanned the process list".format( current_monkey.hostname), event_type=zero_trust_consts.EVENT_TYPE_MONKEY_LOCAL) events = [process_list_event] av_processes = filter_av_processes(process_list_json["process_list"]) for process in av_processes: events.append( Event.create_event( title="Found AV process", message= "The process '{}' was recognized as an Anti Virus process. Process " "details: {}".format(process[1]['name'], json.dumps(process[1])), event_type=zero_trust_consts.EVENT_TYPE_MONKEY_LOCAL)) if len(av_processes) > 0: test_status = zero_trust_consts.STATUS_PASSED else: test_status = zero_trust_consts.STATUS_FAILED AggregateFinding.create_or_add_to_existing( test=zero_trust_consts.TEST_ENDPOINT_SECURITY_EXISTS, status=test_status, events=events)
def get_report_data(): monkeys = Monkey.get_tunneled_monkeys() monkeys = [monkey.get_network_info() for monkey in monkeys] status = ScanStatus.USED.value if monkeys else ScanStatus.UNSCANNED.value data = T1090.get_base_data_by_status(status) data.update({'proxies': monkeys}) return data
def test_create_findings_for_all_done_pairs(self): self.fail_if_not_testing_env() self.clean_finding_db() all_subnets = [FIRST_SUBNET, SECOND_SUBNET, THIRD_SUBNET] monkey = Monkey(guid=str(uuid.uuid4()), ip_addresses=[FIRST_SUBNET]) # no findings self.assertEquals(len(Finding.objects(test=TEST_SEGMENTATION)), 0) # This is like the monkey is done and sent done telem create_or_add_findings_for_all_pairs(all_subnets, monkey) # There are 2 subnets in which the monkey is NOT self.assertEquals( len(Finding.objects(test=TEST_SEGMENTATION, status=STATUS_PASSED)), 2) # This is a monkey from 2nd subnet communicated with 1st subnet. SegmentationFinding.create_or_add_to_existing_finding( [FIRST_SUBNET, SECOND_SUBNET], STATUS_FAILED, Event.create_event(title="sdf", message="asd", event_type=EVENT_TYPE_MONKEY_NETWORK)) self.assertEquals( len(Finding.objects(test=TEST_SEGMENTATION, status=STATUS_PASSED)), 1) self.assertEquals( len(Finding.objects(test=TEST_SEGMENTATION, status=STATUS_FAILED)), 1) self.assertEquals(len(Finding.objects(test=TEST_SEGMENTATION)), 2)
def generate_new_report(): """ Generates new report based on telemetries, replaces old report in db with new one. :return: Report object """ report = \ { 'techniques': {}, 'meta': {'latest_monkey_modifytime': Monkey.get_latest_modifytime()}, 'name': REPORT_NAME } for tech_id, tech_info in list( AttackConfig.get_techniques_for_report().items()): try: technique_report_data = TECHNIQUES[tech_id].get_report_data() technique_report_data.update(tech_info) report['techniques'].update({tech_id: technique_report_data}) except KeyError as e: LOG.error( "Attack technique does not have it's report component added " "to attack report service. %s" % e) mongo.db.attack_report.replace_one({'name': REPORT_NAME}, report, upsert=True) return report
def test_open_data_endpoints(telemetry_json): services = telemetry_json["data"]["machine"]["services"] current_monkey = Monkey.get_single_monkey_by_guid(telemetry_json['monkey_guid']) found_http_server_status = zero_trust_consts.STATUS_PASSED found_elastic_search_server = zero_trust_consts.STATUS_PASSED events = [ Event.create_event( title="Scan Telemetry", message="Monkey on {} tried to perform a network scan, the target was {}.".format( current_monkey.hostname, telemetry_json["data"]["machine"]["ip_addr"]), event_type=zero_trust_consts.EVENT_TYPE_MONKEY_NETWORK, timestamp=telemetry_json["timestamp"] ) ] for service_name, service_data in list(services.items()): events.append(Event.create_event( title="Scan telemetry analysis", message="Scanned service: {}.".format(service_name), event_type=zero_trust_consts.EVENT_TYPE_MONKEY_NETWORK )) if service_name in HTTP_SERVERS_SERVICES_NAMES: found_http_server_status = zero_trust_consts.STATUS_FAILED events.append(Event.create_event( title="Scan telemetry analysis", message="Service {} on {} recognized as an open data endpoint! Service details: {}".format( service_data["display_name"], telemetry_json["data"]["machine"]["ip_addr"], json.dumps(service_data) ), event_type=zero_trust_consts.EVENT_TYPE_MONKEY_NETWORK )) if service_name == ES_SERVICE: found_elastic_search_server = zero_trust_consts.STATUS_FAILED events.append(Event.create_event( title="Scan telemetry analysis", message="Service {} on {} recognized as an open data endpoint! Service details: {}".format( service_data["display_name"], telemetry_json["data"]["machine"]["ip_addr"], json.dumps(service_data) ), event_type=zero_trust_consts.EVENT_TYPE_MONKEY_NETWORK )) AggregateFinding.create_or_add_to_existing( test=zero_trust_consts.TEST_DATA_ENDPOINT_HTTP, status=found_http_server_status, events=events ) AggregateFinding.create_or_add_to_existing( test=zero_trust_consts.TEST_DATA_ENDPOINT_ELASTIC, status=found_elastic_search_server, events=events ) add_malicious_activity_to_timeline(events)
def process_scan_telemetry(telemetry_json): update_edges_and_nodes_based_on_scan_telemetry(telemetry_json) check_open_data_endpoints(telemetry_json) current_monkey = Monkey.get_single_monkey_by_guid( telemetry_json["monkey_guid"]) target_ip = telemetry_json["data"]["machine"]["ip_addr"] check_segmentation_violation(current_monkey, target_ip)
def process_scan_telemetry(telemetry_json): update_edges_and_nodes_based_on_scan_telemetry(telemetry_json) test_open_data_endpoints(telemetry_json) current_monkey = Monkey.get_single_monkey_by_guid( telemetry_json['monkey_guid']) target_ip = telemetry_json['data']['machine']['ip_addr'] test_segmentation_violation(current_monkey, target_ip)
def get(self): NodeService.update_dead_monkeys() island_monkey = NodeService.get_monkey_island_monkey() if island_monkey is not None: is_monkey_running = not Monkey.get_single_monkey_by_id(island_monkey["_id"]).is_dead() else: is_monkey_running = False return jsonify(is_running=is_monkey_running)
def test_process_environment_telemetry(self): # Arrange monkey_guid = str(uuid.uuid4()) a_monkey = Monkey(guid=monkey_guid) a_monkey.save() dispatcher = SystemInfoTelemetryDispatcher() on_premise = "On Premise" telem_json = { "data": { "collectors": { "EnvironmentCollector": {"environment": on_premise}, } }, "monkey_guid": monkey_guid } dispatcher.dispatch_collector_results_to_relevant_processors(telem_json) assert Monkey.get_single_monkey_by_guid(monkey_guid).environment == on_premise
def get_monkey_group(monkey): if len(set(monkey["ip_addresses"]).intersection( local_ip_addresses())) != 0: monkey_type = "island_monkey" else: monkey_type = "manual" if NodeService.get_monkey_manual_run( monkey) else "monkey" monkey_os = NodeService.get_monkey_os(monkey) monkey_running = "" if Monkey.get_single_monkey_by_id( monkey["_id"]).is_dead() else "_running" return "%s_%s%s" % (monkey_type, monkey_os, monkey_running)
def generate_map_nodes(): monkeys = [m for m in Monkey.objects() if m.get_os() == "windows"] return [ { 'id': monkey.guid, 'label': '{0} : {1}'.format(monkey.hostname, monkey.ip_addresses[0]), 'group': 'critical' if monkey.critical_services is not None else 'normal', 'services': monkey.critical_services, 'hostname': monkey.hostname } for monkey in monkeys ]
def test_generate_map_nodes_parsing(self): self.fail_if_not_testing_env() self.clean_monkey_db() monkey_id = str(uuid.uuid4()) hostname = "A_Windows_PC_1" windows_monkey_with_services = Monkey( guid=monkey_id, hostname=hostname, critical_services=["aCriticalService", "Domain Controller"], ip_addresses=["1.1.1.1"], description="windows 10") windows_monkey_with_services.save() map_nodes = PTHReportService.generate_map_nodes() self.assertEquals(map_nodes[0]["id"], monkey_id) self.assertEquals(map_nodes[0]["label"], "A_Windows_PC_1 : 1.1.1.1") self.assertEquals(map_nodes[0]["group"], "critical") self.assertEquals(len(map_nodes[0]["services"]), 2) self.assertEquals(map_nodes[0]["hostname"], hostname)
def get_monkey_group(monkey): keywords = [] if len(set(monkey["ip_addresses"]).intersection(local_ip_addresses())) != 0: keywords.extend(["island", "monkey"]) else: monkey_type = "manual" if NodeService.get_monkey_manual_run(monkey) else "monkey" keywords.append(monkey_type) keywords.append(NodeService.get_monkey_os(monkey)) if not Monkey.get_single_monkey_by_id(monkey["_id"]).is_dead(): keywords.append("running") return NodeStates.get_by_keywords(keywords).value
def process_exploit_telemetry(telemetry_json): encrypt_exploit_creds(telemetry_json) edge = get_edge_by_scan_or_exploit_telemetry(telemetry_json) update_network_with_exploit(edge, telemetry_json) update_node_credentials_from_successful_attempts(edge, telemetry_json) test_machine_exploited( current_monkey=Monkey.get_single_monkey_by_guid(telemetry_json['monkey_guid']), exploit_successful=telemetry_json['data']['result'], exploiter=telemetry_json['data']['exploiter'], target_ip=telemetry_json['data']['machine']['ip_addr'], timestamp=telemetry_json['timestamp'])
def generate_report(): domain_issues = ReportService.get_domain_issues() issues = ReportService.get_issues() config_users = ReportService.get_config_users() config_passwords = ReportService.get_config_passwords() cross_segment_issues = ReportService.get_cross_segment_issues() monkey_latest_modify_time = Monkey.get_latest_modifytime() scanned_nodes = ReportService.get_scanned() exploited_nodes = ReportService.get_exploited() report = \ { 'overview': { 'manual_monkeys': ReportService.get_manual_monkeys(), 'config_users': config_users, 'config_passwords': config_passwords, 'config_exploits': ReportService.get_config_exploits(), 'config_ips': ReportService.get_config_ips(), 'config_scan': ReportService.get_config_scan(), 'monkey_start_time': ReportService.get_first_monkey_time().strftime("%d/%m/%Y %H:%M:%S"), 'monkey_duration': ReportService.get_monkey_duration(), 'issues': ReportService.get_issues_overview(issues, config_users, config_passwords), 'warnings': ReportService.get_warnings_overview(issues, cross_segment_issues), 'cross_segment_issues': cross_segment_issues }, 'glance': { 'scanned': scanned_nodes, 'exploited': exploited_nodes, 'stolen_creds': ReportService.get_stolen_creds(), 'azure_passwords': ReportService.get_azure_creds(), 'ssh_keys': ReportService.get_ssh_keys(), 'strong_users': PTHReportService.get_strong_users_on_crit_details(), 'pth_map': PTHReportService.get_pth_map() }, 'recommendations': { 'issues': issues, 'domain_issues': domain_issues }, 'meta': { 'latest_monkey_modifytime': monkey_latest_modify_time } } ReportExporterManager().export(report) mongo.db.report.drop() mongo.db.report.insert_one( ReportService.encode_dot_char_before_mongo_insert(report)) return report
def process_state_telemetry(telemetry_json): monkey = NodeService.get_monkey_by_guid(telemetry_json['monkey_guid']) NodeService.add_communication_info( monkey, telemetry_json['command_control_channel']) if telemetry_json['data']['done']: NodeService.set_monkey_dead(monkey, True) else: NodeService.set_monkey_dead(monkey, False) if telemetry_json['data']['done']: current_monkey = Monkey.get_single_monkey_by_guid( telemetry_json['monkey_guid']) test_passed_findings_for_unreached_segments(current_monkey)
def get_latest_report(): """ Gets latest report (by retrieving it from db or generating a new one). :return: report dict. """ if AttackReportService.is_report_generated(): monkey_modifytime = Monkey.get_latest_modifytime() latest_report = mongo.db.attack_report.find_one({"name": REPORT_NAME}) report_modifytime = latest_report["meta"]["latest_monkey_modifytime"] if monkey_modifytime and report_modifytime and monkey_modifytime == report_modifytime: return latest_report return safe_generate_attack_report()
def is_latest_report_exists(): """ This function checks if a monkey report was already generated and if it's the latest one. :return: True if report is the latest one, False if there isn't a report or its not the latest. """ latest_report_doc = mongo.db.report.find_one({}, {"meta.latest_monkey_modifytime": 1}) if latest_report_doc: report_latest_modifytime = latest_report_doc["meta"]["latest_monkey_modifytime"] latest_monkey_modifytime = Monkey.get_latest_modifytime() return report_latest_modifytime == latest_monkey_modifytime return False
def process_exploit_telemetry(telemetry_json): encrypt_exploit_creds(telemetry_json) edge = get_edge_by_scan_or_exploit_telemetry(telemetry_json) update_network_with_exploit(edge, telemetry_json) update_node_credentials_from_successful_attempts(edge, telemetry_json) add_exploit_extracted_creds_to_config(telemetry_json) check_machine_exploited( current_monkey=Monkey.get_single_monkey_by_guid( telemetry_json["monkey_guid"]), exploit_successful=telemetry_json["data"]["result"], exploiter=telemetry_json["data"]["exploiter"], target_ip=telemetry_json["data"]["machine"]["ip_addr"], timestamp=telemetry_json["timestamp"], )
def generate_map_nodes(): monkeys = [m for m in Monkey.objects() if m.get_os() == "windows"] return [{ "id": monkey.guid, "label": "{0} : {1}".format(monkey.hostname, monkey.ip_addresses[0]), "group": "critical" if monkey.critical_services is not None else "normal", "services": monkey.critical_services, "hostname": monkey.hostname, } for monkey in monkeys]
def process_state_telemetry(telemetry_json): monkey = NodeService.get_monkey_by_guid(telemetry_json["monkey_guid"]) NodeService.add_communication_info( monkey, telemetry_json["command_control_channel"]) if telemetry_json["data"]["done"]: NodeService.set_monkey_dead(monkey, True) else: NodeService.set_monkey_dead(monkey, False) if telemetry_json["data"]["done"]: current_monkey = Monkey.get_single_monkey_by_guid( telemetry_json["monkey_guid"]) check_passed_findings_for_unreached_segments(current_monkey) if telemetry_json["data"]["version"]: logger.info(f"monkey {telemetry_json['monkey_guid']} has version " f"{telemetry_json['data']['version']}")
def _get_uninfected_island_net_edges(): edges = [] monkey_ids = [x.id for x in Monkey.objects() if "tunnel" not in x] count = 0 for monkey_id in monkey_ids: count += 1 # generating fake ID, because front end requires unique ID's for each edge. Collision improbable fake_id = ObjectId(hex(count)[2:].zfill(24)) island_id = ObjectId("000000000000000000000000") monkey_label = NodeService.get_label_for_endpoint(monkey_id) island_label = NodeService.get_label_for_endpoint(island_id) island_pseudo_edge = DisplayedEdgeService.generate_pseudo_edge(edge_id=fake_id, src_node_id=monkey_id, dst_node_id=island_id, src_label=monkey_label, dst_label=island_label) edges.append(island_pseudo_edge) return edges
def generate_report(): domain_issues = ReportService.get_domain_issues() issues = ReportService.get_issues() config_users = ReportService.get_config_users() config_passwords = ReportService.get_config_passwords() issue_set = ReportService.get_issue_set(issues, config_users, config_passwords) cross_segment_issues = ReportService.get_cross_segment_issues() monkey_latest_modify_time = Monkey.get_latest_modifytime() scanned_nodes = ReportService.get_scanned() exploited_cnt = len(get_monkey_exploited()) report = { "overview": { "manual_monkeys": ReportService.get_manual_monkey_hostnames(), "config_users": config_users, "config_passwords": config_passwords, "config_exploits": ReportService.get_config_exploits(), "config_ips": ReportService.get_config_ips(), "config_scan": ReportService.get_config_scan(), "monkey_start_time": ReportService.get_first_monkey_time().strftime( "%d/%m/%Y %H:%M:%S" ), "monkey_duration": ReportService.get_monkey_duration(), "issues": issue_set, "cross_segment_issues": cross_segment_issues, }, "glance": { "scanned": scanned_nodes, "exploited_cnt": exploited_cnt, "stolen_creds": ReportService.get_stolen_creds(), "azure_passwords": ReportService.get_azure_creds(), "ssh_keys": ReportService.get_ssh_keys(), "strong_users": PTHReportService.get_strong_users_on_crit_details(), }, "recommendations": {"issues": issues, "domain_issues": domain_issues}, "meta": {"latest_monkey_modifytime": monkey_latest_modify_time}, } ReportExporterManager().export(report) mongo.db.report.drop() mongo.db.report.insert_one(ReportService.encode_dot_char_before_mongo_insert(report)) return report
def test_tunneling_violation(tunnel_telemetry_json): if tunnel_telemetry_json['data']['proxy'] is not None: # Monkey is tunneling, create findings tunnel_host_ip = get_tunnel_host_ip_from_proxy_field( tunnel_telemetry_json) current_monkey = Monkey.get_single_monkey_by_guid( tunnel_telemetry_json['monkey_guid']) tunneling_events = [ Event.create_event( title="Tunneling event", message="Monkey on {hostname} tunneled traffic through {proxy}." .format(hostname=current_monkey.hostname, proxy=tunnel_host_ip), event_type=EVENT_TYPE_MONKEY_NETWORK, timestamp=tunnel_telemetry_json['timestamp']) ] AggregateFinding.create_or_add_to_existing(test=TEST_TUNNELING, status=STATUS_FAILED, events=tunneling_events) add_malicious_activity_to_timeline(tunneling_events)