async def start_calc(self): #call this only after setting the current ADID!!!! try: adinfo = self.session.query(ADInfo).get(self.ad_id) self.domain_name = str(adinfo.distinguishedName).replace( ',', '.').replace('DC=', '') await self.log_msg('Adding gplink edges') self.gplink_edges() #await self.log_msg() #self.groupmembership_edges() await self.log_msg('Adding trusts edges') self.trust_edges() await self.log_msg('Adding sqladmin edges') self.sqladmin_edges() await self.log_msg('Adding hassession edges') self.hasession_edges() await self.log_msg('Adding localgroup edges') self.localgroup_edges() await self.log_msg('Adding password sharing edges') self.passwordsharing_edges() await self.log_msg('Adding allowedtoact sharing edges') self.allowedtoact_edges() self.session.commit() _, err = await self.calc_sds_mp() if err is not None: raise err adinfo = self.session.query(ADInfo).get(self.ad_id) adinfo.edges_finished = True self.session.commit() return True, None except Exception as e: logger.exception('edge calculation error!') return False, e
def edge_calc_writer(outqueue, file_path, ad_id, append_to_file): """ Separate process to write all edges in a file as CSV """ try: buffer = '' if append_to_file is True: mode = 'a+' else: mode = 'w' with open(file_path, 'w', newline='') as f: while True: data = outqueue.get() if data is None: return src, dst, label = data buffer += '%s,%s,%s,%s\r\n' if len(buffer) > 1000000: print('writing!') f.write(buffer) buffer = '' except Exception as e: logger.exception('edge_calc_writer')
async def run(self): try: self.session = get_session(self.db_conn) if self.ad_id is None and self.graph_id is not None: #recalc! self.session.query(Edge).filter_by(graph_id = self.graph_id).filter(Edge.label != 'member').delete() self.session.commit() res = self.session.query(GraphInfo).get(self.graph_id) for giad in self.session.query(GraphInfoAD).filter_by(graph_id = self.graph_id).all(): self.ad_id = giad.ad_id _, err = await self.start_calc() if err is not None: raise err else: _, err = await self.start_calc() if err is not None: raise err return True, None except Exception as e: logger.exception('edge calculation error!') return False, e finally: try: self.session.close() except: pass
def add_credentials_impacket(self, impacket_file): self.get_dbsession() ctr = 0 ctr_fail = 0 try: for cred in Credential.from_impacket_file(impacket_file, self.domain_id): try: self.dbsession.add(cred) if ctr % 10000 == 0: logger.info(ctr) self.dbsession.commit() except exc.IntegrityError as e: ctr_fail += 1 self.dbsession.rollback() continue else: ctr += 1 self.dbsession.commit() logger.info('Added %d users. Failed inserts: %d' % (ctr, ctr_fail)) except Exception as e: logger.exception() finally: self.dbsession.close()
def lookup_oid(self, oint, ad_id, token): try: if oint not in self.edgeinfo_cache: edgeinfo = self.db_session.query(EdgeLookup).get(oint) self.edgeinfo_cache[oint] = edgeinfo.oid return self.edgeinfo_cache[oint] except Exception as e: logger.exception('lookup_oid')
async def generate_member_targets(self): try: subq = self.session.query(JackDawTokenGroup.guid).distinct(JackDawTokenGroup.guid).filter(JackDawTokenGroup.ad_id == self.ad_id) q = self.session.query(JackDawADUser.dn, JackDawADUser.objectSid, JackDawADUser.objectGUID).filter_by(ad_id = self.ad_id).filter(~JackDawADUser.objectGUID.in_(subq)) await self.resumption_target_gen_member(q, JackDawADUser.id, 'user', LDAPAgentCommand.MEMBERSHIPS) q = self.session.query(JackDawADMachine.dn, JackDawADMachine.objectSid, JackDawADMachine.objectGUID).filter_by(ad_id = self.ad_id).filter(~JackDawADMachine.objectGUID.in_(subq)) await self.resumption_target_gen_member(q, JackDawADMachine.id, 'machine', LDAPAgentCommand.MEMBERSHIPS) q = self.session.query(JackDawADGroup.dn, JackDawADGroup.sid, JackDawADGroup.guid).filter_by(ad_id = self.ad_id).filter(~JackDawADGroup.guid.in_(subq)) await self.resumption_target_gen_member(q, JackDawADGroup.id, 'group', LDAPAgentCommand.MEMBERSHIPS) except Exception as e: logger.exception('generate_sd_targets')
def run(self): try: if self.append_to_file is True: mode = 'ab+' else: mode = 'wb' output_file_path = self.output_file_dir.joinpath('temp_edges.gz') self.out_file = GzipFile(output_file_path, mode) self.gplink_edges() self.groupmembership_edges() self.trust_edges() self.sqladmin_edges() self.hasession_edges() self.localgroup_edges() self.passwordsharing_edges() self.out_file.close() sdcalc = SDEgdeCalc(self.session, self.ad_id, output_file_path, worker_count=self.worker_count, buffer_size=self.buffer_size, append_to_file=True) sdcalc.run() #output_file_path_final = self.output_file_dir.joinpath('edges.gz') #with GzipFile(output_file_path_final,'wb') as o: # with GzipFile(output_file_path,'rb') as f: # for line in f: # line = line.strip() # line = line.decode() # src, dst, *rest = line.split(',') # t = ','.join(rest) # src = self.lookup.insert(src) # dst = self.lookup.insert(dst) # o.write( ('%s,%s,%s\r\n' % (src,dst,t)).encode()) # #output_file_path_maps = self.output_file_dir.joinpath('maps.gz') #with GzipFile(output_file_path_maps,'wb') as o: # for k in self.lookup.lookup: # o.write(('%s,%s\r\n' % (k, self.lookup.lookup[k])).encode()) output_file_path.unlink() except Exception as e: logger.exception('edge calculation error!') finally: try: if self.out_file is not None: self.out_file.close() except: pass print('Done!')
def run(self): try: logger.debug('[ACL] Starting sd edge calc') self.inqueue = mp.Queue(self.buffer_size) self.outqueue = mp.Queue(self.buffer_size) logger.debug('[ACL] Starting processes') self.writer = mp.Process(target=edge_calc_writer, args=(self.outqueue, self.output_file_path, self.ad_id, self.append_to_file)) self.writer.daemon = True self.writer.start() self.workers = [ mp.Process(target=edge_calc_worker, args=(self.inqueue, self.outqueue)) for i in range(self.worker_count) ] for proc in self.workers: proc.daemon = True proc.start() print(1) logger.debug('[ACL] data generation') total = self.session.query(func.count( JackDawSD.id)).filter_by(ad_id=self.ad_id).scalar() q = self.session.query(JackDawSD).filter_by(ad_id=self.ad_id) for adsd in tqdm(windowed_query(q, JackDawSD.id, 10), total=total): self.inqueue.put(adsd) for _ in range(procno): self.inqueue.put(None) logger.debug('Gen done!') logger.debug('[ACL] Added %s edges' % (p_cnt)) logger.debug('[ACL] joining workers') for proc in self.workers: proc.join() logger.debug('[ACL] workers finished, waiting for writer') self.outqueue.put(None) self.writer.join() logger.debug('[ACL] All Finished!') except: logger.exception('[ACL]')
async def worker(self): while True: try: target = await self.worker_q.get() if target is None: return try: await self.scan_host(target) except: #exception should be handled in scan_host continue except Exception as e: logger.exception('WORKER ERROR') raise
async def run(self): try: self.session = get_session(self.db_conn) if self.ad_id is None and self.graph_id is not None: #recalc! self.session.query(Edge).filter_by( graph_id=self.graph_id).filter( Edge.label != 'member').delete() self.session.commit() res = self.session.query(GraphInfo).get(self.graph_id) self.ad_id = res.ad_id adinfo = self.session.query(ADInfo).get(self.ad_id) self.domain_name = str(adinfo.distinguishedName).replace( ',', '.').replace('DC=', '') await self.log_msg('Adding gplink edges') self.gplink_edges() #await self.log_msg() #self.groupmembership_edges() await self.log_msg('Adding trusts edges') self.trust_edges() await self.log_msg('Adding sqladmin edges') self.sqladmin_edges() await self.log_msg('Adding hassession edges') self.hasession_edges() await self.log_msg('Adding localgroup edges') self.localgroup_edges() await self.log_msg('Adding password sharing edges') self.passwordsharing_edges() self.session.commit() _, err = await self.calc_sds_mp() if err is not None: raise err adinfo = self.session.query(ADInfo).get(self.ad_id) adinfo.edges_finished = True self.session.commit() return True, None except Exception as e: logger.exception('edge calculation error!') return False, e finally: try: self.session.close() except: pass
async def do_listads(self, cmd): """ Lists all available adid in the DB Sends back a NestOpListADRes reply or ERR in case of failure """ try: reply = NestOpListADRes() for i in self.db_session.query(ADInfo.id).all(): reply.adids.append(i[0]) await self.send_reply(cmd, reply) await self.send_ok(cmd) except Exception as e: logger.exception('do_listads') await self.send_error(cmd, e)
async def do_changead(self, cmd): """ Changes the current AD to another, specified by ad_id in the command. Doesnt have a dedicated reply, OK means change is succsess, ERR means its not """ try: res = self.db_session.query(ADInfo).get(cmd.adid) print(res) if res is None: await self.send_error(cmd, 'No such AD in database') return self.ad_id = res.id await self.send_ok(cmd) except Exception as e: logger.exception('do_listads') await self.send_error(cmd, e)
async def stop_sds_collection(self, sds_p): sds_p.disable = True try: self.sd_file.close() cnt = 0 with gzip.GzipFile(self.sd_file_path, 'r') as f: for line in tqdm(f, desc='Uploading security descriptors to DB', total=self.spn_finish_ctr): sd = JackDawSD.from_json(line.strip()) self.session.add(sd) cnt += 1 if cnt % 10000 == 0: self.session.commit() self.session.commit() os.remove(self.sd_file_path) except Exception as e: logger.exception('Error while uploading sds from file to DB')
async def generate_targets(self): try: q = self.session.query(Machine).filter_by(ad_id=self.ad_id) for machine in windowed_query(q, Machine.id, 100): try: dns_name = machine.dNSHostName if dns_name is None or dns_name == '': dns_name = '%s.%s' % (str(machine.sAMAccountName[:-1]), str(self.domain_name)) await self.in_q.put((machine.objectSid, dns_name)) except: continue #signaling the ed of target generation await self.in_q.put(None) except Exception as e: logger.exception('smb generate_targets')
async def stop_memberships_collection(self, member_p): member_p.disable = True try: self.token_file.close() cnt = 0 with gzip.GzipFile(self.token_file_path, 'r') as f: for line in tqdm(f, desc='Uploading memberships to DB', total=self.member_finish_ctr): sd = JackDawTokenGroup.from_json(line.strip()) self.session.add(sd) cnt += 1 if cnt % 10000 == 0: self.session.commit() self.session.commit() os.remove(self.token_file_path) except Exception as e: logger.exception('Error while uploading memberships from file to DB')
async def generate_member_targets(self): try: subq = self.session.query(EdgeLookup.oid).filter_by(ad_id = self.ad_id).filter(EdgeLookup.id == Edge.src).filter(Edge.label == 'member').filter(Edge.ad_id == self.ad_id) q = self.session.query(ADUser.dn, ADUser.objectSid, ADUser.objectGUID)\ .filter_by(ad_id = self.ad_id)\ .filter(~ADUser.objectSid.in_(subq)) await self.resumption_target_gen_member(q, ADUser.id, 'user', LDAPAgentCommand.MEMBERSHIPS) q = self.session.query(Machine.dn, Machine.objectSid, Machine.objectGUID)\ .filter_by(ad_id = self.ad_id)\ .filter(~Machine.objectSid.in_(subq)) await self.resumption_target_gen_member(q, Machine.id, 'machine', LDAPAgentCommand.MEMBERSHIPS) q = self.session.query(Group.dn, Group.objectSid, Group.objectGUID)\ .filter_by(ad_id = self.ad_id)\ .filter(~Group.objectSid.in_(subq)) await self.resumption_target_gen_member(q, Group.id, 'group', LDAPAgentCommand.MEMBERSHIPS) except Exception as e: logger.exception('generate_member_targets')
def run(self): try: logger.debug('[ACL] Starting sd edge calc') logger.debug('[ACL] data generation') total = self.session.query(func.count( JackDawSD.id)).filter_by(ad_id=self.ad_id).scalar() q = self.session.query(JackDawSD).filter_by(ad_id=self.ad_id) for adsd in tqdm(windowed_query(q, JackDawSD.id, self.worker_count), total=total): self.inqueue.put(adsd) logger.debug('[ACL] All Finished!') except: logger.exception('[ACL]')
def calc_acl_edges_mp(self, session, adid, construct): try: #ACE edges calc with multiprocessing inqueue = mp.Queue() outqueue = mp.Queue() procno = mp.cpu_count() logger.debug('[ACL] Starting processes') procs = [ mp.Process(target=acl_calc_mp, args=(inqueue, outqueue, construct)) for i in range(procno) ] for proc in procs: proc.daemon = True proc.start() logger.debug('[ACL] Starting generator thread') acl_gen_th = threading.Thread(target=acl_calc_gen, args=(session, adid, inqueue, procno)) acl_gen_th.daemon = True acl_gen_th.start() p_cnt = 0 while True: res = outqueue.get() if res is None: procno -= 1 logger.debug('[ACL] Proc X - Finished!') if procno == 0: break continue ace_sid, sid, label = res self.add_edge(ace_sid, sid, construct, label=label) p_cnt += 1 logger.debug('[ACL] Added %s edges' % (p_cnt)) logger.debug('[ACL] joining processes') for proc in procs: proc.join() logger.debug('[ACL] Finished!') except Exception as e: logger.exception('[ACL]')
async def prepare_targets(self): try: if self.resumption is True: self.total_targets = 1 if self.members_target_file_handle is not None: raise Exception('Resumption doesnt use the target file handle!') self.members_target_file_handle = gzip.GzipFile('member_targets.gz','wb') await self.generate_member_targets() else: self.members_target_file_handle.seek(0,0) for line in self.members_target_file_handle: self.total_members_to_poll += 1 return True, None except Exception as err: logger.exception('prep targets') return False, err
def edge_calc_writer(outqueue, file_path, ad_id, append_to_file): """ Separate process to write all edges in a file as CSV """ try: buffer = b'' if append_to_file is True: mode = 'ab+' else: mode = 'wb' with GzipFile(file_path, mode) as f: while True: data = outqueue.get() if data is None: return f.write(data) except Exception as e: logger.exception('edge_calc_writer')
async def arun(self): try: res = await self.setup() if res is False: return while True: res = await self.agent_in_q.get() if res is None: return if res.command == LDAPAgentCommand.DOMAININFO: await self.get_domain_info() elif res.command == LDAPAgentCommand.USERS: await self.get_all_users() elif res.command == LDAPAgentCommand.MACHINES: await self.get_all_machines() elif res.command == LDAPAgentCommand.GROUPS: await self.get_all_groups() elif res.command == LDAPAgentCommand.OUS: await self.get_all_ous() elif res.command == LDAPAgentCommand.GPOS: await self.get_all_gpos() elif res.command == LDAPAgentCommand.SPNSERVICES: await self.get_all_spnservices() elif res.command == LDAPAgentCommand.SCHEMA: await self.get_all_schemaentries() #elif res.command == LDAPAgentCommand.MEMBERSHIPS: # await self.get_all_effective_memberships() elif res.command == LDAPAgentCommand.MEMBERSHIPS: await self.get_effective_memberships(res.data) elif res.command == LDAPAgentCommand.SDS: await self.get_sds(res.data) elif res.command == LDAPAgentCommand.TRUSTS: await self.get_all_trusts() except Exception as e: logger.exception('Agent main!') finally: if self.ldap is not None: await self.ldap._con.disconnect()
async def generate_sd_targets(self): try: subq = self.session.query(JackDawSD.guid).filter(JackDawSD.ad_id == self.ad_id) #total_sds_to_poll += self.session.query(func.count(JackDawADMachine.id)).filter_by(ad_id = self.ad_id).filter(~JackDawADMachine.objectGUID.in_(subq)).scalar() #total_sds_to_poll += self.session.query(func.count(JackDawADGPO.id)).filter_by(ad_id = self.ad_id).filter(~JackDawADGPO.objectGUID.in_(subq)).scalar() #total_sds_to_poll += self.session.query(func.count(JackDawADOU.id)).filter_by(ad_id = self.ad_id).filter(~JackDawADOU.objectGUID.in_(subq)).scalar() #total_sds_to_poll += self.session.query(func.count(JackDawADGroup.id)).filter_by(ad_id = self.ad_id).filter(~JackDawADGroup.guid.in_(subq)).scalar() q = self.session.query(JackDawADUser.dn, JackDawADUser.objectSid, JackDawADUser.objectGUID).filter_by(ad_id = self.ad_id).filter(~JackDawADUser.objectGUID.in_(subq)) await self.resumption_target_gen(q, JackDawADUser.id, 'user', LDAPAgentCommand.SDS) q = self.session.query(JackDawADMachine.dn, JackDawADMachine.objectSid, JackDawADMachine.objectGUID).filter_by(ad_id = self.ad_id).filter(~JackDawADMachine.objectGUID.in_(subq)) await self.resumption_target_gen(q, JackDawADMachine.id, 'machine', LDAPAgentCommand.SDS) q = self.session.query(JackDawADGroup.dn, JackDawADGroup.sid, JackDawADGroup.guid).filter_by(ad_id = self.ad_id).filter(~JackDawADGroup.guid.in_(subq)) await self.resumption_target_gen(q, JackDawADGroup.id, 'group', LDAPAgentCommand.SDS) q = self.session.query(JackDawADOU.dn, JackDawADOU.objectGUID).filter_by(ad_id = self.ad_id).filter(~JackDawADOU.objectGUID.in_(subq)) await self.resumption_target_gen_2(q, JackDawADOU.id, 'ou', LDAPAgentCommand.SDS) q = self.session.query(JackDawADGPO.dn, JackDawADGPO.objectGUID).filter_by(ad_id = self.ad_id).filter(~JackDawADGPO.objectGUID.in_(subq)) await self.resumption_target_gen_2(q, JackDawADGPO.id, 'gpo', LDAPAgentCommand.SDS) logger.debug('generate_sd_targets finished!') except Exception as e: logger.exception('generate_sd_targets')
async def generate_targets(self): try: q = self.session.query(Machine).filter_by(ad_id=self.ad_id) for filter in self.target_filters: if filter == 'live': filter_after = datetime.datetime.today( ) - datetime.timedelta(days=90) q = q.filter(Machine.pwdLastSet >= filter_after) for machine in windowed_query(q, Machine.id, 100): try: dns_name = machine.dNSHostName if dns_name is None or dns_name == '': dns_name = '%s.%s' % (str(machine.sAMAccountName[:-1]), str(self.domain_name)) await self.in_q.put((machine.objectSid, dns_name)) except: continue #signaling the ed of target generation await self.in_q.put(None) except Exception as e: logger.exception('smb generate_targets')
async def generate_sd_targets(self): try: subq = self.session.query( JackDawSD.guid).filter(JackDawSD.ad_id == self.ad_id) q = self.session.query( ADInfo.distinguishedName, ADInfo.objectSid, ADInfo.objectGUID).filter_by( id=self.ad_id).filter(~ADInfo.objectGUID.in_(subq)) await self.resumption_target_gen(q, ADInfo.id, 'domain', LDAPAgentCommand.SDS) q = self.session.query( ADUser.dn, ADUser.objectSid, ADUser.objectGUID).filter_by( ad_id=self.ad_id).filter(~ADUser.objectGUID.in_(subq)) await self.resumption_target_gen(q, ADUser.id, 'user', LDAPAgentCommand.SDS) q = self.session.query( Machine.dn, Machine.objectSid, Machine.objectGUID).filter_by( ad_id=self.ad_id).filter(~Machine.objectGUID.in_(subq)) await self.resumption_target_gen(q, Machine.id, 'machine', LDAPAgentCommand.SDS) q = self.session.query( Group.dn, Group.objectSid, Group.objectGUID).filter_by( ad_id=self.ad_id).filter(~Group.objectGUID.in_(subq)) await self.resumption_target_gen(q, Group.id, 'group', LDAPAgentCommand.SDS) q = self.session.query(ADOU.dn, ADOU.objectGUID).filter_by( ad_id=self.ad_id).filter(~ADOU.objectGUID.in_(subq)) await self.resumption_target_gen_2(q, ADOU.id, 'ou', LDAPAgentCommand.SDS) q = self.session.query(GPO.dn, GPO.objectGUID).filter_by( ad_id=self.ad_id).filter(~GPO.objectGUID.in_(subq)) await self.resumption_target_gen_2(q, GPO.id, 'gpo', LDAPAgentCommand.SDS) logger.debug('generate_sd_targets finished!') except Exception as e: logger.exception('generate_sd_targets')
async def do_load_ad(self, cmd): try: # loads an AD scan and sends all results to the client # sanity check if the AD exists res = self.db_session.query(ADInfo).get(cmd.adid) if res is None: await self.send_error(cmd, 'No AD ID exists with that ID') return #sending machines for computer in self.db_session.query(Machine).filter_by(ad_id = cmd.adid).all(): await asyncio.sleep(0) reply = NestOpComputerRes() reply.token = cmd.token reply.name = computer.sAMAccountName reply.adid = computer.ad_id reply.sid = computer.objectSid reply.domainname = computer.dNSHostName reply.osver = computer.operatingSystem reply.ostype = computer.operatingSystemVersion reply.description = computer.description if computer.UAC_SERVER_TRUST_ACCOUNT is True: reply.computertype = 'DOMAIN_CONTROLLER' elif computer.operatingSystem is not None: if computer.operatingSystem.lower().find('windows') != -1: if computer.operatingSystem.lower().find('server') != -1: reply.computertype = 'SERVER' else: reply.computertype = 'WORKSTATION' else: reply.computertype = 'NIX' else: reply.computertype = 'DUNNO' await self.websocket.send(reply.to_json()) #sending users for user in self.db_session.query(ADUser).filter_by(ad_id = cmd.adid).all(): await asyncio.sleep(0) reply = NestOpUserRes() reply.token = cmd.token reply.name = user.sAMAccountName reply.adid = user.ad_id reply.sid = user.objectSid reply.kerberoast = 1 if user.servicePrincipalName is not None else 0 reply.asreproast = int(user.UAC_DONT_REQUIRE_PREAUTH) reply.nopassw = int(user.UAC_PASSWD_NOTREQD) reply.cleartext = int(user.UAC_ENCRYPTED_TEXT_PASSWORD_ALLOWED) reply.smartcard = int(user.UAC_SMARTCARD_REQUIRED) reply.active = int(user.canLogon) reply.description = user.description if user.adminCount is not None: reply.is_admin = int(user.adminCount) else: reply.is_admin = 0 await self.websocket.send(reply.to_json()) #sending localgroups for lgroup in self.db_session.query(LocalGroup).filter_by(ad_id = cmd.adid).all(): await asyncio.sleep(0) reply = NestOpSMBLocalGroupRes() reply.token = cmd.token reply.adid = lgroup.ad_id reply.machinesid = lgroup.machine_sid reply.usersid = lgroup.sid reply.groupname = lgroup.groupname await self.websocket.send(reply.to_json()) #sending smb shares for share in self.db_session.query(NetShare).filter_by(ad_id = cmd.adid).all(): await asyncio.sleep(0) reply = NestOpSMBShareRes() reply.token = cmd.token reply.adid = share.ad_id reply.machinesid = share.machine_sid reply.netname = share.netname await self.websocket.send(reply.to_json()) #sending smb sessions for session in self.db_session.query(NetSession).filter_by(ad_id = cmd.adid).all(): await asyncio.sleep(0) reply = NestOpSMBSessionRes() reply.token = cmd.token reply.adid = session.ad_id reply.machinesid = session.machine_sid reply.username = session.username await self.websocket.send(reply.to_json()) #sending groups for group in self.db_session.query(Group).filter_by(ad_id = cmd.adid).all(): await asyncio.sleep(0) reply = NestOpGroupRes() reply.token = cmd.token reply.adid = group.ad_id reply.name = group.sAMAccountName reply.dn = group.dn reply.guid = group.objectGUID reply.sid = group.objectSid reply.description = group.description await self.websocket.send(reply.to_json()) await self.send_ok(cmd) except Exception as e: await self.send_error(cmd, "Error! Reason: %s" % e) logger.exception('do_load_ad')
async def do_gather(self, cmd): try: progress_queue = asyncio.Queue() gatheringmonitor_task = asyncio.create_task(self.__gathermonitor(cmd, progress_queue)) ldap_url = cmd.ldap_url if ldap_url == 'auto': if platform.system().lower() == 'windows': from winacl.functions.highlevel import get_logon_info logon = get_logon_info() ldap_url = 'ldap+sspi-ntlm://%s\\%s:jackdaw@%s' % (logon['domain'], logon['username'], logon['logonserver']) else: raise Exception('ldap auto mode selected, but it is not supported on this platform') smb_url = cmd.smb_url if smb_url == 'auto': if platform.system().lower() == 'windows': from winacl.functions.highlevel import get_logon_info logon = get_logon_info() smb_url = 'smb2+sspi-ntlm://%s\\%s:jackdaw@%s' % (logon['domain'], logon['username'], logon['logonserver']) else: raise Exception('smb auto mode selected, but it is not supported on this platform') kerberos_url = cmd.kerberos_url dns = cmd.dns if dns == 'auto': if platform.system().lower() == 'windows': from jackdaw.gatherer.rdns.dnstest import get_correct_dns_win srv_domain = '%s.%s' % (logon['logonserver'], logon['dnsdomainname']) dns = await get_correct_dns_win(srv_domain) if dns is None: dns = None #failed to get dns else: dns = str(dns) else: raise Exception('dns auto mode selected, but it is not supported on this platform') print(ldap_url) print(smb_url) print(dns) with multiprocessing.Pool() as mp_pool: gatherer = Gatherer( self.db_url, self.work_dir, ldap_url, smb_url, kerb_url=kerberos_url, ldap_worker_cnt=int(cmd.ldap_workers), smb_worker_cnt=int(cmd.smb_worker_cnt), mp_pool=mp_pool, smb_gather_types=['all'], progress_queue=progress_queue, show_progress=self.show_progress, calc_edges=True, ad_id=None, dns=dns, stream_data=cmd.stream_data ) res, err = await gatherer.run() if err is not None: print('gatherer returned error') await self.send_error(cmd, str(err)) return #####testing await asyncio.sleep(20) ####### await self.send_ok(cmd) except Exception as e: logger.exception('do_gather') await self.send_error(cmd, str(e)) finally: if gatheringmonitor_task is not None: gatheringmonitor_task.cancel() progress_queue = None
async def __gathermonitor(self, cmd, results_queue): try: usernames_testing = [] machine_sids_testing = [] temp_tok_testing = None temp_adid_testing = None temp_started = False while True: try: msg = await results_queue.get() if msg is None: break #print(msg) if msg.type in STANDARD_PROGRESS_MSG_TYPES: ##### TESTING temp_tok_testing = cmd.token temp_adid_testing = msg.adid ###### reply = NestOpGatherStatus() reply.token = cmd.token reply.current_progress_type = msg.type.value reply.msg_type = msg.msg_type.value reply.adid = msg.adid reply.domain_name = msg.domain_name reply.total = msg.total reply.step_size = msg.step_size reply.basic_running = [] if msg.running is not None: reply.basic_running = [x for x in msg.running] reply.basic_finished = msg.finished reply.smb_errors = msg.errors reply.smb_sessions = msg.sessions reply.smb_shares = msg.shares reply.smb_groups = msg.groups await self.websocket.send(reply.to_json()) ####TESTINGTESTING!!!! if msg.type.value != 'LDAP_BASIC': if temp_started is False: asyncio.create_task(self.spam_sessions(temp_tok_testing, temp_adid_testing, machine_sids_testing, usernames_testing)) temp_started = True elif msg.type == GathererProgressType.USER: usernames_testing.append(msg.data.sAMAccountName) reply = NestOpUserRes() reply.token = cmd.token reply.name = msg.data.sAMAccountName reply.adid = msg.data.ad_id reply.sid = msg.data.objectSid reply.kerberoast = 1 if msg.data.servicePrincipalName is not None else 0 reply.asreproast = int(msg.data.UAC_DONT_REQUIRE_PREAUTH) reply.nopassw = int(msg.data.UAC_PASSWD_NOTREQD) reply.cleartext = int(msg.data.UAC_ENCRYPTED_TEXT_PASSWORD_ALLOWED) reply.smartcard = int(msg.data.UAC_SMARTCARD_REQUIRED) reply.active = int(msg.data.canLogon) reply.description = msg.data.description reply.is_admin = int(msg.data.adminCout) await self.websocket.send(reply.to_json()) elif msg.type == GathererProgressType.MACHINE: machine_sids_testing.append(msg.data.objectSid) reply = NestOpComputerRes() reply.token = cmd.token reply.name = msg.data.sAMAccountName reply.adid = msg.data.ad_id reply.sid = msg.data.objectSid reply.domainname = msg.data.dNSHostName reply.osver = msg.data.operatingSystem reply.ostype = msg.data.operatingSystemVersion reply.description = msg.data.description if msg.data.UAC_SERVER_TRUST_ACCOUNT is True: reply.computertype = 'DOMAIN_CONTROLLER' elif msg.data.operatingSystem is not None: if msg.data.operatingSystem.lower().find('windows') != -1: if msg.data.operatingSystem.lower().find('server') != -1: reply.computertype = 'SERVER' else: reply.computertype = 'WORKSTATION' else: reply.computertype = 'NIX' else: reply.computertype = 'DUNNO' await self.websocket.send(reply.to_json()) elif msg.type == GathererProgressType.SMBLOCALGROUP: reply = NestOpSMBLocalGroupRes() reply.token = cmd.token reply.adid = msg.data.ad_id reply.machinesid = msg.data.machine_sid reply.usersid = msg.data.sid reply.groupname = msg.data.groupname await self.websocket.send(reply.to_json()) elif msg.type == GathererProgressType.SMBSHARE: reply = NestOpSMBShareRes() reply.token = cmd.token reply.adid = msg.data.ad_id reply.machinesid = msg.data.machine_sid reply.netname = msg.data.netname await self.websocket.send(reply.to_json()) elif msg.type == GathererProgressType.SMBSESSION: reply = NestOpSMBSessionRes() reply.token = cmd.token reply.adid = msg.data.ad_id reply.machinesid = msg.data.machine_sid reply.username = msg.data.username await self.websocket.send(reply.to_json()) elif msg.type == GathererProgressType.GROUP: reply = NestOpGroupRes() reply.token = cmd.token reply.adid = msg.data.ad_id reply.name = msg.data.sAMAccountName reply.dn = msg.data.dn reply.guid = msg.data.objectGUID reply.sid = msg.data.objectSid reply.description = msg.data.description await self.websocket.send(reply.to_json()) except asyncio.CancelledError: return except Exception as e: logger.exception('resmon processing error!') #await self.send_error(cmd, str(e)) except asyncio.CancelledError: return except Exception as e: print('resmon died! %s' % e) await self.send_error(cmd, str(e))
def construct(self, construct): """ Fills the network graph from database to memory """ #self.ad_id = ad_id session = self.get_session() adinfo = session.query(JackDawADInfo).get(construct.ad_id) self.domain_sid = str(adinfo.objectSid) #self.calc_acl_edges(session, construct) #adding group nodes logger.debug('Adding group nodes') cnt = 0 for group in adinfo.groups: self.add_sid_to_node(group.sid, 'group', construct, name=group.name) cnt += 1 logger.debug('Added %s group nodes' % cnt) logger.debug('Adding user nodes') cnt = 0 for user in adinfo.users: self.add_sid_to_node(user.objectSid, 'user', construct, name=user.sAMAccountName) cnt += 1 logger.debug('Added %s user nodes' % cnt) logger.debug('Adding machine nodes') cnt = 0 for user in adinfo.computers: self.add_sid_to_node(user.objectSid, 'machine', construct, name=user.sAMAccountName) cnt += 1 logger.debug('Added %s machine nodes' % cnt) logger.debug('Adding hassession edges') cnt = 0 for res in session.query( JackDawADUser.objectSid, JackDawADMachine.objectSid ).filter(NetSession.username == JackDawADUser.sAMAccountName).filter( NetSession.source == JackDawADMachine.sAMAccountName).distinct( NetSession.username): self.add_edge(res[0], res[1], construct, label='hasSession') self.add_edge(res[1], res[0], construct, label='hasSession') cnt += 2 logger.debug('Added %s hassession edges' % cnt) logger.debug('Adding localgroup edges') cnt = 0 for res in session.query( JackDawADUser.objectSid, JackDawADMachine.objectSid, LocalGroup.groupname).filter( JackDawADMachine.id == LocalGroup.machine_id ).filter(JackDawADMachine.ad_id == construct.ad_id).filter( JackDawADUser.ad_id == construct.ad_id).filter( JackDawADUser.objectSid == LocalGroup.sid).all(): label = None if res[2] == 'Remote Desktop Users': label = 'canRDP' weight = 1 elif res[2] == 'Distributed COM Users': label = 'executeDCOM' weight = 1 elif res[2] == 'Administrators': label = 'adminTo' weight = 1 self.add_edge(res[0], res[1], construct, label=label, weight=weight) cnt += 1 logger.debug('Added %s localgroup edges' % cnt) # TODO: implement this! #if self.show_constrained_delegations == True: # pass # TODO: implement this! #if self.show_unconstrained_delegations == True: # pass # TODO: implement this! #for relation in construct.custom_relations: # relation.calc() # self.add_edge(res.sid, res.target_sid) #print('adding membership edges') #adding membership edges logger.debug('Adding membership edges') cnt = 0 q = session.query(JackDawTokenGroup).filter_by(ad_id=construct.ad_id) for tokengroup in windowed_query(q, JackDawTokenGroup.id, 10000): #for tokengroup in adinfo.group_lookups: self.add_sid_to_node(tokengroup.sid, 'unknown', construct) self.add_sid_to_node(tokengroup.member_sid, 'unknown', construct) if tokengroup.is_user == True: try: self.add_edge(tokengroup.sid, tokengroup.member_sid, construct, label='member') cnt += 1 except AssertionError as e: logger.exception() elif tokengroup.is_machine == True: try: self.add_edge(tokengroup.sid, tokengroup.member_sid, construct, label='member') cnt += 1 except AssertionError as e: logger.exception() elif tokengroup.is_group == True: try: self.add_edge(tokengroup.sid, tokengroup.member_sid, construct, label='member') cnt += 1 except AssertionError as e: logger.exception() logger.debug('Added %s membership edges' % cnt) #adding ACL edges #self.calc_acl_edges(session, construct) #self.calc_acl_edges(adinfo, construct) self.calc_acl_edges_mp(session, construct.ad_id, construct) logger.info('Adding password sharing edges') cnt = 0 def get_sid_by_nthash(ad_id, nt_hash): return session.query( JackDawADUser.objectSid, Credential.username).filter_by( ad_id=ad_id).filter(Credential.username == JackDawADUser.sAMAccountName).filter( Credential.nt_hash == nt_hash) dup_nthashes_qry = session.query( Credential.nt_hash).filter(Credential.history_no == 0).filter( Credential.ad_id == construct.ad_id).filter( Credential.username != 'NA').filter( Credential.domain != '<LOCAL>').group_by( Credential.nt_hash).having( func.count(Credential.nt_hash) > 1) for res in dup_nthashes_qry.all(): sidd = {} for sid, _ in get_sid_by_nthash(construct.ad_id, res[0]).all(): sidd[sid] = 1 for sid1 in sidd: for sid2 in sidd: if sid1 == sid2: continue self.add_edge(sid1, sid2, construct, label='pwsharing') cnt += 1 logger.info('Added %s password sharing edges' % cnt)
def all_shortest_paths(self, src_sid=None, dst_sid=None): nv = GraphData() if not src_sid and not dst_sid: raise Exception('Either source or destination MUST be specified') elif not src_sid and dst_sid: try: #for each node we calculate the shortest path to the destination node, silently skip the ones who do not have path to dst inqueue = mp.Queue() outqueue = mp.Queue() procno = mp.cpu_count() logger.debug('[DST_CALC] Starting processes') procs = [ mp.Process(target=short_worker, args=(inqueue, outqueue, self.graph, dst_sid)) for i in range(procno) ] for proc in procs: proc.daemon = True proc.start() logger.debug('[DST_CALC] Starting generator thread') node_gen_th = threading.Thread(target=short_node_gen, args=(self.graph, inqueue, dst_sid, procno)) node_gen_th.daemon = True node_gen_th.start() p_cnt = 0 while True: path = outqueue.get() if path is None: procno -= 1 logger.debug('[DST_CALC] Proc X - Finished!') if procno == 0: break continue self.__add_path(nv, path) p_cnt += 1 logger.debug('[DST_CALC] Found %s paths to dst node %s' % (p_cnt, dst_sid)) logger.debug('[DST_CALC] joining processes') for proc in procs: proc.join() logger.debug('[DST_CALC] Finished!') except Exception as e: logger.exception('[DST_CALC]') elif src_sid and not dst_sid: #for each node we calculate the shortest path to the destination node, silently skip the ones who do not have path to dst for node in self.graph.nodes: if node == src_sid: continue try: for path in nx.all_shortest_paths(self.graph, source=src_sid, target=node): self.__add_path(nv, path) except nx.exception.NetworkXNoPath: continue else: #for each node we calculate the shortest path to the destination node, silently skip the ones who do not have path to dst for path in nx.all_shortest_paths(self.graph, source=src_sid, target=dst_sid): self.__add_path(nv, path) return nv
async def print_progress(self): if self.show_progress is False: try: while True: msg = await self.progress_queue.get() if msg is None: return continue except Exception as e: logger.exception('Progress bar crashed') logger.debug('Setting up progress bars') pos = 0 ldap_info_pbar = tqdm(desc='MSG: ', ascii=True, position=pos) self.progress_bars.append(ldap_info_pbar) pos += 1 if self.ldap_url is not None: ldap_basic_pbar = tqdm(desc='LDAP basic enum ', ascii=True, position=pos) self.progress_bars.append(ldap_basic_pbar) pos += 1 ldap_sd_pbar = tqdm(desc='LDAP SD enum ', ascii=True, position=pos) self.progress_bars.append(ldap_sd_pbar) pos += 1 if self.store_to_db is True: ldap_sdupload_pbar = tqdm(desc='LDAP SD upload ', ascii=True, position=pos) self.progress_bars.append(ldap_sdupload_pbar) pos += 1 ldap_member_pbar = tqdm(desc='LDAP membership enum ', ascii=True, position=pos) self.progress_bars.append(ldap_member_pbar) pos += 1 if self.store_to_db is True: ldap_memberupload_pbar = tqdm(desc='LDAP membership upload', ascii=True, position=pos) self.progress_bars.append(ldap_memberupload_pbar) pos += 1 if self.kerb_url is not None: kerb_pbar = tqdm(desc='KERBEROAST ', ascii=True, position=pos) self.progress_bars.append(kerb_pbar) pos += 1 if self.rdns_resolver is not None: dns_pbar = tqdm(desc='DNS enum ', ascii=True, position=pos) self.progress_bars.append(dns_pbar) pos += 1 if self.smb_url is not None: smb_pbar = tqdm(desc='SMB enum ', ascii=True, position=pos) self.progress_bars.append(smb_pbar) pos += 1 if self.calculate_edges is True: sdcalc_pbar = tqdm(desc='SD edges calc ', ascii=True, position=pos) self.progress_bars.append(sdcalc_pbar) pos += 1 sdcalcupload_pbar = tqdm(desc='SD edges upload ', ascii=True, position=pos) self.progress_bars.append(sdcalcupload_pbar) pos += 1 self.progress_refresh_task = asyncio.create_task( self.progress_refresh()) logger.debug('waiting for progress messages') while True: try: msg = await self.progress_queue.get() if msg is None: for pbar in self.progress_bars: pbar.refresh() return if msg.type == GathererProgressType.BASIC: if msg.msg_type == MSGTYPE.PROGRESS: if ldap_basic_pbar.total is None: ldap_basic_pbar.total = msg.total ldap_basic_pbar.update(msg.step_size) if msg.msg_type == MSGTYPE.FINISHED: ldap_basic_pbar.refresh() elif msg.type == GathererProgressType.SD: if msg.msg_type == MSGTYPE.PROGRESS: if ldap_sd_pbar.total is None: ldap_sd_pbar.total = msg.total ldap_sd_pbar.update(msg.step_size) if msg.msg_type == MSGTYPE.FINISHED: ldap_sd_pbar.refresh() elif msg.type == GathererProgressType.SDUPLOAD: if msg.msg_type == MSGTYPE.PROGRESS: if ldap_sdupload_pbar.total is None: ldap_sdupload_pbar.total = msg.total ldap_sdupload_pbar.update(msg.step_size) if msg.msg_type == MSGTYPE.FINISHED: ldap_sdupload_pbar.refresh() elif msg.type == GathererProgressType.MEMBERS: if msg.msg_type == MSGTYPE.PROGRESS: if ldap_member_pbar.total is None: ldap_member_pbar.total = msg.total ldap_member_pbar.update(msg.step_size) if msg.msg_type == MSGTYPE.FINISHED: ldap_member_pbar.refresh() elif msg.type == GathererProgressType.MEMBERSUPLOAD: if msg.msg_type == MSGTYPE.PROGRESS: if ldap_memberupload_pbar.total is None: ldap_memberupload_pbar.total = msg.total ldap_memberupload_pbar.update(msg.step_size) if msg.msg_type == MSGTYPE.FINISHED: ldap_memberupload_pbar.refresh() elif msg.type == GathererProgressType.KERBEROAST: if msg.msg_type == MSGTYPE.PROGRESS: if kerb_pbar.total is None: kerb_pbar.total = msg.total kerb_pbar.update(msg.step_size) if msg.msg_type == MSGTYPE.FINISHED: kerb_pbar.refresh() elif msg.type == GathererProgressType.DNS: if msg.msg_type == MSGTYPE.PROGRESS: if dns_pbar.total is None: dns_pbar.total = msg.total dns_pbar.update(msg.step_size) if msg.msg_type == MSGTYPE.FINISHED: dns_pbar.refresh() elif msg.type == GathererProgressType.SMB: if msg.msg_type == MSGTYPE.PROGRESS: if smb_pbar.total is None: smb_pbar.total = msg.total smb_pbar.update(msg.step_size) if msg.msg_type == MSGTYPE.FINISHED: smb_pbar.refresh() elif msg.type == GathererProgressType.SDCALC: if msg.msg_type == MSGTYPE.PROGRESS: if sdcalc_pbar.total is None: sdcalc_pbar.total = msg.total sdcalc_pbar.update(msg.step_size) if msg.msg_type == MSGTYPE.FINISHED: sdcalc_pbar.refresh() elif msg.type == GathererProgressType.SDCALCUPLOAD: if msg.msg_type == MSGTYPE.PROGRESS: if sdcalcupload_pbar.total is None: sdcalcupload_pbar.total = msg.total sdcalcupload_pbar.update(msg.step_size) if msg.msg_type == MSGTYPE.FINISHED: sdcalcupload_pbar.refresh() elif msg.type == GathererProgressType.INFO: ldap_info_pbar.display('MSG: %s' % str(msg.text)) elif msg.type == GathererProgressType.REFRESH: for pbar in self.progress_bars: pbar.refresh() except asyncio.CancelledError: return except Exception as e: logger.exception('Progress bar crashed') return