def create(graph_id, dbsession, work_dir, backend_object, db_url): if isinstance(work_dir, str): work_dir = Path(work_dir) sqlite_file = None if db_url.lower().startswith('sqlite') is True: sqlite_file = db_url.replace('sqlite:///', '') logger.error(sqlite_file) gi = dbsession.query(GraphInfo).get(graph_id) graphid = gi.id graph_cache_dir = work_dir.joinpath('graphcache') graph_dir = graph_cache_dir.joinpath(str(gi.id)) try: graph_dir.mkdir(parents=True, exist_ok=False) except Exception as e: logger.warning( 'Graph cache dir with ID %s already exists, skipping! Err %s' % (str(gi.id), str(e))) return graphid backend_object.create(dbsession, str(gi.id), graph_dir, sqlite_file=sqlite_file) return graphid
def create(adids): if len(adids) != 1: logger.warning( 'More than one adid requested, but only one is supported currently!' ) for ad_id in adids: domaininfo = current_app.db.session.query(ADInfo).get(ad_id) domain_sid = domaininfo.objectSid domain_id = domaininfo.id for gi in current_app.db.session.query(GraphInfo).filter_by( ad_id=domain_id).all(): graphid = gi.id graph_cache_dir = current_app.config['JACKDAW_WORK_DIR'].joinpath( 'graphcache') graph_dir = graph_cache_dir.joinpath(str(gi.id)) try: graph_dir.mkdir(parents=True, exist_ok=False) except Exception as e: logger.warning( 'Graph cache dir with ID %s already exists, skipping! Err %s' % (str(gi.id), str(e))) continue current_app.config.get('JACKDAW_GRAPH_BACKEND_OBJ').create( current_app.db.session, domain_id, str(gi.id), graph_dir) #TODO: fix this, need noew UI to handle the logic :( return {'graphid': graphid}
def create(dbsession, graph_id, graph_dir, sqlite_file = None): logger.info('Create called!') graph_id = int(graph_id) graph_file = graph_dir.joinpath(JackDawDomainGraphIGraph.graph_file_name) logger.debug('Creating a new graph file: %s' % graph_file) adids = dbsession.query(GraphInfoAD.ad_id).filter_by(graph_id = graph_id).all() if adids is None: raise Exception('No ADIDS were found for graph %s' % graph_id) using_sqlite_tool = False if sqlite_file is not None: logger.info('Trying sqlite3 dumping method...') # This is a hack. # Problem: using sqlalchemy to dump a large table (to get the graph data file) is extremely resource intensive # Solution: if sqlite is used as the database backend we can use the sqlite3 cmdline utility to do the dumping much faster # sf = str(sqlite_file) gf = str(graph_file) if platform.system() == 'Windows': sf = sf.replace('\\', '\\\\') gf = gf.replace('\\', '\\\\') qry_str = '.open %s\r\n.mode csv\r\n.output %s\r\n.separator " "\r\nSELECT src,dst FROM adedges, adedgelookup WHERE adedges.graph_id = %s AND adedgelookup.id = adedges.src AND adedgelookup.oid IS NOT NULL;\r\n.exit' % (sf, gf, graph_id) with open('buildnode.sql', 'w', newline='') as f: f.write(qry_str) import subprocess import shlex cmd = 'cat buildnode.sql | sqlite3' if platform.system() == 'Windows': cmd = 'type buildnode.sql | sqlite3' process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) _, stderr = process.communicate() process.wait() if process.returncode == 0: using_sqlite_tool = True logger.info('sqlite3 dumping method OK!') else: logger.warning('Failed to use the sqlite3 tool to speed up graph datafile generation. Reason: %s' % stderr) if using_sqlite_tool is False: for ad_id in adids: ad_id = ad_id[0] t2 = dbsession.query(func.count(Edge.id)).filter_by(graph_id = graph_id).filter(EdgeLookup.id == Edge.src).filter(EdgeLookup.oid != None).scalar() q = dbsession.query(Edge).filter_by(graph_id = graph_id).filter(EdgeLookup.id == Edge.src).filter(EdgeLookup.oid != None) with open(graph_file, 'w', newline = '') as f: for edge in tqdm(windowed_query(q,Edge.id, 10000), desc = 'edge', total = t2): r = '%s %s\r\n' % (edge.src, edge.dst) f.write(r) logger.info('Graphcache file created!')
def check_jobs(self, finished_type): self.session.commit() if finished_type is not None: logger.debug('%s enumeration finished!' % MSLDAP_JOB_TYPES_INV[finished_type]) del self.running_enums[MSLDAP_JOB_TYPES_INV[finished_type]] self.finished_enums.append(MSLDAP_JOB_TYPES_INV[finished_type]) lr = len(self.running_enums) if self.enum_types_len == len(self.finished_enums): #everything finished return True if lr == self.agent_cnt: #enums still running with max performance return False if lr < self.agent_cnt: #we can start a new enum for _ in range(self.agent_cnt - lr): if len(self.enum_types) > 0: next_type = self.enum_types.pop(0) else: return False if next_type == 'adinfo': self.enum_domain() #this must be the first! self.running_enums[next_type] = 1 return False elif next_type == 'users': self.enum_users() elif next_type == 'machines': self.enum_machines() elif next_type == 'sds': self.enum_sds() elif next_type == 'memberships': self.enum_memberships() elif next_type == 'ous': self.enum_ous() elif next_type == 'gpos': self.enum_gpos() elif next_type == 'groups': self.enum_groups() elif next_type == 'spns': self.enum_spnservices() elif next_type == 'trusts': self.enum_trusts() else: logger.warning('Unknown next_type! %s' % next_type) self.running_enums[next_type] = 1 return False
def create(adids): if len(adids) != 1: logger.warning( 'More than one adid requested, but only one is supported currently!' ) sqlite_file = None if current_app.config['SQLALCHEMY_DATABASE_URI'].lower().startswith( 'sqlite') is True: sqlite_file = current_app.config['SQLALCHEMY_DATABASE_URI'].replace( 'sqlite:///', '') logger.error(sqlite_file) for ad_id in adids: domaininfo = current_app.db.session.query(ADInfo).get(ad_id) domain_sid = domaininfo.objectSid domain_id = domaininfo.id res = current_app.db.session.query(GraphInfoAD).filter_by( ad_id=ad_id).first() gi = current_app.db.session.query(GraphInfo).get(res.graph_id) graphid = gi.id graph_cache_dir = current_app.config['JACKDAW_WORK_DIR'].joinpath( 'graphcache') graph_dir = graph_cache_dir.joinpath(str(gi.id)) try: graph_dir.mkdir(parents=True, exist_ok=False) except Exception as e: logger.warning( 'Graph cache dir with ID %s already exists, skipping! Err %s' % (str(gi.id), str(e))) continue current_app.config.get('JACKDAW_GRAPH_BACKEND_OBJ').create( current_app.db.session, str(gi.id), graph_dir, sqlite_file=sqlite_file) #TODO: fix this, need noew UI to handle the logic :( return {'graphid': graphid}
async def run(self): try: adinfo = self.session.query(ADInfo).get(self.ad_id) self.domain_name = str(adinfo.distinguishedName).replace( ',', '.').replace('DC=', '') qs = self.agent_cnt self.agent_in_q = asyncio.Queue(qs) self.agent_out_q = asyncio.Queue(qs * 40) self.token_file_path = os.path.join( str(self.work_dir), 'token_' + datetime.datetime.utcnow().strftime("%Y%m%d_%H%M%S") + '.gzip') self.token_file = gzip.GzipFile(self.token_file_path, 'w') logger.debug('Polling members') _, res = await self.prepare_targets() if res is not None: raise res for _ in range(self.agent_cnt): agent = LDAPGathererAgent(self.ldap_mgr, self.agent_in_q, self.agent_out_q) self.agents.append(asyncio.create_task(agent.arun())) asyncio.create_task(self.start_jobs()) if self.progress_queue is None: self.member_progress = tqdm(desc='Collecting members', total=self.total_members_to_poll, position=0, leave=True) else: msg = GathererProgress() msg.type = GathererProgressType.MEMBERS msg.msg_type = MSGTYPE.STARTED msg.adid = self.ad_id msg.domain_name = self.domain_name await self.progress_queue.put(msg) await asyncio.sleep(0) acnt = self.total_members_to_poll last_stat_cnt = 0 while acnt > 0: try: res = await self.agent_out_q.get() res_type, res = res if res_type == LDAPAgentCommand.MEMBERSHIP: self.member_finish_ctr += 1 res.ad_id = self.ad_id res.graph_id = self.graph_id self.token_file.write(res.to_json().encode() + b'\r\n') await asyncio.sleep(0) elif res_type == LDAPAgentCommand.MEMBERSHIP_FINISHED: if self.show_progress is True: self.member_progress.update() if acnt % self.progress_step_size == 0 and self.progress_queue is not None: last_stat_cnt += self.progress_step_size now = datetime.datetime.utcnow() td = (now - self.progress_last_updated).total_seconds() self.progress_last_updated = now msg = GathererProgress() msg.type = GathererProgressType.MEMBERS msg.msg_type = MSGTYPE.PROGRESS msg.adid = self.ad_id msg.domain_name = self.domain_name msg.total = self.total_members_to_poll msg.total_finished = self.total_members_to_poll - acnt if td > 0: msg.speed = str(self.progress_step_size // td) msg.step_size = self.progress_step_size await self.progress_queue.put(msg) acnt -= 1 elif res_type == LDAPAgentCommand.EXCEPTION: logger.warning(str(res)) except Exception as e: logger.exception('Members enumeration error!') raise e if self.progress_queue is not None: now = datetime.datetime.utcnow() td = (now - self.progress_last_updated).total_seconds() self.progress_last_updated = now msg = GathererProgress() msg.type = GathererProgressType.MEMBERS msg.msg_type = MSGTYPE.PROGRESS msg.adid = self.ad_id msg.domain_name = self.domain_name msg.total = self.total_members_to_poll msg.total_finished = self.total_members_to_poll if td > 0: msg.speed = str( (self.total_members_to_poll - last_stat_cnt) // td) msg.step_size = (self.total_members_to_poll - last_stat_cnt) await self.progress_queue.put(msg) await self.stop_memberships_collection() adinfo = self.session.query(ADInfo).get(self.ad_id) adinfo.ldap_members_finished = True self.session.commit() return True, None except Exception as e: logger.exception('Members enumeration error main!') await self.stop_memberships_collection() return False, e finally: try: self.session.close() except: pass
def run(self): logger.info( '[+] Starting LDAP information acqusition. This might take a while...' ) self.setup() logger.debug('setup finished!') self.domaininfo_ctr += 1 job = LDAPAgentJob(LDAPAgentCommand.DOMAININFO, None) self.agent_in_q.put(job) while True: res = self.agent_out_q.get() #print(res) res_type, res = res if res_type == LDAPAgentCommand.DOMAININFO: self.enum_domain(res) elif res_type == LDAPAgentCommand.USER: self.enum_user(res) elif res_type == LDAPAgentCommand.MACHINE: self.enum_machine(res) elif res_type == LDAPAgentCommand.SPNSERVICE: self.store_spnservice(res) elif res_type == LDAPAgentCommand.GROUP: self.enum_group(res) elif res_type == LDAPAgentCommand.OU: self.enum_ou(res) elif res_type == LDAPAgentCommand.SD: self.store_sd(res) elif res_type == LDAPAgentCommand.GPO: self.enum_gpo(res) elif res_type == LDAPAgentCommand.MEMBERSHIP: self.store_membership(res) elif res_type == LDAPAgentCommand.EXCEPTION: logger.warning(res) elif res_type == LDAPAgentCommand.SPNSERVICES_FINISHED: logger.debug('SPN enumeration finished!') self.spn_finish_ctr += 1 elif res_type == LDAPAgentCommand.USERS_FINISHED: logger.debug('Users enumeration finished!') self.user_finish_ctr += 1 elif res_type == LDAPAgentCommand.MACHINES_FINISHED: logger.debug('Machines enumeration finished!') self.machine_finish_ctr += 1 elif res_type == LDAPAgentCommand.OUS_FINISHED: logger.debug('OUs enumeration finished!') self.ou_finish_ctr += 1 elif res_type == LDAPAgentCommand.GROUPS_FINISHED: logger.debug('Groups enumeration finished!') self.group_finish_ctr += 1 elif res_type == LDAPAgentCommand.MEMBERSHIPS_FINISHED: logger.debug('Memberships enumeration finished!') self.member_finish_ctr += 1 elif res_type == LDAPAgentCommand.SDS_FINISHED: logger.debug('Secuirty Descriptor enumeration finished!') self.sd_finish_ctr += 1 elif res_type == LDAPAgentCommand.DOMAININFO_FINISHED: logger.debug('Domaininfo enumeration finished!') self.domaininfo_finish_ctr += 1 elif res_type == LDAPAgentCommand.GPOS_FINISHED: logger.debug('GPOs enumeration finished!') self.gpo_finish_ctr += 1 if self.check_status() == True: break self.stop_agents() logger.info('[+] LDAP information acqusition finished!') return self.ad_id
def run(self): logger.info('[+] Starting LDAP information acqusition. This might take a while...') self.setup() logger.debug('setup finished!') self.check_jobs(None) while True: res = self.agent_out_q.get() self.update_progress() res_type, res = res if res_type == LDAPAgentCommand.DOMAININFO: self.domaininfo_ctr += 1 self.store_domain(res) elif res_type == LDAPAgentCommand.USER: self.user_ctr += 1 self.store_user(res) elif res_type == LDAPAgentCommand.MACHINE: self.machine_ctr += 1 self.store_machine(res) elif res_type == LDAPAgentCommand.GROUP: self.group_ctr += 1 self.store_group(res) elif res_type == LDAPAgentCommand.OU: self.ou_ctr += 1 self.store_ous(res) elif res_type == LDAPAgentCommand.GPO: self.gpo_ctr += 1 self.store_gpo(res) elif res_type == LDAPAgentCommand.SPNSERVICE: self.spn_ctr += 1 self.store_spn(res) elif res_type == LDAPAgentCommand.SD: self.sd_ctr += 1 self.store_sd(res) elif res_type == LDAPAgentCommand.MEMBERSHIP: self.member_ctr += 1 self.store_membership(res) elif res_type == LDAPAgentCommand.TRUSTS: self.trust_ctr += 1 self.store_trust(res) elif res_type == LDAPAgentCommand.EXCEPTION: logger.warning(str(res)) elif res_type.name.endswith('FINISHED'): if self.check_jobs(res_type) is True: break self.stop_agents() logger.info('[+] LDAP information acqusition finished!') return self.ad_id
def run(self): self.setup() logger.info('setup finished!') self.domaininfo_ctr += 1 job = LDAPAgentJob(LDAPAgentCommand.DOMAININFO, None) self.agent_in_q.put(job) while True: res = self.agent_out_q.get() #print(res) res_type, res = res if res_type == LDAPAgentCommand.DOMAININFO: self.enum_domain(res) elif res_type == LDAPAgentCommand.USER: self.enum_user(res) elif res_type == LDAPAgentCommand.MACHINE: self.enum_machine(res) elif res_type == LDAPAgentCommand.SPNSERVICE: self.store_spnservice(res) elif res_type == LDAPAgentCommand.GROUP: self.enum_group(res) elif res_type == LDAPAgentCommand.OU: self.enum_ou(res) elif res_type == LDAPAgentCommand.SD: self.store_sd(res) elif res_type == LDAPAgentCommand.MEMBERSHIP: self.store_membership(res) elif res_type == LDAPAgentCommand.EXCEPTION: logger.warning(res) elif res_type == LDAPAgentCommand.SPNSERVICES_FINISHED: self.spn_finish_ctr += 1 elif res_type == LDAPAgentCommand.USERS_FINISHED: self.user_finish_ctr += 1 elif res_type == LDAPAgentCommand.MACHINES_FINISHED: self.machine_finish_ctr += 1 elif res_type == LDAPAgentCommand.OUS_FINISHED: self.ou_finish_ctr += 1 elif res_type == LDAPAgentCommand.GROUPS_FINISHED: self.group_finish_ctr += 1 elif res_type == LDAPAgentCommand.MEMBERSHIPS_FINISHED: self.member_finish_ctr += 1 elif res_type == LDAPAgentCommand.SDS_FINISHED: self.sd_finish_ctr += 1 elif res_type == LDAPAgentCommand.DOMAININFO_FINISHED: self.domaininfo_finish_ctr += 1 if self.check_status() == True: break self.stop_agents() return self.ad_id
async def run(self): logger.info( '[+] Starting LDAP information acqusition. This might take a while...' ) await self.setup() if self.progress_queue is not None: msg = LDAPEnumeratorProgress() msg.type = 'LDAP' msg.msg_type = 'STARTED' msg.adid = self.ad_id msg.domain_name = self.domain_name await self.progress_queue.put(msg) await self.check_jobs(None) while True: try: res = await self.agent_out_q.get() await self.update_progress() res_type, res = res if res_type == LDAPAgentCommand.DOMAININFO: self.domaininfo_ctr += 1 await self.store_domain(res) elif res_type == LDAPAgentCommand.USER: self.user_ctr += 1 await self.store_user(res) elif res_type == LDAPAgentCommand.MACHINE: self.machine_ctr += 1 await self.store_machine(res) elif res_type == LDAPAgentCommand.GROUP: self.group_ctr += 1 await self.store_group(res) elif res_type == LDAPAgentCommand.OU: self.ou_ctr += 1 await self.store_ous(res) elif res_type == LDAPAgentCommand.GPO: self.gpo_ctr += 1 await self.store_gpo(res) elif res_type == LDAPAgentCommand.SPNSERVICE: self.spn_ctr += 1 await self.store_spn(res) elif res_type == LDAPAgentCommand.SD: self.sd_ctr += 1 await self.store_sd(res) elif res_type == LDAPAgentCommand.MEMBERSHIP: self.member_ctr += 1 await self.store_membership(res) elif res_type == LDAPAgentCommand.TRUSTS: self.trust_ctr += 1 await self.store_trust(res) elif res_type == LDAPAgentCommand.EXCEPTION: logger.warning(str(res)) elif res_type.name.endswith('FINISHED'): t = await self.check_jobs(res_type) if t is True: break except Exception as e: logger.exception('ldap enumerator main!') await self.stop_agents() return None await self.stop_agents() logger.info('[+] LDAP information acqusition finished!') return self.ad_id
async def run(self): logger.debug('Basecollector started!') qs = self.agent_cnt self.agent_in_q = asyncio.Queue() #AsyncProcessQueue() self.agent_out_q = asyncio.Queue(qs) #AsyncProcessQueue(1000) if self.show_progress is True: self.total_progress = tqdm(desc='LDAP info entries', ascii=True) for _ in range(self.agent_cnt): agent = LDAPGathererAgent(self.ldap_mgr, self.agent_in_q, self.agent_out_q) self.agents.append(asyncio.create_task(agent.arun())) if self.progress_queue is not None: msg = GathererProgress() msg.type = GathererProgressType.BASIC msg.msg_type = MSGTYPE.STARTED msg.adid = self.ad_id msg.domain_name = self.domain_name await self.progress_queue.put(msg) await self.check_jobs(None) logger.debug('basecollector setup complete!') try: while True: res = await self.agent_out_q.get() await self.update_progress() res_type, res = res if res_type == LDAPAgentCommand.DOMAININFO: self.domaininfo_ctr += 1 await self.store_domain(res) elif res_type == LDAPAgentCommand.USER: self.user_ctr += 1 await self.store_user(res) if self.user_ctr % 1000 == 0: self.session.commit() elif res_type == LDAPAgentCommand.MACHINE: self.machine_ctr += 1 await self.store_machine(res) if self.machine_ctr % 1000 == 0: self.session.commit() elif res_type == LDAPAgentCommand.GROUP: self.group_ctr += 1 await self.store_group(res) if self.group_ctr % 1000 == 0: self.session.commit() elif res_type == LDAPAgentCommand.OU: self.ou_ctr += 1 await self.store_ous(res) elif res_type == LDAPAgentCommand.GPO: self.gpo_ctr += 1 await self.store_gpo(res) elif res_type == LDAPAgentCommand.SPNSERVICE: self.spn_ctr += 1 await self.store_spn(res) elif res_type == LDAPAgentCommand.TRUSTS: self.trust_ctr += 1 await self.store_trust(res) elif res_type == LDAPAgentCommand.EXCEPTION: logger.warning(str(res)) elif res_type.name.endswith('FINISHED'): t = await self.check_jobs(res_type) if t is True: break return self.ad_id, self.graph_id, None except Exception as e: logger.exception('ldap enumerator main!') return None, None, e finally: await self.stop_agents()
async def run(self): try: adinfo = self.session.query(ADInfo).get(self.ad_id) self.domain_name = str(adinfo.distinguishedName).replace( ',', '.').replace('DC=', '') qs = self.agent_cnt self.agent_in_q = asyncio.Queue(qs) #AsyncProcessQueue() self.agent_out_q = asyncio.Queue(qs) #AsyncProcessQueue(1000) self.sd_file_path = 'sd_' + datetime.datetime.utcnow().strftime( "%Y%m%d_%H%M%S") + '.gzip' self.sd_file = gzip.GzipFile(self.sd_file_path, 'w') logger.debug('Polling sds') _, res = await self.prepare_targets() if res is not None: raise res for _ in range(self.agent_cnt): agent = LDAPGathererAgent(self.ldap_mgr, self.agent_in_q, self.agent_out_q) self.agents.append(asyncio.create_task(agent.arun())) asyncio.create_task(self.start_jobs()) if self.show_progress is True: self.sds_progress = tqdm(desc='Collecting SDs', total=self.total_targets, position=0, leave=True) if self.progress_queue is not None: msg = GathererProgress() msg.type = GathererProgressType.SD msg.msg_type = MSGTYPE.STARTED msg.adid = self.ad_id msg.domain_name = self.domain_name await self.progress_queue.put(msg) acnt = self.total_targets last_stat_cnt = 0 while acnt > 0: try: res = await self.agent_out_q.get() res_type, res = res if res_type == LDAPAgentCommand.SD: await self.store_sd(res) if self.show_progress is True: self.sds_progress.update() if self.progress_queue is not None: if acnt % self.progress_step_size == 0: last_stat_cnt += self.progress_step_size now = datetime.datetime.utcnow() td = (now - self.progress_last_updated ).total_seconds() self.progress_last_updated = now msg = GathererProgress() msg.type = GathererProgressType.SD msg.msg_type = MSGTYPE.PROGRESS msg.adid = self.ad_id msg.domain_name = self.domain_name msg.total = self.total_targets msg.total_finished = self.total_targets - acnt if td > 0: msg.speed = str(self.progress_step_size // td) msg.step_size = self.progress_step_size await self.progress_queue.put(msg) elif res_type == LDAPAgentCommand.EXCEPTION: logger.warning(str(res)) acnt -= 1 except Exception as e: logger.exception('SDs enumeration error!') raise e if self.progress_queue is not None: now = datetime.datetime.utcnow() td = (now - self.progress_last_updated).total_seconds() self.progress_last_updated = now msg = GathererProgress() msg.type = GathererProgressType.SD msg.msg_type = MSGTYPE.PROGRESS msg.adid = self.ad_id msg.domain_name = self.domain_name msg.total = self.total_targets msg.total_finished = self.total_targets if td > 0: msg.speed = str((self.total_targets - last_stat_cnt) // td) msg.step_size = self.total_targets - last_stat_cnt await self.progress_queue.put(msg) adinfo = self.session.query(ADInfo).get(self.ad_id) adinfo.ldap_sds_finished = True self.session.commit() return True, None except Exception as e: logger.exception('SDs enumeration main error') if self.progress_queue is not None: msg = GathererProgress() msg.type = GathererProgressType.SD msg.msg_type = MSGTYPE.ERROR msg.adid = self.ad_id msg.domain_name = self.domain_name msg.error = e await self.progress_queue.put(msg) return False, e finally: await self.stop_sds_collection() try: self.session.close() except: pass
async def run(self): logger.info('[+] Starting LDAP information acqusition. This might take a while...') await self.setup() if self.resumption is False: res = await self.run_init_gathering() self.session.commit() if res is None: return False try: logger.debug('Polling sds') total_sds_to_poll = 0 subq = self.session.query(JackDawSD.guid).filter(JackDawSD.ad_id == self.ad_id) total_sds_to_poll += self.session.query(func.count(JackDawADUser.id)).filter_by(ad_id = self.ad_id).filter(~JackDawADUser.objectGUID.in_(subq)).scalar() total_sds_to_poll += self.session.query(func.count(JackDawADMachine.id)).filter_by(ad_id = self.ad_id).filter(~JackDawADMachine.objectGUID.in_(subq)).scalar() total_sds_to_poll += self.session.query(func.count(JackDawADGPO.id)).filter_by(ad_id = self.ad_id).filter(~JackDawADGPO.objectGUID.in_(subq)).scalar() total_sds_to_poll += self.session.query(func.count(JackDawADOU.id)).filter_by(ad_id = self.ad_id).filter(~JackDawADOU.objectGUID.in_(subq)).scalar() total_sds_to_poll += self.session.query(func.count(JackDawADGroup.id)).filter_by(ad_id = self.ad_id).filter(~JackDawADGroup.guid.in_(subq)).scalar() asyncio.create_task(self.generate_sd_targets()) sds_p = tqdm(desc='Collecting SDs', total=total_sds_to_poll) logger.info(total_sds_to_poll) acnt = total_sds_to_poll while acnt > 0: try: res = await self.agent_out_q.get() res_type, res = res if res_type == LDAPAgentCommand.SD: sds_p.update() await self.store_sd(res) elif res_type == LDAPAgentCommand.EXCEPTION: logger.warning(str(res)) acnt -= 1 except Exception as e: logger.exception('SDs enumeration error!') raise e except Exception as e: logger.exception('SDs enumeration main error') await self.stop_sds_collection(sds_p) return None await self.stop_sds_collection(sds_p) try: logger.debug('Polling members') total_members_to_poll = 0 subq = self.session.query(JackDawTokenGroup.guid).distinct(JackDawTokenGroup.guid).filter(JackDawTokenGroup.ad_id == self.ad_id) total_members_to_poll += self.session.query(func.count(JackDawADUser.id)).filter_by(ad_id = self.ad_id).filter(~JackDawADUser.objectGUID.in_(subq)).scalar() total_members_to_poll += self.session.query(func.count(JackDawADMachine.id)).filter_by(ad_id = self.ad_id).filter(~JackDawADMachine.objectGUID.in_(subq)).scalar() total_members_to_poll += self.session.query(func.count(JackDawADGroup.id)).filter_by(ad_id = self.ad_id).filter(~JackDawADGroup.guid.in_(subq)).scalar() asyncio.create_task(self.generate_member_targets()) member_p = tqdm(desc='Collecting members', total=total_members_to_poll) acnt = total_members_to_poll while acnt > 0: try: res = await self.agent_out_q.get() res_type, res = res if res_type == LDAPAgentCommand.MEMBERSHIP: res.ad_id = self.ad_id self.token_file.write(res.to_json().encode() + b'\r\n') elif res_type == LDAPAgentCommand.MEMBERSHIP_FINISHED: member_p.update() acnt -= 1 elif res_type == LDAPAgentCommand.EXCEPTION: logger.warning(str(res)) except Exception as e: logger.exception('Members enumeration error!') raise e except Exception as e: logger.exception('Members enumeration error main!') await self.stop_memberships_collection(member_p) return None await self.stop_memberships_collection(member_p) await self.stop_agents() logger.info('[+] LDAP information acqusition finished!') return self.ad_id