def create(graph_id, dbsession, work_dir, backend_object, db_url):
if isinstance(work_dir, str):
work_dir = Path(work_dir)
sqlite_file = None
if db_url.lower().startswith('sqlite') is True:
sqlite_file = db_url.replace('sqlite:///', '')
logger.error(sqlite_file)
gi = dbsession.query(GraphInfo).get(graph_id)
graphid = gi.id
graph_cache_dir = work_dir.joinpath('graphcache')
graph_dir = graph_cache_dir.joinpath(str(gi.id))
try:
graph_dir.mkdir(parents=True, exist_ok=False)
except Exception as e:
logger.warning(
'Graph cache dir with ID %s already exists, skipping! Err %s' %
(str(gi.id), str(e)))
return graphid
backend_object.create(dbsession,
str(gi.id),
graph_dir,
sqlite_file=sqlite_file)
return graphid
def create(adids):
if len(adids) != 1:
logger.warning(
'More than one adid requested, but only one is supported currently!'
)
for ad_id in adids:
domaininfo = current_app.db.session.query(ADInfo).get(ad_id)
domain_sid = domaininfo.objectSid
domain_id = domaininfo.id
for gi in current_app.db.session.query(GraphInfo).filter_by(
ad_id=domain_id).all():
graphid = gi.id
graph_cache_dir = current_app.config['JACKDAW_WORK_DIR'].joinpath(
'graphcache')
graph_dir = graph_cache_dir.joinpath(str(gi.id))
try:
graph_dir.mkdir(parents=True, exist_ok=False)
except Exception as e:
logger.warning(
'Graph cache dir with ID %s already exists, skipping! Err %s'
% (str(gi.id), str(e)))
continue
current_app.config.get('JACKDAW_GRAPH_BACKEND_OBJ').create(
current_app.db.session, domain_id, str(gi.id), graph_dir)
#TODO: fix this, need noew UI to handle the logic :(
return {'graphid': graphid}
def create(dbsession, graph_id, graph_dir, sqlite_file = None):
logger.info('Create called!')
graph_id = int(graph_id)
graph_file = graph_dir.joinpath(JackDawDomainGraphIGraph.graph_file_name)
logger.debug('Creating a new graph file: %s' % graph_file)
adids = dbsession.query(GraphInfoAD.ad_id).filter_by(graph_id = graph_id).all()
if adids is None:
raise Exception('No ADIDS were found for graph %s' % graph_id)
using_sqlite_tool = False
if sqlite_file is not None:
logger.info('Trying sqlite3 dumping method...')
# This is a hack.
# Problem: using sqlalchemy to dump a large table (to get the graph data file) is extremely resource intensive
# Solution: if sqlite is used as the database backend we can use the sqlite3 cmdline utility to do the dumping much faster
#
sf = str(sqlite_file)
gf = str(graph_file)
if platform.system() == 'Windows':
sf = sf.replace('\\', '\\\\')
gf = gf.replace('\\', '\\\\')
qry_str = '.open %s\r\n.mode csv\r\n.output %s\r\n.separator " "\r\nSELECT src,dst FROM adedges, adedgelookup WHERE adedges.graph_id = %s AND adedgelookup.id = adedges.src AND adedgelookup.oid IS NOT NULL;\r\n.exit' % (sf, gf, graph_id)
with open('buildnode.sql', 'w', newline='') as f:
f.write(qry_str)
import subprocess
import shlex
cmd = 'cat buildnode.sql | sqlite3'
if platform.system() == 'Windows':
cmd = 'type buildnode.sql | sqlite3'
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
_, stderr = process.communicate()
process.wait()
if process.returncode == 0:
using_sqlite_tool = True
logger.info('sqlite3 dumping method OK!')
else:
logger.warning('Failed to use the sqlite3 tool to speed up graph datafile generation. Reason: %s' % stderr)
if using_sqlite_tool is False:
for ad_id in adids:
ad_id = ad_id[0]
t2 = dbsession.query(func.count(Edge.id)).filter_by(graph_id = graph_id).filter(EdgeLookup.id == Edge.src).filter(EdgeLookup.oid != None).scalar()
q = dbsession.query(Edge).filter_by(graph_id = graph_id).filter(EdgeLookup.id == Edge.src).filter(EdgeLookup.oid != None)
with open(graph_file, 'w', newline = '') as f:
for edge in tqdm(windowed_query(q,Edge.id, 10000), desc = 'edge', total = t2):
r = '%s %s\r\n' % (edge.src, edge.dst)
f.write(r)
logger.info('Graphcache file created!')
def check_jobs(self, finished_type):
self.session.commit()
if finished_type is not None:
logger.debug('%s enumeration finished!' %
MSLDAP_JOB_TYPES_INV[finished_type])
del self.running_enums[MSLDAP_JOB_TYPES_INV[finished_type]]
self.finished_enums.append(MSLDAP_JOB_TYPES_INV[finished_type])
lr = len(self.running_enums)
if self.enum_types_len == len(self.finished_enums):
#everything finished
return True
if lr == self.agent_cnt:
#enums still running with max performance
return False
if lr < self.agent_cnt:
#we can start a new enum
for _ in range(self.agent_cnt - lr):
if len(self.enum_types) > 0:
next_type = self.enum_types.pop(0)
else:
return False
if next_type == 'adinfo':
self.enum_domain()
#this must be the first!
self.running_enums[next_type] = 1
return False
elif next_type == 'users':
self.enum_users()
elif next_type == 'machines':
self.enum_machines()
elif next_type == 'sds':
self.enum_sds()
elif next_type == 'memberships':
self.enum_memberships()
elif next_type == 'ous':
self.enum_ous()
elif next_type == 'gpos':
self.enum_gpos()
elif next_type == 'groups':
self.enum_groups()
elif next_type == 'spns':
self.enum_spnservices()
elif next_type == 'trusts':
self.enum_trusts()
else:
logger.warning('Unknown next_type! %s' % next_type)
self.running_enums[next_type] = 1
return False
def create(adids):
if len(adids) != 1:
logger.warning(
'More than one adid requested, but only one is supported currently!'
)
sqlite_file = None
if current_app.config['SQLALCHEMY_DATABASE_URI'].lower().startswith(
'sqlite') is True:
sqlite_file = current_app.config['SQLALCHEMY_DATABASE_URI'].replace(
'sqlite:///', '')
logger.error(sqlite_file)
for ad_id in adids:
domaininfo = current_app.db.session.query(ADInfo).get(ad_id)
domain_sid = domaininfo.objectSid
domain_id = domaininfo.id
res = current_app.db.session.query(GraphInfoAD).filter_by(
ad_id=ad_id).first()
gi = current_app.db.session.query(GraphInfo).get(res.graph_id)
graphid = gi.id
graph_cache_dir = current_app.config['JACKDAW_WORK_DIR'].joinpath(
'graphcache')
graph_dir = graph_cache_dir.joinpath(str(gi.id))
try:
graph_dir.mkdir(parents=True, exist_ok=False)
except Exception as e:
logger.warning(
'Graph cache dir with ID %s already exists, skipping! Err %s' %
(str(gi.id), str(e)))
continue
current_app.config.get('JACKDAW_GRAPH_BACKEND_OBJ').create(
current_app.db.session,
str(gi.id),
graph_dir,
sqlite_file=sqlite_file)
#TODO: fix this, need noew UI to handle the logic :(
return {'graphid': graphid}
async def run(self):
try:
adinfo = self.session.query(ADInfo).get(self.ad_id)
self.domain_name = str(adinfo.distinguishedName).replace(
',', '.').replace('DC=', '')
qs = self.agent_cnt
self.agent_in_q = asyncio.Queue(qs)
self.agent_out_q = asyncio.Queue(qs * 40)
self.token_file_path = os.path.join(
str(self.work_dir), 'token_' +
datetime.datetime.utcnow().strftime("%Y%m%d_%H%M%S") + '.gzip')
self.token_file = gzip.GzipFile(self.token_file_path, 'w')
logger.debug('Polling members')
_, res = await self.prepare_targets()
if res is not None:
raise res
for _ in range(self.agent_cnt):
agent = LDAPGathererAgent(self.ldap_mgr, self.agent_in_q,
self.agent_out_q)
self.agents.append(asyncio.create_task(agent.arun()))
asyncio.create_task(self.start_jobs())
if self.progress_queue is None:
self.member_progress = tqdm(desc='Collecting members',
total=self.total_members_to_poll,
position=0,
leave=True)
else:
msg = GathererProgress()
msg.type = GathererProgressType.MEMBERS
msg.msg_type = MSGTYPE.STARTED
msg.adid = self.ad_id
msg.domain_name = self.domain_name
await self.progress_queue.put(msg)
await asyncio.sleep(0)
acnt = self.total_members_to_poll
last_stat_cnt = 0
while acnt > 0:
try:
res = await self.agent_out_q.get()
res_type, res = res
if res_type == LDAPAgentCommand.MEMBERSHIP:
self.member_finish_ctr += 1
res.ad_id = self.ad_id
res.graph_id = self.graph_id
self.token_file.write(res.to_json().encode() + b'\r\n')
await asyncio.sleep(0)
elif res_type == LDAPAgentCommand.MEMBERSHIP_FINISHED:
if self.show_progress is True:
self.member_progress.update()
if acnt % self.progress_step_size == 0 and self.progress_queue is not None:
last_stat_cnt += self.progress_step_size
now = datetime.datetime.utcnow()
td = (now -
self.progress_last_updated).total_seconds()
self.progress_last_updated = now
msg = GathererProgress()
msg.type = GathererProgressType.MEMBERS
msg.msg_type = MSGTYPE.PROGRESS
msg.adid = self.ad_id
msg.domain_name = self.domain_name
msg.total = self.total_members_to_poll
msg.total_finished = self.total_members_to_poll - acnt
if td > 0:
msg.speed = str(self.progress_step_size // td)
msg.step_size = self.progress_step_size
await self.progress_queue.put(msg)
acnt -= 1
elif res_type == LDAPAgentCommand.EXCEPTION:
logger.warning(str(res))
except Exception as e:
logger.exception('Members enumeration error!')
raise e
if self.progress_queue is not None:
now = datetime.datetime.utcnow()
td = (now - self.progress_last_updated).total_seconds()
self.progress_last_updated = now
msg = GathererProgress()
msg.type = GathererProgressType.MEMBERS
msg.msg_type = MSGTYPE.PROGRESS
msg.adid = self.ad_id
msg.domain_name = self.domain_name
msg.total = self.total_members_to_poll
msg.total_finished = self.total_members_to_poll
if td > 0:
msg.speed = str(
(self.total_members_to_poll - last_stat_cnt) // td)
msg.step_size = (self.total_members_to_poll - last_stat_cnt)
await self.progress_queue.put(msg)
await self.stop_memberships_collection()
adinfo = self.session.query(ADInfo).get(self.ad_id)
adinfo.ldap_members_finished = True
self.session.commit()
return True, None
except Exception as e:
logger.exception('Members enumeration error main!')
await self.stop_memberships_collection()
return False, e
finally:
try:
self.session.close()
except:
pass
def run(self):
logger.info(
'[+] Starting LDAP information acqusition. This might take a while...'
)
self.setup()
logger.debug('setup finished!')
self.domaininfo_ctr += 1
job = LDAPAgentJob(LDAPAgentCommand.DOMAININFO, None)
self.agent_in_q.put(job)
while True:
res = self.agent_out_q.get()
#print(res)
res_type, res = res
if res_type == LDAPAgentCommand.DOMAININFO:
self.enum_domain(res)
elif res_type == LDAPAgentCommand.USER:
self.enum_user(res)
elif res_type == LDAPAgentCommand.MACHINE:
self.enum_machine(res)
elif res_type == LDAPAgentCommand.SPNSERVICE:
self.store_spnservice(res)
elif res_type == LDAPAgentCommand.GROUP:
self.enum_group(res)
elif res_type == LDAPAgentCommand.OU:
self.enum_ou(res)
elif res_type == LDAPAgentCommand.SD:
self.store_sd(res)
elif res_type == LDAPAgentCommand.GPO:
self.enum_gpo(res)
elif res_type == LDAPAgentCommand.MEMBERSHIP:
self.store_membership(res)
elif res_type == LDAPAgentCommand.EXCEPTION:
logger.warning(res)
elif res_type == LDAPAgentCommand.SPNSERVICES_FINISHED:
logger.debug('SPN enumeration finished!')
self.spn_finish_ctr += 1
elif res_type == LDAPAgentCommand.USERS_FINISHED:
logger.debug('Users enumeration finished!')
self.user_finish_ctr += 1
elif res_type == LDAPAgentCommand.MACHINES_FINISHED:
logger.debug('Machines enumeration finished!')
self.machine_finish_ctr += 1
elif res_type == LDAPAgentCommand.OUS_FINISHED:
logger.debug('OUs enumeration finished!')
self.ou_finish_ctr += 1
elif res_type == LDAPAgentCommand.GROUPS_FINISHED:
logger.debug('Groups enumeration finished!')
self.group_finish_ctr += 1
elif res_type == LDAPAgentCommand.MEMBERSHIPS_FINISHED:
logger.debug('Memberships enumeration finished!')
self.member_finish_ctr += 1
elif res_type == LDAPAgentCommand.SDS_FINISHED:
logger.debug('Secuirty Descriptor enumeration finished!')
self.sd_finish_ctr += 1
elif res_type == LDAPAgentCommand.DOMAININFO_FINISHED:
logger.debug('Domaininfo enumeration finished!')
self.domaininfo_finish_ctr += 1
elif res_type == LDAPAgentCommand.GPOS_FINISHED:
logger.debug('GPOs enumeration finished!')
self.gpo_finish_ctr += 1
if self.check_status() == True:
break
self.stop_agents()
logger.info('[+] LDAP information acqusition finished!')
return self.ad_id
def run(self):
logger.info('[+] Starting LDAP information acqusition. This might take a while...')
self.setup()
logger.debug('setup finished!')
self.check_jobs(None)
while True:
res = self.agent_out_q.get()
self.update_progress()
res_type, res = res
if res_type == LDAPAgentCommand.DOMAININFO:
self.domaininfo_ctr += 1
self.store_domain(res)
elif res_type == LDAPAgentCommand.USER:
self.user_ctr += 1
self.store_user(res)
elif res_type == LDAPAgentCommand.MACHINE:
self.machine_ctr += 1
self.store_machine(res)
elif res_type == LDAPAgentCommand.GROUP:
self.group_ctr += 1
self.store_group(res)
elif res_type == LDAPAgentCommand.OU:
self.ou_ctr += 1
self.store_ous(res)
elif res_type == LDAPAgentCommand.GPO:
self.gpo_ctr += 1
self.store_gpo(res)
elif res_type == LDAPAgentCommand.SPNSERVICE:
self.spn_ctr += 1
self.store_spn(res)
elif res_type == LDAPAgentCommand.SD:
self.sd_ctr += 1
self.store_sd(res)
elif res_type == LDAPAgentCommand.MEMBERSHIP:
self.member_ctr += 1
self.store_membership(res)
elif res_type == LDAPAgentCommand.TRUSTS:
self.trust_ctr += 1
self.store_trust(res)
elif res_type == LDAPAgentCommand.EXCEPTION:
logger.warning(str(res))
elif res_type.name.endswith('FINISHED'):
if self.check_jobs(res_type) is True:
break
self.stop_agents()
logger.info('[+] LDAP information acqusition finished!')
return self.ad_id
def run(self):
self.setup()
logger.info('setup finished!')
self.domaininfo_ctr += 1
job = LDAPAgentJob(LDAPAgentCommand.DOMAININFO, None)
self.agent_in_q.put(job)
while True:
res = self.agent_out_q.get()
#print(res)
res_type, res = res
if res_type == LDAPAgentCommand.DOMAININFO:
self.enum_domain(res)
elif res_type == LDAPAgentCommand.USER:
self.enum_user(res)
elif res_type == LDAPAgentCommand.MACHINE:
self.enum_machine(res)
elif res_type == LDAPAgentCommand.SPNSERVICE:
self.store_spnservice(res)
elif res_type == LDAPAgentCommand.GROUP:
self.enum_group(res)
elif res_type == LDAPAgentCommand.OU:
self.enum_ou(res)
elif res_type == LDAPAgentCommand.SD:
self.store_sd(res)
elif res_type == LDAPAgentCommand.MEMBERSHIP:
self.store_membership(res)
elif res_type == LDAPAgentCommand.EXCEPTION:
logger.warning(res)
elif res_type == LDAPAgentCommand.SPNSERVICES_FINISHED:
self.spn_finish_ctr += 1
elif res_type == LDAPAgentCommand.USERS_FINISHED:
self.user_finish_ctr += 1
elif res_type == LDAPAgentCommand.MACHINES_FINISHED:
self.machine_finish_ctr += 1
elif res_type == LDAPAgentCommand.OUS_FINISHED:
self.ou_finish_ctr += 1
elif res_type == LDAPAgentCommand.GROUPS_FINISHED:
self.group_finish_ctr += 1
elif res_type == LDAPAgentCommand.MEMBERSHIPS_FINISHED:
self.member_finish_ctr += 1
elif res_type == LDAPAgentCommand.SDS_FINISHED:
self.sd_finish_ctr += 1
elif res_type == LDAPAgentCommand.DOMAININFO_FINISHED:
self.domaininfo_finish_ctr += 1
if self.check_status() == True:
break
self.stop_agents()
return self.ad_id
async def run(self):
logger.info(
'[+] Starting LDAP information acqusition. This might take a while...'
)
await self.setup()
if self.progress_queue is not None:
msg = LDAPEnumeratorProgress()
msg.type = 'LDAP'
msg.msg_type = 'STARTED'
msg.adid = self.ad_id
msg.domain_name = self.domain_name
await self.progress_queue.put(msg)
await self.check_jobs(None)
while True:
try:
res = await self.agent_out_q.get()
await self.update_progress()
res_type, res = res
if res_type == LDAPAgentCommand.DOMAININFO:
self.domaininfo_ctr += 1
await self.store_domain(res)
elif res_type == LDAPAgentCommand.USER:
self.user_ctr += 1
await self.store_user(res)
elif res_type == LDAPAgentCommand.MACHINE:
self.machine_ctr += 1
await self.store_machine(res)
elif res_type == LDAPAgentCommand.GROUP:
self.group_ctr += 1
await self.store_group(res)
elif res_type == LDAPAgentCommand.OU:
self.ou_ctr += 1
await self.store_ous(res)
elif res_type == LDAPAgentCommand.GPO:
self.gpo_ctr += 1
await self.store_gpo(res)
elif res_type == LDAPAgentCommand.SPNSERVICE:
self.spn_ctr += 1
await self.store_spn(res)
elif res_type == LDAPAgentCommand.SD:
self.sd_ctr += 1
await self.store_sd(res)
elif res_type == LDAPAgentCommand.MEMBERSHIP:
self.member_ctr += 1
await self.store_membership(res)
elif res_type == LDAPAgentCommand.TRUSTS:
self.trust_ctr += 1
await self.store_trust(res)
elif res_type == LDAPAgentCommand.EXCEPTION:
logger.warning(str(res))
elif res_type.name.endswith('FINISHED'):
t = await self.check_jobs(res_type)
if t is True:
break
except Exception as e:
logger.exception('ldap enumerator main!')
await self.stop_agents()
return None
await self.stop_agents()
logger.info('[+] LDAP information acqusition finished!')
return self.ad_id
async def run(self):
logger.debug('Basecollector started!')
qs = self.agent_cnt
self.agent_in_q = asyncio.Queue() #AsyncProcessQueue()
self.agent_out_q = asyncio.Queue(qs) #AsyncProcessQueue(1000)
if self.show_progress is True:
self.total_progress = tqdm(desc='LDAP info entries', ascii=True)
for _ in range(self.agent_cnt):
agent = LDAPGathererAgent(self.ldap_mgr, self.agent_in_q,
self.agent_out_q)
self.agents.append(asyncio.create_task(agent.arun()))
if self.progress_queue is not None:
msg = GathererProgress()
msg.type = GathererProgressType.BASIC
msg.msg_type = MSGTYPE.STARTED
msg.adid = self.ad_id
msg.domain_name = self.domain_name
await self.progress_queue.put(msg)
await self.check_jobs(None)
logger.debug('basecollector setup complete!')
try:
while True:
res = await self.agent_out_q.get()
await self.update_progress()
res_type, res = res
if res_type == LDAPAgentCommand.DOMAININFO:
self.domaininfo_ctr += 1
await self.store_domain(res)
elif res_type == LDAPAgentCommand.USER:
self.user_ctr += 1
await self.store_user(res)
if self.user_ctr % 1000 == 0:
self.session.commit()
elif res_type == LDAPAgentCommand.MACHINE:
self.machine_ctr += 1
await self.store_machine(res)
if self.machine_ctr % 1000 == 0:
self.session.commit()
elif res_type == LDAPAgentCommand.GROUP:
self.group_ctr += 1
await self.store_group(res)
if self.group_ctr % 1000 == 0:
self.session.commit()
elif res_type == LDAPAgentCommand.OU:
self.ou_ctr += 1
await self.store_ous(res)
elif res_type == LDAPAgentCommand.GPO:
self.gpo_ctr += 1
await self.store_gpo(res)
elif res_type == LDAPAgentCommand.SPNSERVICE:
self.spn_ctr += 1
await self.store_spn(res)
elif res_type == LDAPAgentCommand.TRUSTS:
self.trust_ctr += 1
await self.store_trust(res)
elif res_type == LDAPAgentCommand.EXCEPTION:
logger.warning(str(res))
elif res_type.name.endswith('FINISHED'):
t = await self.check_jobs(res_type)
if t is True:
break
return self.ad_id, self.graph_id, None
except Exception as e:
logger.exception('ldap enumerator main!')
return None, None, e
finally:
await self.stop_agents()
async def run(self):
try:
adinfo = self.session.query(ADInfo).get(self.ad_id)
self.domain_name = str(adinfo.distinguishedName).replace(
',', '.').replace('DC=', '')
qs = self.agent_cnt
self.agent_in_q = asyncio.Queue(qs) #AsyncProcessQueue()
self.agent_out_q = asyncio.Queue(qs) #AsyncProcessQueue(1000)
self.sd_file_path = 'sd_' + datetime.datetime.utcnow().strftime(
"%Y%m%d_%H%M%S") + '.gzip'
self.sd_file = gzip.GzipFile(self.sd_file_path, 'w')
logger.debug('Polling sds')
_, res = await self.prepare_targets()
if res is not None:
raise res
for _ in range(self.agent_cnt):
agent = LDAPGathererAgent(self.ldap_mgr, self.agent_in_q,
self.agent_out_q)
self.agents.append(asyncio.create_task(agent.arun()))
asyncio.create_task(self.start_jobs())
if self.show_progress is True:
self.sds_progress = tqdm(desc='Collecting SDs',
total=self.total_targets,
position=0,
leave=True)
if self.progress_queue is not None:
msg = GathererProgress()
msg.type = GathererProgressType.SD
msg.msg_type = MSGTYPE.STARTED
msg.adid = self.ad_id
msg.domain_name = self.domain_name
await self.progress_queue.put(msg)
acnt = self.total_targets
last_stat_cnt = 0
while acnt > 0:
try:
res = await self.agent_out_q.get()
res_type, res = res
if res_type == LDAPAgentCommand.SD:
await self.store_sd(res)
if self.show_progress is True:
self.sds_progress.update()
if self.progress_queue is not None:
if acnt % self.progress_step_size == 0:
last_stat_cnt += self.progress_step_size
now = datetime.datetime.utcnow()
td = (now - self.progress_last_updated
).total_seconds()
self.progress_last_updated = now
msg = GathererProgress()
msg.type = GathererProgressType.SD
msg.msg_type = MSGTYPE.PROGRESS
msg.adid = self.ad_id
msg.domain_name = self.domain_name
msg.total = self.total_targets
msg.total_finished = self.total_targets - acnt
if td > 0:
msg.speed = str(self.progress_step_size //
td)
msg.step_size = self.progress_step_size
await self.progress_queue.put(msg)
elif res_type == LDAPAgentCommand.EXCEPTION:
logger.warning(str(res))
acnt -= 1
except Exception as e:
logger.exception('SDs enumeration error!')
raise e
if self.progress_queue is not None:
now = datetime.datetime.utcnow()
td = (now - self.progress_last_updated).total_seconds()
self.progress_last_updated = now
msg = GathererProgress()
msg.type = GathererProgressType.SD
msg.msg_type = MSGTYPE.PROGRESS
msg.adid = self.ad_id
msg.domain_name = self.domain_name
msg.total = self.total_targets
msg.total_finished = self.total_targets
if td > 0:
msg.speed = str((self.total_targets - last_stat_cnt) // td)
msg.step_size = self.total_targets - last_stat_cnt
await self.progress_queue.put(msg)
adinfo = self.session.query(ADInfo).get(self.ad_id)
adinfo.ldap_sds_finished = True
self.session.commit()
return True, None
except Exception as e:
logger.exception('SDs enumeration main error')
if self.progress_queue is not None:
msg = GathererProgress()
msg.type = GathererProgressType.SD
msg.msg_type = MSGTYPE.ERROR
msg.adid = self.ad_id
msg.domain_name = self.domain_name
msg.error = e
await self.progress_queue.put(msg)
return False, e
finally:
await self.stop_sds_collection()
try:
self.session.close()
except:
pass
async def run(self):
logger.info('[+] Starting LDAP information acqusition. This might take a while...')
await self.setup()
if self.resumption is False:
res = await self.run_init_gathering()
self.session.commit()
if res is None:
return False
try:
logger.debug('Polling sds')
total_sds_to_poll = 0
subq = self.session.query(JackDawSD.guid).filter(JackDawSD.ad_id == self.ad_id)
total_sds_to_poll += self.session.query(func.count(JackDawADUser.id)).filter_by(ad_id = self.ad_id).filter(~JackDawADUser.objectGUID.in_(subq)).scalar()
total_sds_to_poll += self.session.query(func.count(JackDawADMachine.id)).filter_by(ad_id = self.ad_id).filter(~JackDawADMachine.objectGUID.in_(subq)).scalar()
total_sds_to_poll += self.session.query(func.count(JackDawADGPO.id)).filter_by(ad_id = self.ad_id).filter(~JackDawADGPO.objectGUID.in_(subq)).scalar()
total_sds_to_poll += self.session.query(func.count(JackDawADOU.id)).filter_by(ad_id = self.ad_id).filter(~JackDawADOU.objectGUID.in_(subq)).scalar()
total_sds_to_poll += self.session.query(func.count(JackDawADGroup.id)).filter_by(ad_id = self.ad_id).filter(~JackDawADGroup.guid.in_(subq)).scalar()
asyncio.create_task(self.generate_sd_targets())
sds_p = tqdm(desc='Collecting SDs', total=total_sds_to_poll)
logger.info(total_sds_to_poll)
acnt = total_sds_to_poll
while acnt > 0:
try:
res = await self.agent_out_q.get()
res_type, res = res
if res_type == LDAPAgentCommand.SD:
sds_p.update()
await self.store_sd(res)
elif res_type == LDAPAgentCommand.EXCEPTION:
logger.warning(str(res))
acnt -= 1
except Exception as e:
logger.exception('SDs enumeration error!')
raise e
except Exception as e:
logger.exception('SDs enumeration main error')
await self.stop_sds_collection(sds_p)
return None
await self.stop_sds_collection(sds_p)
try:
logger.debug('Polling members')
total_members_to_poll = 0
subq = self.session.query(JackDawTokenGroup.guid).distinct(JackDawTokenGroup.guid).filter(JackDawTokenGroup.ad_id == self.ad_id)
total_members_to_poll += self.session.query(func.count(JackDawADUser.id)).filter_by(ad_id = self.ad_id).filter(~JackDawADUser.objectGUID.in_(subq)).scalar()
total_members_to_poll += self.session.query(func.count(JackDawADMachine.id)).filter_by(ad_id = self.ad_id).filter(~JackDawADMachine.objectGUID.in_(subq)).scalar()
total_members_to_poll += self.session.query(func.count(JackDawADGroup.id)).filter_by(ad_id = self.ad_id).filter(~JackDawADGroup.guid.in_(subq)).scalar()
asyncio.create_task(self.generate_member_targets())
member_p = tqdm(desc='Collecting members', total=total_members_to_poll)
acnt = total_members_to_poll
while acnt > 0:
try:
res = await self.agent_out_q.get()
res_type, res = res
if res_type == LDAPAgentCommand.MEMBERSHIP:
res.ad_id = self.ad_id
self.token_file.write(res.to_json().encode() + b'\r\n')
elif res_type == LDAPAgentCommand.MEMBERSHIP_FINISHED:
member_p.update()
acnt -= 1
elif res_type == LDAPAgentCommand.EXCEPTION:
logger.warning(str(res))
except Exception as e:
logger.exception('Members enumeration error!')
raise e
except Exception as e:
logger.exception('Members enumeration error main!')
await self.stop_memberships_collection(member_p)
return None
await self.stop_memberships_collection(member_p)
await self.stop_agents()
logger.info('[+] LDAP information acqusition finished!')
return self.ad_id
