async def start_calc(self):
#call this only after setting the current ADID!!!!
try:
adinfo = self.session.query(ADInfo).get(self.ad_id)
self.domain_name = str(adinfo.distinguishedName).replace(
',', '.').replace('DC=', '')
await self.log_msg('Adding gplink edges')
self.gplink_edges()
#await self.log_msg()
#self.groupmembership_edges()
await self.log_msg('Adding trusts edges')
self.trust_edges()
await self.log_msg('Adding sqladmin edges')
self.sqladmin_edges()
await self.log_msg('Adding hassession edges')
self.hasession_edges()
await self.log_msg('Adding localgroup edges')
self.localgroup_edges()
await self.log_msg('Adding password sharing edges')
self.passwordsharing_edges()
await self.log_msg('Adding allowedtoact sharing edges')
self.allowedtoact_edges()
self.session.commit()
_, err = await self.calc_sds_mp()
if err is not None:
raise err
adinfo = self.session.query(ADInfo).get(self.ad_id)
adinfo.edges_finished = True
self.session.commit()
return True, None
except Exception as e:
logger.exception('edge calculation error!')
return False, e
def edge_calc_writer(outqueue, file_path, ad_id, append_to_file):
"""
Separate process to write all edges in a file as CSV
"""
try:
buffer = ''
if append_to_file is True:
mode = 'a+'
else:
mode = 'w'
with open(file_path, 'w', newline='') as f:
while True:
data = outqueue.get()
if data is None:
return
src, dst, label = data
buffer += '%s,%s,%s,%s\r\n'
if len(buffer) > 1000000:
print('writing!')
f.write(buffer)
buffer = ''
except Exception as e:
logger.exception('edge_calc_writer')
async def run(self):
try:
self.session = get_session(self.db_conn)
if self.ad_id is None and self.graph_id is not None:
#recalc!
self.session.query(Edge).filter_by(graph_id = self.graph_id).filter(Edge.label != 'member').delete()
self.session.commit()
res = self.session.query(GraphInfo).get(self.graph_id)
for giad in self.session.query(GraphInfoAD).filter_by(graph_id = self.graph_id).all():
self.ad_id = giad.ad_id
_, err = await self.start_calc()
if err is not None:
raise err
else:
_, err = await self.start_calc()
if err is not None:
raise err
return True, None
except Exception as e:
logger.exception('edge calculation error!')
return False, e
finally:
try:
self.session.close()
except:
pass
def add_credentials_impacket(self, impacket_file):
self.get_dbsession()
ctr = 0
ctr_fail = 0
try:
for cred in Credential.from_impacket_file(impacket_file,
self.domain_id):
try:
self.dbsession.add(cred)
if ctr % 10000 == 0:
logger.info(ctr)
self.dbsession.commit()
except exc.IntegrityError as e:
ctr_fail += 1
self.dbsession.rollback()
continue
else:
ctr += 1
self.dbsession.commit()
logger.info('Added %d users. Failed inserts: %d' % (ctr, ctr_fail))
except Exception as e:
logger.exception()
finally:
self.dbsession.close()
def lookup_oid(self, oint, ad_id, token):
try:
if oint not in self.edgeinfo_cache:
edgeinfo = self.db_session.query(EdgeLookup).get(oint)
self.edgeinfo_cache[oint] = edgeinfo.oid
return self.edgeinfo_cache[oint]
except Exception as e:
logger.exception('lookup_oid')
async def generate_member_targets(self):
try:
subq = self.session.query(JackDawTokenGroup.guid).distinct(JackDawTokenGroup.guid).filter(JackDawTokenGroup.ad_id == self.ad_id)
q = self.session.query(JackDawADUser.dn, JackDawADUser.objectSid, JackDawADUser.objectGUID).filter_by(ad_id = self.ad_id).filter(~JackDawADUser.objectGUID.in_(subq))
await self.resumption_target_gen_member(q, JackDawADUser.id, 'user', LDAPAgentCommand.MEMBERSHIPS)
q = self.session.query(JackDawADMachine.dn, JackDawADMachine.objectSid, JackDawADMachine.objectGUID).filter_by(ad_id = self.ad_id).filter(~JackDawADMachine.objectGUID.in_(subq))
await self.resumption_target_gen_member(q, JackDawADMachine.id, 'machine', LDAPAgentCommand.MEMBERSHIPS)
q = self.session.query(JackDawADGroup.dn, JackDawADGroup.sid, JackDawADGroup.guid).filter_by(ad_id = self.ad_id).filter(~JackDawADGroup.guid.in_(subq))
await self.resumption_target_gen_member(q, JackDawADGroup.id, 'group', LDAPAgentCommand.MEMBERSHIPS)
except Exception as e:
logger.exception('generate_sd_targets')
def run(self):
try:
if self.append_to_file is True:
mode = 'ab+'
else:
mode = 'wb'
output_file_path = self.output_file_dir.joinpath('temp_edges.gz')
self.out_file = GzipFile(output_file_path, mode)
self.gplink_edges()
self.groupmembership_edges()
self.trust_edges()
self.sqladmin_edges()
self.hasession_edges()
self.localgroup_edges()
self.passwordsharing_edges()
self.out_file.close()
sdcalc = SDEgdeCalc(self.session,
self.ad_id,
output_file_path,
worker_count=self.worker_count,
buffer_size=self.buffer_size,
append_to_file=True)
sdcalc.run()
#output_file_path_final = self.output_file_dir.joinpath('edges.gz')
#with GzipFile(output_file_path_final,'wb') as o:
# with GzipFile(output_file_path,'rb') as f:
# for line in f:
# line = line.strip()
# line = line.decode()
# src, dst, *rest = line.split(',')
# t = ','.join(rest)
# src = self.lookup.insert(src)
# dst = self.lookup.insert(dst)
# o.write( ('%s,%s,%s\r\n' % (src,dst,t)).encode())
#
#output_file_path_maps = self.output_file_dir.joinpath('maps.gz')
#with GzipFile(output_file_path_maps,'wb') as o:
# for k in self.lookup.lookup:
# o.write(('%s,%s\r\n' % (k, self.lookup.lookup[k])).encode())
output_file_path.unlink()
except Exception as e:
logger.exception('edge calculation error!')
finally:
try:
if self.out_file is not None:
self.out_file.close()
except:
pass
print('Done!')
def run(self):
try:
logger.debug('[ACL] Starting sd edge calc')
self.inqueue = mp.Queue(self.buffer_size)
self.outqueue = mp.Queue(self.buffer_size)
logger.debug('[ACL] Starting processes')
self.writer = mp.Process(target=edge_calc_writer,
args=(self.outqueue,
self.output_file_path, self.ad_id,
self.append_to_file))
self.writer.daemon = True
self.writer.start()
self.workers = [
mp.Process(target=edge_calc_worker,
args=(self.inqueue, self.outqueue))
for i in range(self.worker_count)
]
for proc in self.workers:
proc.daemon = True
proc.start()
print(1)
logger.debug('[ACL] data generation')
total = self.session.query(func.count(
JackDawSD.id)).filter_by(ad_id=self.ad_id).scalar()
q = self.session.query(JackDawSD).filter_by(ad_id=self.ad_id)
for adsd in tqdm(windowed_query(q, JackDawSD.id, 10), total=total):
self.inqueue.put(adsd)
for _ in range(procno):
self.inqueue.put(None)
logger.debug('Gen done!')
logger.debug('[ACL] Added %s edges' % (p_cnt))
logger.debug('[ACL] joining workers')
for proc in self.workers:
proc.join()
logger.debug('[ACL] workers finished, waiting for writer')
self.outqueue.put(None)
self.writer.join()
logger.debug('[ACL] All Finished!')
except:
logger.exception('[ACL]')
async def worker(self):
while True:
try:
target = await self.worker_q.get()
if target is None:
return
try:
await self.scan_host(target)
except:
#exception should be handled in scan_host
continue
except Exception as e:
logger.exception('WORKER ERROR')
raise
async def run(self):
try:
self.session = get_session(self.db_conn)
if self.ad_id is None and self.graph_id is not None:
#recalc!
self.session.query(Edge).filter_by(
graph_id=self.graph_id).filter(
Edge.label != 'member').delete()
self.session.commit()
res = self.session.query(GraphInfo).get(self.graph_id)
self.ad_id = res.ad_id
adinfo = self.session.query(ADInfo).get(self.ad_id)
self.domain_name = str(adinfo.distinguishedName).replace(
',', '.').replace('DC=', '')
await self.log_msg('Adding gplink edges')
self.gplink_edges()
#await self.log_msg()
#self.groupmembership_edges()
await self.log_msg('Adding trusts edges')
self.trust_edges()
await self.log_msg('Adding sqladmin edges')
self.sqladmin_edges()
await self.log_msg('Adding hassession edges')
self.hasession_edges()
await self.log_msg('Adding localgroup edges')
self.localgroup_edges()
await self.log_msg('Adding password sharing edges')
self.passwordsharing_edges()
self.session.commit()
_, err = await self.calc_sds_mp()
if err is not None:
raise err
adinfo = self.session.query(ADInfo).get(self.ad_id)
adinfo.edges_finished = True
self.session.commit()
return True, None
except Exception as e:
logger.exception('edge calculation error!')
return False, e
finally:
try:
self.session.close()
except:
pass
async def do_listads(self, cmd):
"""
Lists all available adid in the DB
Sends back a NestOpListADRes reply or ERR in case of failure
"""
try:
reply = NestOpListADRes()
for i in self.db_session.query(ADInfo.id).all():
reply.adids.append(i[0])
await self.send_reply(cmd, reply)
await self.send_ok(cmd)
except Exception as e:
logger.exception('do_listads')
await self.send_error(cmd, e)
async def do_changead(self, cmd):
"""
Changes the current AD to another, specified by ad_id in the command.
Doesnt have a dedicated reply, OK means change is succsess, ERR means its not
"""
try:
res = self.db_session.query(ADInfo).get(cmd.adid)
print(res)
if res is None:
await self.send_error(cmd, 'No such AD in database')
return
self.ad_id = res.id
await self.send_ok(cmd)
except Exception as e:
logger.exception('do_listads')
await self.send_error(cmd, e)
async def stop_sds_collection(self, sds_p):
sds_p.disable = True
try:
self.sd_file.close()
cnt = 0
with gzip.GzipFile(self.sd_file_path, 'r') as f:
for line in tqdm(f, desc='Uploading security descriptors to DB', total=self.spn_finish_ctr):
sd = JackDawSD.from_json(line.strip())
self.session.add(sd)
cnt += 1
if cnt % 10000 == 0:
self.session.commit()
self.session.commit()
os.remove(self.sd_file_path)
except Exception as e:
logger.exception('Error while uploading sds from file to DB')
async def generate_targets(self):
try:
q = self.session.query(Machine).filter_by(ad_id=self.ad_id)
for machine in windowed_query(q, Machine.id, 100):
try:
dns_name = machine.dNSHostName
if dns_name is None or dns_name == '':
dns_name = '%s.%s' % (str(machine.sAMAccountName[:-1]),
str(self.domain_name))
await self.in_q.put((machine.objectSid, dns_name))
except:
continue
#signaling the ed of target generation
await self.in_q.put(None)
except Exception as e:
logger.exception('smb generate_targets')
async def stop_memberships_collection(self, member_p):
member_p.disable = True
try:
self.token_file.close()
cnt = 0
with gzip.GzipFile(self.token_file_path, 'r') as f:
for line in tqdm(f, desc='Uploading memberships to DB', total=self.member_finish_ctr):
sd = JackDawTokenGroup.from_json(line.strip())
self.session.add(sd)
cnt += 1
if cnt % 10000 == 0:
self.session.commit()
self.session.commit()
os.remove(self.token_file_path)
except Exception as e:
logger.exception('Error while uploading memberships from file to DB')
async def generate_member_targets(self):
try:
subq = self.session.query(EdgeLookup.oid).filter_by(ad_id = self.ad_id).filter(EdgeLookup.id == Edge.src).filter(Edge.label == 'member').filter(Edge.ad_id == self.ad_id)
q = self.session.query(ADUser.dn, ADUser.objectSid, ADUser.objectGUID)\
.filter_by(ad_id = self.ad_id)\
.filter(~ADUser.objectSid.in_(subq))
await self.resumption_target_gen_member(q, ADUser.id, 'user', LDAPAgentCommand.MEMBERSHIPS)
q = self.session.query(Machine.dn, Machine.objectSid, Machine.objectGUID)\
.filter_by(ad_id = self.ad_id)\
.filter(~Machine.objectSid.in_(subq))
await self.resumption_target_gen_member(q, Machine.id, 'machine', LDAPAgentCommand.MEMBERSHIPS)
q = self.session.query(Group.dn, Group.objectSid, Group.objectGUID)\
.filter_by(ad_id = self.ad_id)\
.filter(~Group.objectSid.in_(subq))
await self.resumption_target_gen_member(q, Group.id, 'group', LDAPAgentCommand.MEMBERSHIPS)
except Exception as e:
logger.exception('generate_member_targets')
def run(self):
try:
logger.debug('[ACL] Starting sd edge calc')
logger.debug('[ACL] data generation')
total = self.session.query(func.count(
JackDawSD.id)).filter_by(ad_id=self.ad_id).scalar()
q = self.session.query(JackDawSD).filter_by(ad_id=self.ad_id)
for adsd in tqdm(windowed_query(q, JackDawSD.id,
self.worker_count),
total=total):
self.inqueue.put(adsd)
logger.debug('[ACL] All Finished!')
except:
logger.exception('[ACL]')
def calc_acl_edges_mp(self, session, adid, construct):
try:
#ACE edges calc with multiprocessing
inqueue = mp.Queue()
outqueue = mp.Queue()
procno = mp.cpu_count()
logger.debug('[ACL] Starting processes')
procs = [
mp.Process(target=acl_calc_mp,
args=(inqueue, outqueue, construct))
for i in range(procno)
]
for proc in procs:
proc.daemon = True
proc.start()
logger.debug('[ACL] Starting generator thread')
acl_gen_th = threading.Thread(target=acl_calc_gen,
args=(session, adid, inqueue,
procno))
acl_gen_th.daemon = True
acl_gen_th.start()
p_cnt = 0
while True:
res = outqueue.get()
if res is None:
procno -= 1
logger.debug('[ACL] Proc X - Finished!')
if procno == 0:
break
continue
ace_sid, sid, label = res
self.add_edge(ace_sid, sid, construct, label=label)
p_cnt += 1
logger.debug('[ACL] Added %s edges' % (p_cnt))
logger.debug('[ACL] joining processes')
for proc in procs:
proc.join()
logger.debug('[ACL] Finished!')
except Exception as e:
logger.exception('[ACL]')
async def prepare_targets(self):
try:
if self.resumption is True:
self.total_targets = 1
if self.members_target_file_handle is not None:
raise Exception('Resumption doesnt use the target file handle!')
self.members_target_file_handle = gzip.GzipFile('member_targets.gz','wb')
await self.generate_member_targets()
else:
self.members_target_file_handle.seek(0,0)
for line in self.members_target_file_handle:
self.total_members_to_poll += 1
return True, None
except Exception as err:
logger.exception('prep targets')
return False, err
def edge_calc_writer(outqueue, file_path, ad_id, append_to_file):
"""
Separate process to write all edges in a file as CSV
"""
try:
buffer = b''
if append_to_file is True:
mode = 'ab+'
else:
mode = 'wb'
with GzipFile(file_path, mode) as f:
while True:
data = outqueue.get()
if data is None:
return
f.write(data)
except Exception as e:
logger.exception('edge_calc_writer')
async def arun(self):
try:
res = await self.setup()
if res is False:
return
while True:
res = await self.agent_in_q.get()
if res is None:
return
if res.command == LDAPAgentCommand.DOMAININFO:
await self.get_domain_info()
elif res.command == LDAPAgentCommand.USERS:
await self.get_all_users()
elif res.command == LDAPAgentCommand.MACHINES:
await self.get_all_machines()
elif res.command == LDAPAgentCommand.GROUPS:
await self.get_all_groups()
elif res.command == LDAPAgentCommand.OUS:
await self.get_all_ous()
elif res.command == LDAPAgentCommand.GPOS:
await self.get_all_gpos()
elif res.command == LDAPAgentCommand.SPNSERVICES:
await self.get_all_spnservices()
elif res.command == LDAPAgentCommand.SCHEMA:
await self.get_all_schemaentries()
#elif res.command == LDAPAgentCommand.MEMBERSHIPS:
# await self.get_all_effective_memberships()
elif res.command == LDAPAgentCommand.MEMBERSHIPS:
await self.get_effective_memberships(res.data)
elif res.command == LDAPAgentCommand.SDS:
await self.get_sds(res.data)
elif res.command == LDAPAgentCommand.TRUSTS:
await self.get_all_trusts()
except Exception as e:
logger.exception('Agent main!')
finally:
if self.ldap is not None:
await self.ldap._con.disconnect()
async def generate_sd_targets(self):
try:
subq = self.session.query(JackDawSD.guid).filter(JackDawSD.ad_id == self.ad_id)
#total_sds_to_poll += self.session.query(func.count(JackDawADMachine.id)).filter_by(ad_id = self.ad_id).filter(~JackDawADMachine.objectGUID.in_(subq)).scalar()
#total_sds_to_poll += self.session.query(func.count(JackDawADGPO.id)).filter_by(ad_id = self.ad_id).filter(~JackDawADGPO.objectGUID.in_(subq)).scalar()
#total_sds_to_poll += self.session.query(func.count(JackDawADOU.id)).filter_by(ad_id = self.ad_id).filter(~JackDawADOU.objectGUID.in_(subq)).scalar()
#total_sds_to_poll += self.session.query(func.count(JackDawADGroup.id)).filter_by(ad_id = self.ad_id).filter(~JackDawADGroup.guid.in_(subq)).scalar()
q = self.session.query(JackDawADUser.dn, JackDawADUser.objectSid, JackDawADUser.objectGUID).filter_by(ad_id = self.ad_id).filter(~JackDawADUser.objectGUID.in_(subq))
await self.resumption_target_gen(q, JackDawADUser.id, 'user', LDAPAgentCommand.SDS)
q = self.session.query(JackDawADMachine.dn, JackDawADMachine.objectSid, JackDawADMachine.objectGUID).filter_by(ad_id = self.ad_id).filter(~JackDawADMachine.objectGUID.in_(subq))
await self.resumption_target_gen(q, JackDawADMachine.id, 'machine', LDAPAgentCommand.SDS)
q = self.session.query(JackDawADGroup.dn, JackDawADGroup.sid, JackDawADGroup.guid).filter_by(ad_id = self.ad_id).filter(~JackDawADGroup.guid.in_(subq))
await self.resumption_target_gen(q, JackDawADGroup.id, 'group', LDAPAgentCommand.SDS)
q = self.session.query(JackDawADOU.dn, JackDawADOU.objectGUID).filter_by(ad_id = self.ad_id).filter(~JackDawADOU.objectGUID.in_(subq))
await self.resumption_target_gen_2(q, JackDawADOU.id, 'ou', LDAPAgentCommand.SDS)
q = self.session.query(JackDawADGPO.dn, JackDawADGPO.objectGUID).filter_by(ad_id = self.ad_id).filter(~JackDawADGPO.objectGUID.in_(subq))
await self.resumption_target_gen_2(q, JackDawADGPO.id, 'gpo', LDAPAgentCommand.SDS)
logger.debug('generate_sd_targets finished!')
except Exception as e:
logger.exception('generate_sd_targets')
async def generate_targets(self):
try:
q = self.session.query(Machine).filter_by(ad_id=self.ad_id)
for filter in self.target_filters:
if filter == 'live':
filter_after = datetime.datetime.today(
) - datetime.timedelta(days=90)
q = q.filter(Machine.pwdLastSet >= filter_after)
for machine in windowed_query(q, Machine.id, 100):
try:
dns_name = machine.dNSHostName
if dns_name is None or dns_name == '':
dns_name = '%s.%s' % (str(machine.sAMAccountName[:-1]),
str(self.domain_name))
await self.in_q.put((machine.objectSid, dns_name))
except:
continue
#signaling the ed of target generation
await self.in_q.put(None)
except Exception as e:
logger.exception('smb generate_targets')
async def generate_sd_targets(self):
try:
subq = self.session.query(
JackDawSD.guid).filter(JackDawSD.ad_id == self.ad_id)
q = self.session.query(
ADInfo.distinguishedName, ADInfo.objectSid,
ADInfo.objectGUID).filter_by(
id=self.ad_id).filter(~ADInfo.objectGUID.in_(subq))
await self.resumption_target_gen(q, ADInfo.id, 'domain',
LDAPAgentCommand.SDS)
q = self.session.query(
ADUser.dn, ADUser.objectSid, ADUser.objectGUID).filter_by(
ad_id=self.ad_id).filter(~ADUser.objectGUID.in_(subq))
await self.resumption_target_gen(q, ADUser.id, 'user',
LDAPAgentCommand.SDS)
q = self.session.query(
Machine.dn, Machine.objectSid, Machine.objectGUID).filter_by(
ad_id=self.ad_id).filter(~Machine.objectGUID.in_(subq))
await self.resumption_target_gen(q, Machine.id, 'machine',
LDAPAgentCommand.SDS)
q = self.session.query(
Group.dn, Group.objectSid, Group.objectGUID).filter_by(
ad_id=self.ad_id).filter(~Group.objectGUID.in_(subq))
await self.resumption_target_gen(q, Group.id, 'group',
LDAPAgentCommand.SDS)
q = self.session.query(ADOU.dn, ADOU.objectGUID).filter_by(
ad_id=self.ad_id).filter(~ADOU.objectGUID.in_(subq))
await self.resumption_target_gen_2(q, ADOU.id, 'ou',
LDAPAgentCommand.SDS)
q = self.session.query(GPO.dn, GPO.objectGUID).filter_by(
ad_id=self.ad_id).filter(~GPO.objectGUID.in_(subq))
await self.resumption_target_gen_2(q, GPO.id, 'gpo',
LDAPAgentCommand.SDS)
logger.debug('generate_sd_targets finished!')
except Exception as e:
logger.exception('generate_sd_targets')
async def do_load_ad(self, cmd):
try:
# loads an AD scan and sends all results to the client
# sanity check if the AD exists
res = self.db_session.query(ADInfo).get(cmd.adid)
if res is None:
await self.send_error(cmd, 'No AD ID exists with that ID')
return
#sending machines
for computer in self.db_session.query(Machine).filter_by(ad_id = cmd.adid).all():
await asyncio.sleep(0)
reply = NestOpComputerRes()
reply.token = cmd.token
reply.name = computer.sAMAccountName
reply.adid = computer.ad_id
reply.sid = computer.objectSid
reply.domainname = computer.dNSHostName
reply.osver = computer.operatingSystem
reply.ostype = computer.operatingSystemVersion
reply.description = computer.description
if computer.UAC_SERVER_TRUST_ACCOUNT is True:
reply.computertype = 'DOMAIN_CONTROLLER'
elif computer.operatingSystem is not None:
if computer.operatingSystem.lower().find('windows') != -1:
if computer.operatingSystem.lower().find('server') != -1:
reply.computertype = 'SERVER'
else:
reply.computertype = 'WORKSTATION'
else:
reply.computertype = 'NIX'
else:
reply.computertype = 'DUNNO'
await self.websocket.send(reply.to_json())
#sending users
for user in self.db_session.query(ADUser).filter_by(ad_id = cmd.adid).all():
await asyncio.sleep(0)
reply = NestOpUserRes()
reply.token = cmd.token
reply.name = user.sAMAccountName
reply.adid = user.ad_id
reply.sid = user.objectSid
reply.kerberoast = 1 if user.servicePrincipalName is not None else 0
reply.asreproast = int(user.UAC_DONT_REQUIRE_PREAUTH)
reply.nopassw = int(user.UAC_PASSWD_NOTREQD)
reply.cleartext = int(user.UAC_ENCRYPTED_TEXT_PASSWORD_ALLOWED)
reply.smartcard = int(user.UAC_SMARTCARD_REQUIRED)
reply.active = int(user.canLogon)
reply.description = user.description
if user.adminCount is not None:
reply.is_admin = int(user.adminCount)
else:
reply.is_admin = 0
await self.websocket.send(reply.to_json())
#sending localgroups
for lgroup in self.db_session.query(LocalGroup).filter_by(ad_id = cmd.adid).all():
await asyncio.sleep(0)
reply = NestOpSMBLocalGroupRes()
reply.token = cmd.token
reply.adid = lgroup.ad_id
reply.machinesid = lgroup.machine_sid
reply.usersid = lgroup.sid
reply.groupname = lgroup.groupname
await self.websocket.send(reply.to_json())
#sending smb shares
for share in self.db_session.query(NetShare).filter_by(ad_id = cmd.adid).all():
await asyncio.sleep(0)
reply = NestOpSMBShareRes()
reply.token = cmd.token
reply.adid = share.ad_id
reply.machinesid = share.machine_sid
reply.netname = share.netname
await self.websocket.send(reply.to_json())
#sending smb sessions
for session in self.db_session.query(NetSession).filter_by(ad_id = cmd.adid).all():
await asyncio.sleep(0)
reply = NestOpSMBSessionRes()
reply.token = cmd.token
reply.adid = session.ad_id
reply.machinesid = session.machine_sid
reply.username = session.username
await self.websocket.send(reply.to_json())
#sending groups
for group in self.db_session.query(Group).filter_by(ad_id = cmd.adid).all():
await asyncio.sleep(0)
reply = NestOpGroupRes()
reply.token = cmd.token
reply.adid = group.ad_id
reply.name = group.sAMAccountName
reply.dn = group.dn
reply.guid = group.objectGUID
reply.sid = group.objectSid
reply.description = group.description
await self.websocket.send(reply.to_json())
await self.send_ok(cmd)
except Exception as e:
await self.send_error(cmd, "Error! Reason: %s" % e)
logger.exception('do_load_ad')
async def do_gather(self, cmd):
try:
progress_queue = asyncio.Queue()
gatheringmonitor_task = asyncio.create_task(self.__gathermonitor(cmd, progress_queue))
ldap_url = cmd.ldap_url
if ldap_url == 'auto':
if platform.system().lower() == 'windows':
from winacl.functions.highlevel import get_logon_info
logon = get_logon_info()
ldap_url = 'ldap+sspi-ntlm://%s\\%s:jackdaw@%s' % (logon['domain'], logon['username'], logon['logonserver'])
else:
raise Exception('ldap auto mode selected, but it is not supported on this platform')
smb_url = cmd.smb_url
if smb_url == 'auto':
if platform.system().lower() == 'windows':
from winacl.functions.highlevel import get_logon_info
logon = get_logon_info()
smb_url = 'smb2+sspi-ntlm://%s\\%s:jackdaw@%s' % (logon['domain'], logon['username'], logon['logonserver'])
else:
raise Exception('smb auto mode selected, but it is not supported on this platform')
kerberos_url = cmd.kerberos_url
dns = cmd.dns
if dns == 'auto':
if platform.system().lower() == 'windows':
from jackdaw.gatherer.rdns.dnstest import get_correct_dns_win
srv_domain = '%s.%s' % (logon['logonserver'], logon['dnsdomainname'])
dns = await get_correct_dns_win(srv_domain)
if dns is None:
dns = None #failed to get dns
else:
dns = str(dns)
else:
raise Exception('dns auto mode selected, but it is not supported on this platform')
print(ldap_url)
print(smb_url)
print(dns)
with multiprocessing.Pool() as mp_pool:
gatherer = Gatherer(
self.db_url,
self.work_dir,
ldap_url,
smb_url,
kerb_url=kerberos_url,
ldap_worker_cnt=int(cmd.ldap_workers),
smb_worker_cnt=int(cmd.smb_worker_cnt),
mp_pool=mp_pool,
smb_gather_types=['all'],
progress_queue=progress_queue,
show_progress=self.show_progress,
calc_edges=True,
ad_id=None,
dns=dns,
stream_data=cmd.stream_data
)
res, err = await gatherer.run()
if err is not None:
print('gatherer returned error')
await self.send_error(cmd, str(err))
return
#####testing
await asyncio.sleep(20)
#######
await self.send_ok(cmd)
except Exception as e:
logger.exception('do_gather')
await self.send_error(cmd, str(e))
finally:
if gatheringmonitor_task is not None:
gatheringmonitor_task.cancel()
progress_queue = None
async def __gathermonitor(self, cmd, results_queue):
try:
usernames_testing = []
machine_sids_testing = []
temp_tok_testing = None
temp_adid_testing = None
temp_started = False
while True:
try:
msg = await results_queue.get()
if msg is None:
break
#print(msg)
if msg.type in STANDARD_PROGRESS_MSG_TYPES:
##### TESTING
temp_tok_testing = cmd.token
temp_adid_testing = msg.adid
######
reply = NestOpGatherStatus()
reply.token = cmd.token
reply.current_progress_type = msg.type.value
reply.msg_type = msg.msg_type.value
reply.adid = msg.adid
reply.domain_name = msg.domain_name
reply.total = msg.total
reply.step_size = msg.step_size
reply.basic_running = []
if msg.running is not None:
reply.basic_running = [x for x in msg.running]
reply.basic_finished = msg.finished
reply.smb_errors = msg.errors
reply.smb_sessions = msg.sessions
reply.smb_shares = msg.shares
reply.smb_groups = msg.groups
await self.websocket.send(reply.to_json())
####TESTINGTESTING!!!!
if msg.type.value != 'LDAP_BASIC':
if temp_started is False:
asyncio.create_task(self.spam_sessions(temp_tok_testing, temp_adid_testing, machine_sids_testing, usernames_testing))
temp_started = True
elif msg.type == GathererProgressType.USER:
usernames_testing.append(msg.data.sAMAccountName)
reply = NestOpUserRes()
reply.token = cmd.token
reply.name = msg.data.sAMAccountName
reply.adid = msg.data.ad_id
reply.sid = msg.data.objectSid
reply.kerberoast = 1 if msg.data.servicePrincipalName is not None else 0
reply.asreproast = int(msg.data.UAC_DONT_REQUIRE_PREAUTH)
reply.nopassw = int(msg.data.UAC_PASSWD_NOTREQD)
reply.cleartext = int(msg.data.UAC_ENCRYPTED_TEXT_PASSWORD_ALLOWED)
reply.smartcard = int(msg.data.UAC_SMARTCARD_REQUIRED)
reply.active = int(msg.data.canLogon)
reply.description = msg.data.description
reply.is_admin = int(msg.data.adminCout)
await self.websocket.send(reply.to_json())
elif msg.type == GathererProgressType.MACHINE:
machine_sids_testing.append(msg.data.objectSid)
reply = NestOpComputerRes()
reply.token = cmd.token
reply.name = msg.data.sAMAccountName
reply.adid = msg.data.ad_id
reply.sid = msg.data.objectSid
reply.domainname = msg.data.dNSHostName
reply.osver = msg.data.operatingSystem
reply.ostype = msg.data.operatingSystemVersion
reply.description = msg.data.description
if msg.data.UAC_SERVER_TRUST_ACCOUNT is True:
reply.computertype = 'DOMAIN_CONTROLLER'
elif msg.data.operatingSystem is not None:
if msg.data.operatingSystem.lower().find('windows') != -1:
if msg.data.operatingSystem.lower().find('server') != -1:
reply.computertype = 'SERVER'
else:
reply.computertype = 'WORKSTATION'
else:
reply.computertype = 'NIX'
else:
reply.computertype = 'DUNNO'
await self.websocket.send(reply.to_json())
elif msg.type == GathererProgressType.SMBLOCALGROUP:
reply = NestOpSMBLocalGroupRes()
reply.token = cmd.token
reply.adid = msg.data.ad_id
reply.machinesid = msg.data.machine_sid
reply.usersid = msg.data.sid
reply.groupname = msg.data.groupname
await self.websocket.send(reply.to_json())
elif msg.type == GathererProgressType.SMBSHARE:
reply = NestOpSMBShareRes()
reply.token = cmd.token
reply.adid = msg.data.ad_id
reply.machinesid = msg.data.machine_sid
reply.netname = msg.data.netname
await self.websocket.send(reply.to_json())
elif msg.type == GathererProgressType.SMBSESSION:
reply = NestOpSMBSessionRes()
reply.token = cmd.token
reply.adid = msg.data.ad_id
reply.machinesid = msg.data.machine_sid
reply.username = msg.data.username
await self.websocket.send(reply.to_json())
elif msg.type == GathererProgressType.GROUP:
reply = NestOpGroupRes()
reply.token = cmd.token
reply.adid = msg.data.ad_id
reply.name = msg.data.sAMAccountName
reply.dn = msg.data.dn
reply.guid = msg.data.objectGUID
reply.sid = msg.data.objectSid
reply.description = msg.data.description
await self.websocket.send(reply.to_json())
except asyncio.CancelledError:
return
except Exception as e:
logger.exception('resmon processing error!')
#await self.send_error(cmd, str(e))
except asyncio.CancelledError:
return
except Exception as e:
print('resmon died! %s' % e)
await self.send_error(cmd, str(e))
def construct(self, construct):
"""
Fills the network graph from database to memory
"""
#self.ad_id = ad_id
session = self.get_session()
adinfo = session.query(JackDawADInfo).get(construct.ad_id)
self.domain_sid = str(adinfo.objectSid)
#self.calc_acl_edges(session, construct)
#adding group nodes
logger.debug('Adding group nodes')
cnt = 0
for group in adinfo.groups:
self.add_sid_to_node(group.sid,
'group',
construct,
name=group.name)
cnt += 1
logger.debug('Added %s group nodes' % cnt)
logger.debug('Adding user nodes')
cnt = 0
for user in adinfo.users:
self.add_sid_to_node(user.objectSid,
'user',
construct,
name=user.sAMAccountName)
cnt += 1
logger.debug('Added %s user nodes' % cnt)
logger.debug('Adding machine nodes')
cnt = 0
for user in adinfo.computers:
self.add_sid_to_node(user.objectSid,
'machine',
construct,
name=user.sAMAccountName)
cnt += 1
logger.debug('Added %s machine nodes' % cnt)
logger.debug('Adding hassession edges')
cnt = 0
for res in session.query(
JackDawADUser.objectSid, JackDawADMachine.objectSid
).filter(NetSession.username == JackDawADUser.sAMAccountName).filter(
NetSession.source == JackDawADMachine.sAMAccountName).distinct(
NetSession.username):
self.add_edge(res[0], res[1], construct, label='hasSession')
self.add_edge(res[1], res[0], construct, label='hasSession')
cnt += 2
logger.debug('Added %s hassession edges' % cnt)
logger.debug('Adding localgroup edges')
cnt = 0
for res in session.query(
JackDawADUser.objectSid, JackDawADMachine.objectSid,
LocalGroup.groupname).filter(
JackDawADMachine.id == LocalGroup.machine_id
).filter(JackDawADMachine.ad_id == construct.ad_id).filter(
JackDawADUser.ad_id == construct.ad_id).filter(
JackDawADUser.objectSid == LocalGroup.sid).all():
label = None
if res[2] == 'Remote Desktop Users':
label = 'canRDP'
weight = 1
elif res[2] == 'Distributed COM Users':
label = 'executeDCOM'
weight = 1
elif res[2] == 'Administrators':
label = 'adminTo'
weight = 1
self.add_edge(res[0],
res[1],
construct,
label=label,
weight=weight)
cnt += 1
logger.debug('Added %s localgroup edges' % cnt)
# TODO: implement this!
#if self.show_constrained_delegations == True:
# pass
# TODO: implement this!
#if self.show_unconstrained_delegations == True:
# pass
# TODO: implement this!
#for relation in construct.custom_relations:
# relation.calc()
# self.add_edge(res.sid, res.target_sid)
#print('adding membership edges')
#adding membership edges
logger.debug('Adding membership edges')
cnt = 0
q = session.query(JackDawTokenGroup).filter_by(ad_id=construct.ad_id)
for tokengroup in windowed_query(q, JackDawTokenGroup.id, 10000):
#for tokengroup in adinfo.group_lookups:
self.add_sid_to_node(tokengroup.sid, 'unknown', construct)
self.add_sid_to_node(tokengroup.member_sid, 'unknown', construct)
if tokengroup.is_user == True:
try:
self.add_edge(tokengroup.sid,
tokengroup.member_sid,
construct,
label='member')
cnt += 1
except AssertionError as e:
logger.exception()
elif tokengroup.is_machine == True:
try:
self.add_edge(tokengroup.sid,
tokengroup.member_sid,
construct,
label='member')
cnt += 1
except AssertionError as e:
logger.exception()
elif tokengroup.is_group == True:
try:
self.add_edge(tokengroup.sid,
tokengroup.member_sid,
construct,
label='member')
cnt += 1
except AssertionError as e:
logger.exception()
logger.debug('Added %s membership edges' % cnt)
#adding ACL edges
#self.calc_acl_edges(session, construct)
#self.calc_acl_edges(adinfo, construct)
self.calc_acl_edges_mp(session, construct.ad_id, construct)
logger.info('Adding password sharing edges')
cnt = 0
def get_sid_by_nthash(ad_id, nt_hash):
return session.query(
JackDawADUser.objectSid, Credential.username).filter_by(
ad_id=ad_id).filter(Credential.username ==
JackDawADUser.sAMAccountName).filter(
Credential.nt_hash == nt_hash)
dup_nthashes_qry = session.query(
Credential.nt_hash).filter(Credential.history_no == 0).filter(
Credential.ad_id == construct.ad_id).filter(
Credential.username != 'NA').filter(
Credential.domain != '<LOCAL>').group_by(
Credential.nt_hash).having(
func.count(Credential.nt_hash) > 1)
for res in dup_nthashes_qry.all():
sidd = {}
for sid, _ in get_sid_by_nthash(construct.ad_id, res[0]).all():
sidd[sid] = 1
for sid1 in sidd:
for sid2 in sidd:
if sid1 == sid2:
continue
self.add_edge(sid1, sid2, construct, label='pwsharing')
cnt += 1
logger.info('Added %s password sharing edges' % cnt)
def all_shortest_paths(self, src_sid=None, dst_sid=None):
nv = GraphData()
if not src_sid and not dst_sid:
raise Exception('Either source or destination MUST be specified')
elif not src_sid and dst_sid:
try:
#for each node we calculate the shortest path to the destination node, silently skip the ones who do not have path to dst
inqueue = mp.Queue()
outqueue = mp.Queue()
procno = mp.cpu_count()
logger.debug('[DST_CALC] Starting processes')
procs = [
mp.Process(target=short_worker,
args=(inqueue, outqueue, self.graph, dst_sid))
for i in range(procno)
]
for proc in procs:
proc.daemon = True
proc.start()
logger.debug('[DST_CALC] Starting generator thread')
node_gen_th = threading.Thread(target=short_node_gen,
args=(self.graph, inqueue,
dst_sid, procno))
node_gen_th.daemon = True
node_gen_th.start()
p_cnt = 0
while True:
path = outqueue.get()
if path is None:
procno -= 1
logger.debug('[DST_CALC] Proc X - Finished!')
if procno == 0:
break
continue
self.__add_path(nv, path)
p_cnt += 1
logger.debug('[DST_CALC] Found %s paths to dst node %s' %
(p_cnt, dst_sid))
logger.debug('[DST_CALC] joining processes')
for proc in procs:
proc.join()
logger.debug('[DST_CALC] Finished!')
except Exception as e:
logger.exception('[DST_CALC]')
elif src_sid and not dst_sid:
#for each node we calculate the shortest path to the destination node, silently skip the ones who do not have path to dst
for node in self.graph.nodes:
if node == src_sid:
continue
try:
for path in nx.all_shortest_paths(self.graph,
source=src_sid,
target=node):
self.__add_path(nv, path)
except nx.exception.NetworkXNoPath:
continue
else:
#for each node we calculate the shortest path to the destination node, silently skip the ones who do not have path to dst
for path in nx.all_shortest_paths(self.graph,
source=src_sid,
target=dst_sid):
self.__add_path(nv, path)
return nv
async def print_progress(self):
if self.show_progress is False:
try:
while True:
msg = await self.progress_queue.get()
if msg is None:
return
continue
except Exception as e:
logger.exception('Progress bar crashed')
logger.debug('Setting up progress bars')
pos = 0
ldap_info_pbar = tqdm(desc='MSG: ', ascii=True, position=pos)
self.progress_bars.append(ldap_info_pbar)
pos += 1
if self.ldap_url is not None:
ldap_basic_pbar = tqdm(desc='LDAP basic enum ',
ascii=True,
position=pos)
self.progress_bars.append(ldap_basic_pbar)
pos += 1
ldap_sd_pbar = tqdm(desc='LDAP SD enum ',
ascii=True,
position=pos)
self.progress_bars.append(ldap_sd_pbar)
pos += 1
if self.store_to_db is True:
ldap_sdupload_pbar = tqdm(desc='LDAP SD upload ',
ascii=True,
position=pos)
self.progress_bars.append(ldap_sdupload_pbar)
pos += 1
ldap_member_pbar = tqdm(desc='LDAP membership enum ',
ascii=True,
position=pos)
self.progress_bars.append(ldap_member_pbar)
pos += 1
if self.store_to_db is True:
ldap_memberupload_pbar = tqdm(desc='LDAP membership upload',
ascii=True,
position=pos)
self.progress_bars.append(ldap_memberupload_pbar)
pos += 1
if self.kerb_url is not None:
kerb_pbar = tqdm(desc='KERBEROAST ',
ascii=True,
position=pos)
self.progress_bars.append(kerb_pbar)
pos += 1
if self.rdns_resolver is not None:
dns_pbar = tqdm(desc='DNS enum ',
ascii=True,
position=pos)
self.progress_bars.append(dns_pbar)
pos += 1
if self.smb_url is not None:
smb_pbar = tqdm(desc='SMB enum ',
ascii=True,
position=pos)
self.progress_bars.append(smb_pbar)
pos += 1
if self.calculate_edges is True:
sdcalc_pbar = tqdm(desc='SD edges calc ',
ascii=True,
position=pos)
self.progress_bars.append(sdcalc_pbar)
pos += 1
sdcalcupload_pbar = tqdm(desc='SD edges upload ',
ascii=True,
position=pos)
self.progress_bars.append(sdcalcupload_pbar)
pos += 1
self.progress_refresh_task = asyncio.create_task(
self.progress_refresh())
logger.debug('waiting for progress messages')
while True:
try:
msg = await self.progress_queue.get()
if msg is None:
for pbar in self.progress_bars:
pbar.refresh()
return
if msg.type == GathererProgressType.BASIC:
if msg.msg_type == MSGTYPE.PROGRESS:
if ldap_basic_pbar.total is None:
ldap_basic_pbar.total = msg.total
ldap_basic_pbar.update(msg.step_size)
if msg.msg_type == MSGTYPE.FINISHED:
ldap_basic_pbar.refresh()
elif msg.type == GathererProgressType.SD:
if msg.msg_type == MSGTYPE.PROGRESS:
if ldap_sd_pbar.total is None:
ldap_sd_pbar.total = msg.total
ldap_sd_pbar.update(msg.step_size)
if msg.msg_type == MSGTYPE.FINISHED:
ldap_sd_pbar.refresh()
elif msg.type == GathererProgressType.SDUPLOAD:
if msg.msg_type == MSGTYPE.PROGRESS:
if ldap_sdupload_pbar.total is None:
ldap_sdupload_pbar.total = msg.total
ldap_sdupload_pbar.update(msg.step_size)
if msg.msg_type == MSGTYPE.FINISHED:
ldap_sdupload_pbar.refresh()
elif msg.type == GathererProgressType.MEMBERS:
if msg.msg_type == MSGTYPE.PROGRESS:
if ldap_member_pbar.total is None:
ldap_member_pbar.total = msg.total
ldap_member_pbar.update(msg.step_size)
if msg.msg_type == MSGTYPE.FINISHED:
ldap_member_pbar.refresh()
elif msg.type == GathererProgressType.MEMBERSUPLOAD:
if msg.msg_type == MSGTYPE.PROGRESS:
if ldap_memberupload_pbar.total is None:
ldap_memberupload_pbar.total = msg.total
ldap_memberupload_pbar.update(msg.step_size)
if msg.msg_type == MSGTYPE.FINISHED:
ldap_memberupload_pbar.refresh()
elif msg.type == GathererProgressType.KERBEROAST:
if msg.msg_type == MSGTYPE.PROGRESS:
if kerb_pbar.total is None:
kerb_pbar.total = msg.total
kerb_pbar.update(msg.step_size)
if msg.msg_type == MSGTYPE.FINISHED:
kerb_pbar.refresh()
elif msg.type == GathererProgressType.DNS:
if msg.msg_type == MSGTYPE.PROGRESS:
if dns_pbar.total is None:
dns_pbar.total = msg.total
dns_pbar.update(msg.step_size)
if msg.msg_type == MSGTYPE.FINISHED:
dns_pbar.refresh()
elif msg.type == GathererProgressType.SMB:
if msg.msg_type == MSGTYPE.PROGRESS:
if smb_pbar.total is None:
smb_pbar.total = msg.total
smb_pbar.update(msg.step_size)
if msg.msg_type == MSGTYPE.FINISHED:
smb_pbar.refresh()
elif msg.type == GathererProgressType.SDCALC:
if msg.msg_type == MSGTYPE.PROGRESS:
if sdcalc_pbar.total is None:
sdcalc_pbar.total = msg.total
sdcalc_pbar.update(msg.step_size)
if msg.msg_type == MSGTYPE.FINISHED:
sdcalc_pbar.refresh()
elif msg.type == GathererProgressType.SDCALCUPLOAD:
if msg.msg_type == MSGTYPE.PROGRESS:
if sdcalcupload_pbar.total is None:
sdcalcupload_pbar.total = msg.total
sdcalcupload_pbar.update(msg.step_size)
if msg.msg_type == MSGTYPE.FINISHED:
sdcalcupload_pbar.refresh()
elif msg.type == GathererProgressType.INFO:
ldap_info_pbar.display('MSG: %s' % str(msg.text))
elif msg.type == GathererProgressType.REFRESH:
for pbar in self.progress_bars:
pbar.refresh()
except asyncio.CancelledError:
return
except Exception as e:
logger.exception('Progress bar crashed')
return
