def save_domain(self, data):
'''保存domain资产的相关结果
'''
domain_app = Domain()
doamin_attr_app = DomainAttr()
result = {'domain': len(data)}
for domain in data:
if 'domain' not in domain:
continue
if self.org_id:
domain['org_id'] = self.org_id
# 保存到domain
domain_id = domain_app.save_and_update(domain)
if domain_id > 0:
# 保存domain的属性
for attr_key in ('CNAME', 'A', 'title'):
if attr_key in domain:
for attr_value in domain[attr_key]:
domain_attr = {
'r_id': domain_id,
'source': self.source,
'tag': attr_key,
'content': attr_value
}
doamin_attr_app.save_and_update(domain_attr)
return result
def _get_domains(org_id, domain_address, ip_address):
'''获取域名
'''
domain_app = Domain()
domain_attr_app = DomainAttr()
org_app = Organization()
api = AssertInfoParser()
domain_list = []
domains = domain_app.gets_by_org_domain_ip(
org_id, domain_address, ip_address, page=1, rows_per_page=100000)
if domains:
for index, domain_row in enumerate(domains):
ips = domain_attr_app.gets(
query={'tag': 'A', 'r_id': domain_row['id']})
domain_info = api.get_domain_info(domain_row['id'])
domain_list.append({
'id': domain_row['id'],
"index": index+1,
"domain": domain_row['domain'],
"ip": ', '.join(set([ip_row['content'] for ip_row in ips])),
"org_name": org_app.get(int(domain_row['org_id']))['org_name'] if domain_row['org_id'] else '',
"create_time": str(domain_row['create_datetime']),
"update_time": str(domain_row['update_datetime']),
'port': ', '.join([str(x) for x in domain_info['port']]),
'title': '\n'.join(domain_info['title']),
'banner': '\n'.join(domain_info['banner'])
})
return domain_list
def save_domain(self, data):
'''保存domain资产的相关结果
'''
domain_app = Domain()
doamin_attr_app = DomainAttr()
result = {'domain': len(data)}
for domain in data:
if 'domain' not in domain:
continue
if self.org_id:
domain['org_id'] = self.org_id
# 保存到domain
domain_id = domain_app.save_and_update(domain)
if domain_id > 0:
# 保存domain的属性
for attr_key in ('CNAME', 'A', 'title', 'whatweb', 'server', 'httpx'):
if attr_key in domain:
for attr_value in domain[attr_key]:
domain_attr = {'r_id': domain_id, 'source': self.source,
'tag': attr_key, 'content': attr_value[0:800]}
try:
doamin_attr_app.save_and_update(domain_attr)
except Exception as e:
logger.error(traceback.format_exc())
logger.error('save domain attr:{}-{}-{}'.format(domain['domain'], domain_attr['tag'],
domain_attr['content']))
return result
def export_domain_memo(self,
org_id=None,
domain_address=None,
ip_address=None,
color_tag=None,
memo_content=None,
date_delta=None):
'''导出Domain相关的备忘录信息
'''
domain_table = Domain()
memo_table = DomainMemo()
memo_list = []
domains = domain_table.gets_by_search(org_id, domain_address,
ip_address, color_tag,
memo_content, date_delta)
if domains:
for domain_row in domains:
memo_obj = memo_table.get(domain_row['id'])
if memo_obj:
memo_list.append('[+]{}'.format(domain_row['domain']))
memo_list.append(memo_obj['content'])
memo_list.append("")
return memo_list
def get_domain_info(self, Id):
'''聚合一个DOMAIN的详情
'''
domain_info = {}
# 获取DOMAIN
domain_obj = Domain().get(Id)
if not domain_obj:
return None
domain_info.update(
domain=domain_obj['domain'],
create_datetime=domain_obj['create_datetime'].strftime(
'%Y-%m-%d %H:%M'),
update_datetime=domain_obj['update_datetime'].strftime(
'%Y-%m-%d %H:%M'))
# 获取组织名称
if domain_obj['org_id']:
organziation__obj = Organization().get(domain_obj['org_id'])
if organziation__obj:
domain_info.update(organization=organziation__obj['org_name'])
else:
domain_info.update(organization='')
domain_attrs_obj = DomainAttr().gets(query={'r_id': domain_obj['id']})
# 获取域名的属性信息：title和ip,whatweb
title_set = set()
banner_set = set()
ip_set = set()
whatweb_set = set()
for domain_attr_obj in domain_attrs_obj:
if domain_attr_obj['tag'] == 'title':
title_set.add(domain_attr_obj['content'])
elif domain_attr_obj['tag'] == 'A':
ip_set.add(domain_attr_obj['content'])
elif domain_attr_obj['tag'] == 'whatweb':
whatweb_set.add(domain_attr_obj['content'])
elif domain_attr_obj['tag'] == 'server':
banner_set.add(domain_attr_obj['content'])
# 获取域名关联的IP端口详情：
port_set = set()
ip_port_list = []
for domain_ip in ip_set:
ip_obj = Ip().gets(query={'ip': domain_ip})
if ip_obj and len(ip_obj) > 0:
#port_list, title_set, banner_set, ports_attr_info
p, t, b, pai = self.get_ip_port_info(ip_obj[0]['ip'],
ip_obj[0]['id'])
port_set.update(p)
title_set.update(t)
banner_set.update(b)
ip_port_list.extend(pai)
domain_info.update(ip=list(ip_set))
domain_info.update(port=list(port_set))
domain_info.update(title=list(title_set))
domain_info.update(whatweb=list(whatweb_set))
domain_info.update(banner=list(banner_set))
domain_info.update(port_attr=ip_port_list)
return domain_info
def domain_asset_info_view():
'''显示一个DOMAIN的详细信息
'''
domain = request.args.get('domain')
domains = Domain().gets(query={'domain': domain})
if domains and len(domains) > 0:
domain_info = AssertInfoParser().get_domain_info(domains[0]['id'])
else:
domain_info = None
return render_template('domain-info.html', domain_info=domain_info)
def __get_ip_domain(self, ip):
'''查询IP关联的域名
'''
domain_set = set()
domain_attrs_obj = DomainAttr().gets(query={'tag': 'A', 'content': ip})
for domain_attr_obj in domain_attrs_obj:
domain_obj = Domain().get(domain_attr_obj['r_id'])
if domain_attr_obj:
domain_set.add(domain_obj['domain'])
return domain_set
def view_dashboard():
'''dashbord页面显示
'''
if request.method == 'GET':
return render_template('dashboard.html')
# 统计信息
dashboard_data = {
'ip_count': Ip().count(),
'domain_count': Domain().count(),
'vulnerability_count': Vulnerability().count(),
'task_active': Task().count({'state': 'STARTED'})
}
return jsonify(dashboard_data)
def domain_asset_info_view():
'''显示一个DOMAIN的详细信息
'''
domain = request.args.get('domain')
domains = Domain().gets(query={'domain': domain})
if domains and len(domains) > 0:
domain_info = AssertInfoParser().get_domain_info(domains[0]['id'])
# 表格背景设置：
table_backgroud_set = False
if 'port_attr' in domain_info and domain_info['port_attr']:
for p in domain_info['port_attr']:
if p['ip'] and p['port']:
table_backgroud_set = not table_backgroud_set
p['table_backgroud_set'] = table_backgroud_set
else:
domain_info = None
return render_template('domain-info.html', domain_info=domain_info)
def view_dashboard():
'''dashbord页面显示
'''
if request.method == 'GET':
return render_template('dashboard.html')
# 统计信息
r = TaskAPI().get_celery_workers()
total = 0
active = 0
if r['status'] == 'success':
for k, v in r['result'].items():
for tk, tv in v['stats']['total'].items():
total += tv
active += len(v['active'])
dashboard_data = {
'ip_count': Ip().count(),
'domain_count': Domain().count(),
'task_total': total,
'task_active': active
}
return jsonify(dashboard_data)
def delete_domain_view(domain_id):
'''删除一个DOMAIN
'''
rows = Domain().delete(domain_id)
return jsonify({'status': 'success', 'msg': rows})
def domain_asset_view():
'''页面上显示域名资产，datatable前端ajax请求进行分页
'''
if request.method == 'GET':
org_table = Organization()
org_list = org_table.gets()
return render_template('domain-list.html', org_list=org_list)
domain_list = []
ip_table = Ip()
org_table = Organization()
domain_table = Domain()
domain_attr_table = DomainAttr()
index = 1
try:
draw = int(request.form.get('draw'))
start = int(request.form.get('start'))
length = int(request.form.get('length'))
org_name = request.form.get('org_name')
ip_address = request.form.get('ip_address')
domain_address = request.form.get('domain_address')
domains = domain_table.gets_by_org_domain_ip(org_name,
domain_address,
ip_address,
page=start // length + 1,
rows_per_page=length)
for domain_row in domains:
ips = domain_attr_table.gets(query={
'tag': 'A',
'r_id': domain_row['id']
})
domain_list.append({
'id':
domain_row['id'],
"index":
index + start,
"domain":
domain_row['domain'],
"ip":
', '.join(
set([
'<a href="/ip-info?ip={0}">{0}</a>'.format(
ip_row['content']) for ip_row in ips
])),
"org_name":
org_table.get(int(domain_row['org_id']))['org_name']
if domain_row['org_id'] else '',
"create_time":
str(domain_row['create_datetime']),
"update_time":
str(domain_row['update_datetime'])
})
index += 1
count = domain_table.count_by_org_domain_ip(org_name, domain_address,
ip_address)
json_data = {
'draw': draw,
'recordsTotal': count,
'recordsFiltered': count,
'data': domain_list
}
except Exception as e:
print(e)
pass
return jsonify(json_data)
def domain_asset_view():
'''页面上显示域名资产，datatable前端ajax请求进行分页
'''
if request.method == 'GET':
org_table = Organization()
org_list = org_table.gets()
if not org_list:
org_list = []
org_list.insert(0, {'id': '', 'org_name': '--全部--'})
data = {
'org_list': org_list,
'domain_address': session.get('domain_address', default=''),
'ip_address_domain': session.get('ip_address_domain', default=''),
'session_org_id': session.get('session_org_id', default=''),
'pocsuite3_poc_files': Pocsuite3().load_poc_files(),
'xray_poc_files': XRay().load_poc_files()
}
return render_template('domain-list.html', data=data)
domain_list = []
org_table = Organization()
domain_table = Domain()
domain_attr_table = DomainAttr()
api = AssertInfoParser()
index = 1
json_data = {}
try:
draw = int(request.form.get('draw'))
start = int(request.form.get('start'))
length = int(request.form.get('length'))
org_id = request.form.get('org_id')
ip_address = request.form.get('ip_address')
domain_address = request.form.get('domain_address')
color_tag = request.form.get('color_tag')
memo_content = request.form.get('memo_content')
date_delta = request.form.get('date_delta')
session['ip_address_domain'] = ip_address
session['domain_address'] = domain_address
session['session_org_id'] = org_id
count = 0
domains = domain_table.gets_by_search(org_id,
domain_address,
ip_address,
color_tag,
memo_content,
date_delta,
page=start // length + 1,
rows_per_page=length)
if domains:
for domain_row in domains:
ips = domain_attr_table.gets(query={
'tag': 'A',
'r_id': domain_row['id']
})
domain_info = api.get_domain_info(domain_row['id'])
# 获取关联的漏洞信息：
vul_info = []
vul_results = Vulnerability().gets(
{'target': domain_row['domain']})
if vul_results and len(vul_results) > 0:
for v in vul_results:
vul_info.append('{}/{}'.format(v['poc_file'],
v['source']))
domain_list.append({
"id":
domain_row['id'],
"index":
index + start,
"color_tag":
domain_info['color_tag'],
"memo_content":
domain_info['memo'],
"domain":
domain_row['domain'],
"ip":
', '.join(
set([
'<a href="/ip-info?ip={0}" target="_blank">{0}</a>'
.format(ip_row['content']) for ip_row in ips
])),
"org_name":
org_table.get(int(domain_row['org_id']))['org_name']
if domain_row['org_id'] else '',
"create_time":
str(domain_row['create_datetime']),
"update_time":
str(domain_row['update_datetime']),
'port':
domain_info['port'],
'title':
', '.join(domain_info['title']),
'banner':
', '.join(domain_info['banner']),
'vulnerability':
'\r\n'.join(vul_info)
})
index += 1
count = domain_table.count_by_search(org_id, domain_address,
ip_address, color_tag,
memo_content, date_delta)
json_data = {
'draw': draw,
'recordsTotal': count,
'recordsFiltered': count,
'data': domain_list
}
except Exception as e:
logger.error(traceback.format_exc())
print(e)
return jsonify(json_data)
def domain_asset_view():
'''页面上显示域名资产，datatable前端ajax请求进行分页
'''
if request.method == 'GET':
org_table = Organization()
org_list = org_table.gets()
if not org_list:
org_list = []
org_list.insert(0, {'id': '', 'org_name': '--组织机构--'})
data = {
'org_list': org_list,
'domain_address': session.get('domain_address', default=''),
'ip_address_domain': session.get('ip_address_domain', default=''),
'session_org_id': session.get('session_org_id', default='')
}
return render_template('domain-list.html', data=data)
domain_list = []
org_table = Organization()
domain_table = Domain()
domain_attr_table = DomainAttr()
api = AssertInfoParser()
index = 1
json_data = {}
try:
draw = int(request.form.get('draw'))
start = int(request.form.get('start'))
length = int(request.form.get('length'))
org_id = request.form.get('org_id')
ip_address = request.form.get('ip_address')
domain_address = request.form.get('domain_address')
session['ip_address_domain'] = ip_address
session['domain_address'] = domain_address
session['session_org_id'] = org_id
count = 0
domains = domain_table.gets_by_org_domain_ip(org_id,
domain_address,
ip_address,
page=start // length + 1,
rows_per_page=length)
if domains:
for domain_row in domains:
ips = domain_attr_table.gets(query={
'tag': 'A',
'r_id': domain_row['id']
})
domain_info = api.get_domain_info(domain_row['id'])
domain_list.append({
'id':
domain_row['id'],
"index":
index + start,
"domain":
domain_row['domain'],
"ip":
', '.join(
set([
'<a href="/ip-info?ip={0}" target="_blank">{0}</a>'
.format(ip_row['content']) for ip_row in ips
])),
"org_name":
org_table.get(int(domain_row['org_id']))['org_name']
if domain_row['org_id'] else '',
"create_time":
str(domain_row['create_datetime']),
"update_time":
str(domain_row['update_datetime']),
'port':
domain_info['port'],
'title':
', '.join(domain_info['title']),
'banner':
', '.join(domain_info['banner'])
})
index += 1
count = domain_table.count_by_org_domain_ip(
org_id, domain_address, ip_address)
json_data = {
'draw': draw,
'recordsTotal': count,
'recordsFiltered': count,
'data': domain_list
}
except Exception as e:
logger.error(traceback.format_exc())
print(e)
return jsonify(json_data)
def get_domain_info(self, Id):
'''聚合一个DOMAIN的详情
'''
domain_info = {}
# 获取DOMAIN
domain_obj = Domain().get(Id)
if not domain_obj:
return None
domain_info.update(
id=domain_obj['id'],
domain=domain_obj['domain'],
create_datetime=domain_obj['create_datetime'].strftime(
'%Y-%m-%d %H:%M'),
update_datetime=domain_obj['update_datetime'].strftime(
'%Y-%m-%d %H:%M'))
# 获取组织名称
if domain_obj['org_id']:
organziation__obj = Organization().get(domain_obj['org_id'])
if organziation__obj:
domain_info.update(organization=organziation__obj['org_name'])
else:
domain_info.update(organization='')
domain_attrs_obj = DomainAttr().gets(query={'r_id': domain_obj['id']})
# 获取域名的属性信息：title和ip,whatweb
title_set = set()
banner_set = set()
ip_set = set()
whatweb_set = set()
httpx_set = set()
for domain_attr_obj in domain_attrs_obj:
if domain_attr_obj['tag'] == 'title':
title_set.add(domain_attr_obj['content'])
elif domain_attr_obj['tag'] == 'A':
ip_set.add(domain_attr_obj['content'])
elif domain_attr_obj['tag'] == 'whatweb':
whatweb_set.add(domain_attr_obj['content'])
elif domain_attr_obj['tag'] == 'server':
banner_set.add(domain_attr_obj['content'])
elif domain_attr_obj['tag'] == 'httpx':
httpx_set.add(domain_attr_obj['content'])
# 获取域名关联的IP端口详情：
port_set = set()
ip_port_list = []
for domain_ip in ip_set:
ip_obj = Ip().gets(query={'ip': domain_ip})
if ip_obj and len(ip_obj) > 0:
# port_list, title_set, banner_set, ports_attr_info
p, t, b, pai, ps = self.get_ip_port_info(
ip_obj[0]['ip'], ip_obj[0]['id'])
port_set.update(p)
title_set.update(t)
banner_set.update(b)
ip_port_list.extend(pai)
domain_info.update(ip=list(ip_set))
domain_info.update(port=list(port_set))
domain_info.update(title=list(title_set))
domain_info.update(whatweb=list(whatweb_set))
domain_info.update(httpx=list(httpx_set))
domain_info.update(banner=list(banner_set))
domain_info.update(port_attr=ip_port_list)
# 获取标记颜色：
color_tag_obj = DomainColorTag().get(domain_obj['id'])
domain_info.update(
color_tag=color_tag_obj['color'] if color_tag_obj else '')
# 获取备忘录信息：
memo_obj = DomainMemo().get(domain_obj['id'])
domain_info.update(memo=memo_obj['content'] if memo_obj else '')
# 获取关联的漏洞信息：
vul_results = Vulnerability().gets({'target': domain_obj['domain']})
if vul_results and len(vul_results) > 0:
vul_info = []
for v in vul_results:
vul_info.append({
'id':
v['id'],
'target':
v['target'],
'url':
v['url'],
'poc_file':
v['poc_file'],
'source':
v['source'],
'update_datetime':
v['update_datetime'].strftime('%Y-%m-%d %H:%M')
})
domain_info.update(vulnerability=vul_info)
else:
domain_info.update(vulnerability=None)
return domain_info