def _get_ips(org_id, domain_address, ip_address, port, content, iplocation, port_status, color_tag, memo_content): '''获取IP ''' ip_table = Ip() aip = AssertInfoParser() ip_list = [] ips = ip_table.gets_by_search(org_id=org_id, domain=domain_address, ip=ip_address, port=port, content=content, iplocation=iplocation, port_status=port_status, color_tag=color_tag, memo_content=memo_content, page=1, rows_per_page=100000) if ips: for i, ip_row in enumerate(ips): ip_info = aip.get_ip_info(ip_row['id']) ip_info.update(index=i + 1) ip_list.append(ip_info) return ip_list
def save_ip(self, data): '''保存ip资产相关的结果 ''' ip_app = Ip() port_app = Port() port_attr_app = PortAttr() result = {'ip': len(data), 'port': 0} for ip in data: # 保存IP if 'ip' not in ip: continue if self.org_id: ip['org_id'] = self.org_id ip_id = ip_app.save_and_update(ip) if ip_id > 0: result['port'] += len(ip['port']) # 保存每个端口数据 if 'port' not in ip: continue for port in ip['port']: port['ip_id'] = ip_id port_id = port_app.save_and_update(port) if port_id > 0: # 保存端口的属性 for attr_key in self.result_attr_keys: if attr_key in port and port[attr_key]: data_port_attr = { 'r_id': port_id, 'source': self.source, 'tag': attr_key, 'content': port[attr_key]} port_attr_app.save_and_update(data_port_attr) return result
def save(self, data): '''保存IP归属地结果到数据库 只更新ip表的location字段 ''' ip_app = Ip() count = 0 for ip in data: if 'location' in ip and ip['location']: if self.org_id: ip['org_id'] = self.org_id count += 1 if ip_app.save_and_update(ip) > 0 else 0 return count
def statistics_ip(self, org_id=None, domain_address=None, ip_address=None, port=None, content=None, iplocation=None, port_status=None, color_tag=None, memo_content=None): '''根据查询条件,统计IP、IP的C段地址和相关的所有端口 ''' ip_table = Ip() port_table = Port() ip_list = [] ip_port_list = [] ip_c_set = set() port_set = set() # 统计每个端口出现的次数 port_count_dict = defaultdict(lambda: 0) ips = ip_table.gets_by_search(org_id=org_id, domain=domain_address, ip=ip_address, port=port, content=content, iplocation=iplocation, port_status=port_status, color_tag=color_tag, memo_content=memo_content, page=1, rows_per_page=100000) if ips: for ip_row in ips: # ip ip_list.append(ip_row['ip']) # C段 ip_c = ip_row['ip'].split('.')[0:3] ip_c.append('0/24') ip_c_set.add('.'.join(ip_c)) # port ports_obj = port_table.gets(query={'ip_id': ip_row['id']}) for port_obj in ports_obj: port_set.add(port_obj['port']) port_count_dict[str(port_obj['port'])] += 1 ip_port_list.append('{}:{}'.format(ip_row['ip'], port_obj['port'])) return ip_list, ip_c_set, port_set, port_count_dict, ip_port_list
def get_ip_info(self, Id): '''聚合一个IP的详情 ''' ip_info = {} # 获取IP ip_obj = Ip().get(Id) if not ip_obj: return None ip_info.update(ip=ip_obj['ip'], location=ip_obj['location'], status=ip_obj['status'], create_datetime=ip_obj['create_datetime'].strftime( '%Y-%m-%d %H:%M'), update_datetime=ip_obj['update_datetime'].strftime( '%Y-%m-%d %H:%M')) # 获取组织名称 if ip_obj['org_id']: organziation__obj = Organization().get(ip_obj['org_id']) if organziation__obj: ip_info.update(organization=organziation__obj['org_name']) else: ip_info.update(Organization='') # 端口、标题、banner、端口详情 port_list, title_set, banner_set, ports_attr_info = self.get_ip_port_info( ip_obj['ip'], ip_obj['id']) ip_info.update(port_attr=ports_attr_info) ip_info.update(title=list(title_set)) ip_info.update(banner=list(banner_set)) ip_info.update(port=port_list) # IP关联的域名 domain_set = self.__get_ip_domain(ip_obj['ip']) ip_info.update(domain=list(domain_set)) return ip_info
def test_ip(): from nemo.core.database.ip import Ip from nemo.core.database.port import Port ip = Ip() port = Port() # row_id = ip.add(data={'ip':'192.168.3.1','org_id':None,'location':'上海','status':'enable'}) # print(row_id) # obj = ip.get(row_id) # print(obj) # data={'ip':'192.168.3.10','org_id':3,'location':'上海2','status':'enable'} # print(ip.update(1,data)) # print(ip.get(1)) #print(org.delete(6)) #print(ip.gets(query={'ip':'192.168.3.10'},page=1,rows_per_page=2)) #print(ip.count()) print(ip.gets_by_range(ip_start='192.168.1.1', ip_end='192.168.3.20'))
def get_domain_info(self, Id): '''聚合一个DOMAIN的详情 ''' domain_info = {} # 获取DOMAIN domain_obj = Domain().get(Id) if not domain_obj: return None domain_info.update( domain=domain_obj['domain'], create_datetime=domain_obj['create_datetime'].strftime( '%Y-%m-%d %H:%M'), update_datetime=domain_obj['update_datetime'].strftime( '%Y-%m-%d %H:%M')) # 获取组织名称 if domain_obj['org_id']: organziation__obj = Organization().get(domain_obj['org_id']) if organziation__obj: domain_info.update(organization=organziation__obj['org_name']) else: domain_info.update(organization='') domain_attrs_obj = DomainAttr().gets(query={'r_id': domain_obj['id']}) # 获取域名的属性信息:title和ip,whatweb title_set = set() banner_set = set() ip_set = set() whatweb_set = set() for domain_attr_obj in domain_attrs_obj: if domain_attr_obj['tag'] == 'title': title_set.add(domain_attr_obj['content']) elif domain_attr_obj['tag'] == 'A': ip_set.add(domain_attr_obj['content']) elif domain_attr_obj['tag'] == 'whatweb': whatweb_set.add(domain_attr_obj['content']) elif domain_attr_obj['tag'] == 'server': banner_set.add(domain_attr_obj['content']) # 获取域名关联的IP端口详情: port_set = set() ip_port_list = [] for domain_ip in ip_set: ip_obj = Ip().gets(query={'ip': domain_ip}) if ip_obj and len(ip_obj) > 0: #port_list, title_set, banner_set, ports_attr_info p, t, b, pai = self.get_ip_port_info(ip_obj[0]['ip'], ip_obj[0]['id']) port_set.update(p) title_set.update(t) banner_set.update(b) ip_port_list.extend(pai) domain_info.update(ip=list(ip_set)) domain_info.update(port=list(port_set)) domain_info.update(title=list(title_set)) domain_info.update(whatweb=list(whatweb_set)) domain_info.update(banner=list(banner_set)) domain_info.update(port_attr=ip_port_list) return domain_info
def save_ip(self, data): '''保存ip资产相关的结果 ''' ip_app = Ip() port_app = Port() port_attr_app = PortAttr() result = {'ip': len(data), 'port': 0} for ip in data: # 保存IP if 'ip' not in ip: continue if self.org_id: ip['org_id'] = self.org_id ip_id = ip_app.save_and_update(ip) if ip_id > 0: # 保存每个端口数据 if 'port' not in ip: continue result['port'] += len(ip['port']) for port in ip['port']: port['ip_id'] = ip_id try: port_id = port_app.save_and_update(port) if port_id > 0: # 保存端口的属性 for attr_key in self.result_attr_keys: if attr_key in port and port[attr_key]: data_port_attr = { 'r_id': port_id, 'source': self.source, 'tag': attr_key, 'content': port[attr_key][:800]} try: port_attr_app.save_and_update(data_port_attr) except Exception as e: logger.error(traceback.format_exc()) logger.error('save port port attr:{}-{}-{}-{}'.format(ip['port'], port['port'], data_port_attr['tag'], data_port_attr['content'])) except Exception as ex: logger.error(traceback.format_exc()) logger.error('save ip port:{}-{}'.format(ip['port'], port['port'])) return result
def ip_asset_info_view(): '''显示一个IP地址的详细信息 ''' ip = request.args.get('ip') ips = Ip().gets(query={'ip': ip}) if ips and len(ips) > 0: ip_info = AssertInfoParser().get_ip_info(ips[0]['id']) else: ip_info = None return render_template('ip-info.html', ip_info=ip_info)
def view_dashboard(): '''dashbord页面显示 ''' if request.method == 'GET': return render_template('dashboard.html') # 统计信息 dashboard_data = { 'ip_count': Ip().count(), 'domain_count': Domain().count(), 'vulnerability_count': Vulnerability().count(), 'task_active': Task().count({'state': 'STARTED'}) } return jsonify(dashboard_data)
def export_ip_memo(self, org_id=None, domain_address=None, ip_address=None, port=None, content=None, iplocation=None, port_status=None, color_tag=None, memo_content=None, date_delta=None): '''导出ip相关的备忘录信息 ''' ip_table = Ip() memo_table = IpMemo() memo_list = [] ips = ip_table.gets_by_search(org_id=org_id, domain=domain_address, ip=ip_address, port=port, content=content, iplocation=iplocation, port_status=port_status, color_tag=color_tag, memo_content=memo_content, date_delta=date_delta, page=1, rows_per_page=100000) if ips: for ip_row in ips: memo_obj = memo_table.get(ip_row['id']) if memo_obj: memo_list.append('[+]{}'.format(ip_row['ip'])) memo_list.append(memo_obj['content']) memo_list.append("") return memo_list
def ip_asset_info_view(): '''显示一个IP地址的详细信息 ''' ip = request.args.get('ip') ips = Ip().gets(query={'ip': ip}) if ips and len(ips) > 0: ip_info = AssertInfoParser().get_ip_info(ips[0]['id']) if 'port_attr' in ip_info and ip_info['port_attr']: # 表格背景设置: table_backgroud_set = False for p in ip_info['port_attr']: if p['ip'] and p['port']: table_backgroud_set = not table_backgroud_set p['table_backgroud_set'] = table_backgroud_set else: ip_info = None return render_template('ip-info.html', ip_info=ip_info)
def view_dashboard(): '''dashbord页面显示 ''' if request.method == 'GET': return render_template('dashboard.html') # 统计信息 r = TaskAPI().get_celery_workers() total = 0 active = 0 if r['status'] == 'success': for k, v in r['result'].items(): for tk, tv in v['stats']['total'].items(): total += tv active += len(v['active']) dashboard_data = { 'ip_count': Ip().count(), 'domain_count': Domain().count(), 'task_total': total, 'task_active': active } return jsonify(dashboard_data)
def ip_asset_view(): '''IP资产列表展示 ''' if request.method == 'GET': org_table = Organization() org_list = org_table.gets() if not org_list: org_list = [] org_list.insert(0, {'id': '', 'org_name': '--全部--'}) data = { 'org_list': org_list, 'ip_address_ip': session.get('ip_address_ip', default=''), 'domain_address': session.get('domain_address', default=''), 'port': session.get('port', default=''), 'session_org_id': session.get('session_org_id', default=''), 'pocsuite3_poc_files': Pocsuite3().load_poc_files(), 'xray_poc_files': XRay().load_poc_files() } return render_template('ip-list.html', data=data) ip_table = Ip() org_table = Organization() ip_color_tag_table = IpColorTag() ip_memo_table = IpMemo() aip = AssertInfoParser() ip_list = [] json_data = {} index = 1 try: draw = int(request.form.get('draw')) start = int(request.form.get('start')) length = int(request.form.get('length')) org_id = request.form.get('org_id') ip_address = request.form.get('ip_address') domain_address = request.form.get('domain_address') port = request.form.get('port') content = request.form.get('content') iplocation = request.form.get('iplocation') port_status = request.form.get('port_status') color_tag = request.form.get('color_tag') memo_content = request.form.get('memo_content') date_delta = request.form.get('date_delta') session['ip_address_ip'] = ip_address session['domain_address'] = domain_address session['port'] = port session['session_org_id'] = org_id count = 0 ips = ip_table.gets_by_search(org_id=org_id, domain=domain_address, ip=ip_address, port=port, content=content, iplocation=iplocation, port_status=port_status, color_tag=color_tag, memo_content=memo_content, date_delta=date_delta, page=(start // length) + 1, rows_per_page=length) if ips: for ip_row in ips: # 查询每一个IP的详细属性 port_list, title_set, banner_set, _, port_status_dict = aip.get_ip_port_info( ip_row['ip'], ip_row['id']) # 端口+HTTP状态码 port_with_status_list = [] for p in port_list: if str(p) in port_status_dict and re.match( r'^\d{3}$', port_status_dict[str(p)]): port_with_status_list.append("{}[{}]".format( p, port_status_dict[str(p)])) else: port_with_status_list.append(str(p)) # 获取颜色标记 color_tag_obj = ip_color_tag_table.get(ip_row['id']) # 获取备忘录信息 memo_obj = ip_memo_table.get(ip_row['id']) # 获取IP关联的漏洞信息: vul_info = [] vul_results = Vulnerability().gets({'target': ip_row['ip']}) if vul_results and len(vul_results) > 0: for v in vul_results: vul_info.append('{}/{}'.format(v['poc_file'], v['source'])) # 显示的数据 ip_list.append({ 'id': ip_row['id'], "index": index + start, 'color_tag': color_tag_obj['color'] if color_tag_obj else '', 'memo_content': memo_obj['content'] if memo_obj else '', 'vulnerability': '\r\n'.join(vul_info), "org_name": org_table.get(int(ip_row['org_id']))['org_name'] if ip_row['org_id'] else '', "ip": ip_row['ip'], "status": ip_row['status'], "location": ip_row['location'].split(',')[0] if ip_row['location'] else '', "create_time": str(ip_row['create_datetime']), "update_time": str(ip_row['update_datetime']), "port": port_with_status_list, "title": ', '.join(list(title_set)), "banner": ', '.join(list(banner_set)) }) index += 1 # 查询的记录数量 count = ip_table.count_by_search(org_id=org_id, domain=domain_address, ip=ip_address, port=port, content=content, iplocation=iplocation, port_status=port_status, color_tag=color_tag, memo_content=memo_content, date_delta=date_delta) json_data = { 'draw': draw, 'recordsTotal': count, 'recordsFiltered': count, 'data': ip_list } except Exception as e: logger.error(traceback.format_exc()) print(e) return jsonify(json_data)
def domain_asset_view(): '''页面上显示域名资产,datatable前端ajax请求进行分页 ''' if request.method == 'GET': org_table = Organization() org_list = org_table.gets() return render_template('domain-list.html', org_list=org_list) domain_list = [] ip_table = Ip() org_table = Organization() domain_table = Domain() domain_attr_table = DomainAttr() index = 1 try: draw = int(request.form.get('draw')) start = int(request.form.get('start')) length = int(request.form.get('length')) org_name = request.form.get('org_name') ip_address = request.form.get('ip_address') domain_address = request.form.get('domain_address') domains = domain_table.gets_by_org_domain_ip(org_name, domain_address, ip_address, page=start // length + 1, rows_per_page=length) for domain_row in domains: ips = domain_attr_table.gets(query={ 'tag': 'A', 'r_id': domain_row['id'] }) domain_list.append({ 'id': domain_row['id'], "index": index + start, "domain": domain_row['domain'], "ip": ', '.join( set([ '<a href="/ip-info?ip={0}">{0}</a>'.format( ip_row['content']) for ip_row in ips ])), "org_name": org_table.get(int(domain_row['org_id']))['org_name'] if domain_row['org_id'] else '', "create_time": str(domain_row['create_datetime']), "update_time": str(domain_row['update_datetime']) }) index += 1 count = domain_table.count_by_org_domain_ip(org_name, domain_address, ip_address) json_data = { 'draw': draw, 'recordsTotal': count, 'recordsFiltered': count, 'data': domain_list } except Exception as e: print(e) pass return jsonify(json_data)
def delete_ip_view(ip_id): '''删除一个IP ''' rows = Ip().delete(ip_id) return jsonify({'status': 'success', 'msg': rows})
def ip_asset_view(): '''IP资产列表展示 ''' if request.method == 'GET': org_table = Organization() org_list = org_table.gets() return render_template('ip-list.html', org_list=org_list) ip_table = Ip() port_table = Port() org_table = Organization() ip_list = [] json_data = {} index = 1 try: draw = int(request.form.get('draw')) start = int(request.form.get('start')) length = int(request.form.get('length')) org_name = request.form.get('org_name') ip_address = request.form.get('ip_address') port = request.form.get('port') ips = ip_table.gets_by_org_ip_port(org_name, ip_address, port, page=(start // length) + 1, rows_per_page=length) for ip_row in ips: ip_list.append({ 'id': ip_row['id'], # 表内序号 "index": index + start, # 显示序号 "org_name": org_table.get(int(ip_row['org_id']))['org_name'] if ip_row['org_id'] else '', "ip": ip_row['ip'], "status": ip_row['status'], "location": ip_row['location'], "port": ', '.join([ '{}'.format(row['port']) for row in port_table.gets(query={'ip_id': ip_row['id']}) ]), "create_time": str(ip_row['create_datetime']), "update_time": str(ip_row['update_datetime']) }) index += 1 count = ip_table.count_by_org_ip_port(org_name, ip_address, port) json_data = { 'draw': draw, 'recordsTotal': count, 'recordsFiltered': count, 'data': ip_list } except Exception as e: print(e) return jsonify(json_data)
def test1(): from nemo.core.database.ip import Ip ip = Ip() data = ip.gets(query={'ip': '192.168.3.10'}) print(data)
def get_domain_info(self, Id): '''聚合一个DOMAIN的详情 ''' domain_info = {} # 获取DOMAIN domain_obj = Domain().get(Id) if not domain_obj: return None domain_info.update( id=domain_obj['id'], domain=domain_obj['domain'], create_datetime=domain_obj['create_datetime'].strftime( '%Y-%m-%d %H:%M'), update_datetime=domain_obj['update_datetime'].strftime( '%Y-%m-%d %H:%M')) # 获取组织名称 if domain_obj['org_id']: organziation__obj = Organization().get(domain_obj['org_id']) if organziation__obj: domain_info.update(organization=organziation__obj['org_name']) else: domain_info.update(organization='') domain_attrs_obj = DomainAttr().gets(query={'r_id': domain_obj['id']}) # 获取域名的属性信息:title和ip,whatweb title_set = set() banner_set = set() ip_set = set() whatweb_set = set() httpx_set = set() for domain_attr_obj in domain_attrs_obj: if domain_attr_obj['tag'] == 'title': title_set.add(domain_attr_obj['content']) elif domain_attr_obj['tag'] == 'A': ip_set.add(domain_attr_obj['content']) elif domain_attr_obj['tag'] == 'whatweb': whatweb_set.add(domain_attr_obj['content']) elif domain_attr_obj['tag'] == 'server': banner_set.add(domain_attr_obj['content']) elif domain_attr_obj['tag'] == 'httpx': httpx_set.add(domain_attr_obj['content']) # 获取域名关联的IP端口详情: port_set = set() ip_port_list = [] for domain_ip in ip_set: ip_obj = Ip().gets(query={'ip': domain_ip}) if ip_obj and len(ip_obj) > 0: # port_list, title_set, banner_set, ports_attr_info p, t, b, pai, ps = self.get_ip_port_info( ip_obj[0]['ip'], ip_obj[0]['id']) port_set.update(p) title_set.update(t) banner_set.update(b) ip_port_list.extend(pai) domain_info.update(ip=list(ip_set)) domain_info.update(port=list(port_set)) domain_info.update(title=list(title_set)) domain_info.update(whatweb=list(whatweb_set)) domain_info.update(httpx=list(httpx_set)) domain_info.update(banner=list(banner_set)) domain_info.update(port_attr=ip_port_list) # 获取标记颜色: color_tag_obj = DomainColorTag().get(domain_obj['id']) domain_info.update( color_tag=color_tag_obj['color'] if color_tag_obj else '') # 获取备忘录信息: memo_obj = DomainMemo().get(domain_obj['id']) domain_info.update(memo=memo_obj['content'] if memo_obj else '') # 获取关联的漏洞信息: vul_results = Vulnerability().gets({'target': domain_obj['domain']}) if vul_results and len(vul_results) > 0: vul_info = [] for v in vul_results: vul_info.append({ 'id': v['id'], 'target': v['target'], 'url': v['url'], 'poc_file': v['poc_file'], 'source': v['source'], 'update_datetime': v['update_datetime'].strftime('%Y-%m-%d %H:%M') }) domain_info.update(vulnerability=vul_info) else: domain_info.update(vulnerability=None) return domain_info
def ip_asset_view(): '''IP资产列表展示 ''' if request.method == 'GET': org_table = Organization() org_list = org_table.gets() if not org_list: org_list = [] org_list.insert(0, {'id': '', 'org_name': '--组织机构--'}) data = {'org_list': org_list, 'ip_address_ip': session.get('ip_address_ip', default=''), 'domain_address': session.get('domain_address', default=''), 'port': session.get('port', default=''), 'session_org_id': session.get('session_org_id', default='')} return render_template('ip-list.html', data=data) ip_table = Ip() org_table = Organization() aip = AssertInfoParser() ip_list = [] json_data = {} index = 1 try: draw = int(request.form.get('draw')) start = int(request.form.get('start')) length = int(request.form.get('length')) org_id = request.form.get('org_id') ip_address = request.form.get('ip_address') domain_address = request.form.get('domain_address') port = request.form.get('port') content = request.form.get('content') iplocation = request.form.get('iplocation') session['ip_address_ip'] = ip_address session['domain_address'] = domain_address session['port'] = port session['session_org_id'] = org_id count = 0 ips = ip_table.gets_by_search(org_id=org_id, domain=domain_address, ip=ip_address, port=port, content=content, iplocation=iplocation, page=(start//length)+1, rows_per_page=length) if ips: for ip_row in ips: port_list, title_set, banner_set, ports_attr_info = aip.get_ip_port_info( ip_row['ip'], ip_row['id']) ip_list.append({ 'id': ip_row['id'], "index": index+start, "org_name": org_table.get(int(ip_row['org_id']))['org_name'] if ip_row['org_id'] else '', "ip": ip_row['ip'], "status": ip_row['status'], "location": ip_row['location'].split(',')[0] if ip_row['location'] else '', "create_time": str(ip_row['create_datetime']), "update_time": str(ip_row['update_datetime']), "port": port_list, "title": ', '.join(list(title_set)), "banner": ', '.join(list(banner_set)) }) index += 1 count = ip_table.count_by_search(org_id=org_id, domain=domain_address, ip=ip_address, port=port, content=content, iplocation=iplocation) json_data = { 'draw': draw, 'recordsTotal': count, 'recordsFiltered': count, 'data': ip_list } except Exception as e: logger.error(traceback.format_exc()) print(e) return jsonify(json_data)
def get_ip_info(self, Id): '''聚合一个IP的详情 ''' ip_info = {} # 获取IP ip_obj = Ip().get(Id) if not ip_obj: return None ip_info.update(id=ip_obj['id'], ip=ip_obj['ip'], location=ip_obj['location'], status=ip_obj['status'], create_datetime=ip_obj['create_datetime'].strftime( '%Y-%m-%d %H:%M'), update_datetime=ip_obj['update_datetime'].strftime( '%Y-%m-%d %H:%M')) # 获取组织名称 if ip_obj['org_id']: organziation__obj = Organization().get(ip_obj['org_id']) if organziation__obj: ip_info.update(organization=organziation__obj['org_name']) else: ip_info.update(Organization='') # 端口、标题、banner、端口详情 port_list, title_set, banner_set, ports_attr_info, port_status_dict = self.get_ip_port_info( ip_obj['ip'], ip_obj['id']) ip_info.update(port_attr=ports_attr_info) ip_info.update(title=list(title_set)) ip_info.update(banner=list(banner_set)) ip_info.update(port=port_list) # IP关联的域名 domain_set = self.__get_ip_domain(ip_obj['ip']) ip_info.update(domain=list(domain_set)) # 获取标记颜色: color_tag_obj = IpColorTag().get(ip_obj['id']) ip_info.update( color_tag=color_tag_obj['color'] if color_tag_obj else '') # 获取备忘录信息: memo_obj = IpMemo().get(ip_obj['id']) ip_info.update(memo=memo_obj['content'] if memo_obj else '') # 获取IP关联的漏洞信息: vul_results = Vulnerability().gets({'target': ip_obj['ip']}) if vul_results and len(vul_results) > 0: vul_info = [] for v in vul_results: vul_info.append({ 'id': v['id'], 'target': v['target'], 'url': v['url'], 'poc_file': v['poc_file'], 'source': v['source'], 'update_datetime': v['update_datetime'].strftime('%Y-%m-%d %H:%M') }) ip_info.update(vulnerability=vul_info) else: ip_info.update(vulnerability=None) return ip_info