def __fetch_iplocation_from_7188(self, ip): '''调用www.hao7188.com的接口查询指定的IP地址 ''' url = 'http://www.hao7188.com/' session = requests.Session() try: r1 = session.get(url, headers=self.headers) if r1.status_code == requests.codes.ok: pattern = r'name="so_token" value="(.*?)">' m = re.findall(pattern, r1.text) token = m[0] if m and len(m) > 0 else '' if token != '': url = 'http://www.hao7188.com/query.html?ip={}&so_token={}'.format( ip, token) r2 = session.get(url, headers=self.headers) if r2.status_code == requests.codes.ok: m = re.findall( r"<script>window\.location\.href = '(.*?)'</script>", r2.text) new_href = m[0] if m and len(m) > 0 else '' if new_href != '': url_new = 'http://www.hao7188.com/{}'.format( new_href) r3 = session.get(url_new, headers=self.headers) p = u'<span>(.*?)</span>' m = re.findall(p, r3.text) return m except: logger.error(traceback.format_exc()) logger.error('fetch ip location from 7188,ip:{}'.format(ip)) return None
def fetch_title(self, url): '''调用httpx进行探测 ''' with NamedTemporaryFile('w+t') as tfile_url, NamedTemporaryFile( 'w+t') as tfile_output: # 将目标写入文件中 tfile_url.write(url) tfile_url.seek(0) httpx_bin = [ os.path.join(self.BIN_PATH, self.BIN_FILE), '-l', tfile_url.name, '-H', '\'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/52.0.2743.116 Safari/537.36 Edge/15.15063\'', '-status-code', '-title', '-web-server', '-follow-redirects', '-no-color', '-content-type', '-silent', '-json', '-o', tfile_output.name ] # 调用httpx进行扫描 try: child = subprocess.Popen(httpx_bin, stdout=subprocess.PIPE) child.wait() result = tfile_output.read() except Exception as e2: logger.error(traceback.format_exc()) logger.error('{} url:{}'.format(self.source, url)) result = None print(e2) return result
def get_tasks(self, limit=10, task_name=None, state=None): '''获取所有task,包括已执行完成的 建议设置filter条件 state: PENDING,STARTED,SUCCESS,FAILURE,REVOKED,RETRY # Query Parameters: limit – maximum number of tasks workername – filter task by workername taskname – filter tasks by taskname state – filter tasks by state ''' api = 'api/tasks' url = '{}/{}'.format(self.api_host, api) params = {} if limit: params['limit'] = limit if state: params['state'] = state if task_name: params['taskname'] = task_name result = {} try: r = requests.get(url, auth=self.auth, params=params, timeout=self.timeout) result = self.__process_result(r) except Exception as e: logger.error(traceback.format_exc()) result['code'] = '-1' result['status'] = str(e) return result
def parse_result(self, content): '''从httpx的返回中提取title和banner ''' result = {} if content and len(content) > 0: try: json_data = json.loads(content) if 'title' in json_data and json_data['title']: result.update(title=json_data['title']) if 'webserver' in json_data and json_data['webserver']: result.update(server=json_data['webserver']) if 'status-code' in json_data and json_data['status-code']: result.update(status=json_data['status-code']) # 去除一些无谓的参数: try: json_data.pop('response-time') json_data.pop('vhost') json_data.pop('http2') json_data.pop('content-length') json_data.pop('method') except: pass result.update(httpx=str(json_data)) except Exception as e: logger.error(traceback.format_exc()) logger.error('{} content:{}'.format(self.source, content)) print(e) return result
def __exe_whatweb(self, url): '''调用nmap对指定IP和端口进行扫描 ''' with NamedTemporaryFile('w+t') as tfile_output: whatweb_bin = [ self.whatweb_bin, '-q', '--color=never', '--log-brief', tfile_output.name, '--max-threads', str(self.whatweb_threads), '--open-timeout', str(5), '--read-timeou', str(10), '-U=Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/52.0.2743.116 Safari/537.36 Edge/15.15063', url ] # 调用whatweb进行扫描 try: child = subprocess.Popen(whatweb_bin, stdout=subprocess.PIPE) child.wait() result = tfile_output.read() if result.startswith('ERROR'): result = None except Exception as e2: logger.error(traceback.format_exc()) logger.error('whatweb url:{}'.format(url)) result = None print(e2) return result
def save_domain(self, data): '''保存domain资产的相关结果 ''' domain_app = Domain() doamin_attr_app = DomainAttr() result = {'domain': len(data)} for domain in data: if 'domain' not in domain: continue if self.org_id: domain['org_id'] = self.org_id # 保存到domain domain_id = domain_app.save_and_update(domain) if domain_id > 0: # 保存domain的属性 for attr_key in ('CNAME', 'A', 'title', 'whatweb', 'server', 'httpx'): if attr_key in domain: for attr_value in domain[attr_key]: domain_attr = {'r_id': domain_id, 'source': self.source, 'tag': attr_key, 'content': attr_value[0:800]} try: doamin_attr_app.save_and_update(domain_attr) except Exception as e: logger.error(traceback.format_exc()) logger.error('save domain attr:{}-{}-{}'.format(domain['domain'], domain_attr['tag'], domain_attr['content'])) return result
def __parse_masscan_output_file(self, output_results): '''解析masscan的扫描结果 ''' results = [] port_service = ParsePortService() try: for line in output_results.split(os.linesep): if line.strip() == '': continue if line.startswith('#'): continue data = line.strip().split(' ') if data[0] == 'open' and data[1] == 'tcp': ip = data[3].strip() port = data[2].strip() results.append({ 'ip': ip, 'status': 'alive', 'port': [{ 'port': port, 'status': 'open', 'service': port_service.get_service(port) }] }) except Exception as e: logger.error(traceback.format_exc()) return results
def vulnerability_list_view(): '''任务列表展示 ''' if request.method == 'GET': return render_template('vulnerability-list.html') vul_list = [] json_data = {} index = 1 try: draw = int(request.form.get('draw')) start = int(request.form.get('start')) length = int(request.form.get('length')) vul_source = request.form.get('vul_source') vul_target = request.form.get('vul_target') vul_poc_file = request.form.get('vul_poc_file') date_delta = request.form.get('date_delta') vul_app = Vulnerability() vul_results = vul_app.gets_by_search(target=vul_target, poc_file=vul_poc_file, source=vul_source, date_delta=date_delta, page=(start // length) + 1, rows_per_page=length) for row in vul_results: vul = { 'id': row['id'], 'index': index, 'target': row['target'], 'url': row['url'], 'poc_file': row['poc_file'], 'source': row['source'] } vul.update(create_datetime=row['create_datetime'].strftime( "%Y-%m-%d %H:%M")) vul.update(update_datetime=row['update_datetime'].strftime( "%Y-%m-%d %H:%M")) vul_list.append(vul) index += 1 count = vul_app.count_by_search( target=vul_target, poc_file=vul_poc_file, source=vul_source, date_delta=date_delta, ) json_data = { 'draw': draw, 'recordsTotal': count, 'recordsFiltered': count, 'data': vul_list } except Exception as e: logger.error(traceback.format_exc()) print(e) return jsonify(json_data)
def task_list_view(): '''任务列表展示 ''' if request.method == 'GET': return render_template('task-list.html') taskapi = TaskAPI() data = [] try: draw = int(request.form.get('draw')) task_status = request.form.get('task_status') task_limit = request.form.get('task_limit', default=None) task_status = None if not task_status else task_status task_result = taskapi.get_tasks(limit=task_limit, state=task_status) if task_result['status'] == 'success': for k, t in task_result['result'].items(): try: task = { 'uuid': t['uuid'], 'name': t['name'].replace('nemo.core.tasks.tasks.', '').replace('_', '-'), 'state': t['state'], 'args': t['args'], 'kwargs': t['kwargs'], 'result': t['result'] } task.update(received=_format_datetime(t['received'])) task.update(started=_format_datetime(t['started'])) task.update(runtime=_format_runtime(t['runtime'])) # 获取worker信息 task_info = taskapi.get_task_info(t['uuid']) if task_info['status'] == 'success': task.update(worker=task_info['result']['worker']) else: task.update(worker='') data.append(task) except Exception as e: print(e) count = len(data) json_data = { 'draw': draw, 'recordsTotal': count, 'recordsFiltered': count, 'data': data } return jsonify(json_data) except Exception as e: logger.error(traceback.format_exc()) print(e)
def get_connect_cursor(): """ get connect and cursor """ try: con = connect_mysql() cur = con.cursor() return con, cur except Exception as e: logger.error(traceback.format_exc()) logger.error("cannot create mysql connect")
def __fetch_iplocation_from_qqwry(self, ip): try: q = QQwry() q.load_file('nemo/common/thirdparty/qqwry/qqwry_lastest.dat') result = q.lookup(ip) return result except: logger.error(traceback.format_exc()) logger.error('fetch ip location from ipcn,ip:{}'.format(ip)) return None
def export_ips(org_id=None, domain_address=None, ip_address=None, port=None, content=None, iplocation=None, port_status=None, color_tag=None, memo_content=None, date_delta=None): '''导出IP为excel文件 ''' wb = load_workbook(template_file_ip) ws = wb.active ips = _get_ips(org_id, domain_address, ip_address, port, content, iplocation, port_status, color_tag, memo_content, date_delta) row_start = 2 for ip in ips: merged_row_start = row_start _copy_cell_style(ws, 2, row_start, 1, 9) if ip['port_attr']: for port in ip['port_attr']: try: _copy_cell_style(ws, 2, row_start, 1, 9) ws.cell(column=5, row=row_start, value="{0}".format(port['port'])) ws.cell(column=6, row=row_start, value="{0}".format(port['source'])) ws.cell(column=7, row=row_start, value="{0}".format(port['tag'])) ws.cell(column=8, row=row_start, value="{0}".format(port['content'])) ws.cell(column=9, row=row_start, value="{0}".format(port['update_datetime'])) except: logger.error(traceback.format_exc()) finally: row_start += 1 else: row_start += 1 ws.merge_cells(start_row=merged_row_start, start_column=1, end_row=row_start - 1, end_column=1) ws.merge_cells(start_row=merged_row_start, start_column=2, end_row=row_start - 1, end_column=2) ws.merge_cells(start_row=merged_row_start, start_column=3, end_row=row_start - 1, end_column=3) ws.merge_cells(start_row=merged_row_start, start_column=4, end_row=row_start - 1, end_column=4) ws.cell(column=1, row=merged_row_start, value="{0}".format(ip['index'])) ws.cell(column=2, row=merged_row_start, value="{0}".format(ip['ip'])) ws.cell(column=3, row=merged_row_start, value="{0}".format('\n'.join(ip['domain']))) ws.cell(column=4, row=merged_row_start, value="{0}".format( ip['location'] if ip['location'] else '')) with NamedTemporaryFile() as tmp: wb.save(tmp.name) tmp.seek(0) data = tmp.read() return data
def execute(self): '''调用nmap执行扫描任务 ''' ip_ports = [] try: ip_ports.extend(self.__nmap_scan(self.target, self.port)) except Exception as e: logger.error(traceback.format_exc()) logger.error('nmap scan target:{},port:{}'.format(self.target,self.port)) return ip_ports
def close_cursor_connect(cur, con): """ close cursor and connect """ try: cur.close() con.close() return True except Exception as e: logger.error(traceback.format_exc()) logger.error("cannot close mysql connect or cusor") return False
def __shodan_search(self, target): '''查询FOFA ''' api = shodan.Shodan(self.key) try: host = api.host(target) return self.__parse_ip_port(host) except Exception as ex: logger.error(traceback.format_exc()) print(ex) return None
def execute_process(con, cur, sql, param=None): """ execute:内部调用 """ cnt = 0 try: cnt = cur.execute(sql, param) con.commit() except Exception as e: con.rollback() logger.error(traceback.format_exc()) logger.error("[sql]:{} [param]:{}".format(sql, param)) return cnt
def queryall_process(con, cur, sql, param=None): """" queryall:内部调用 """ rows = None try: cur.execute(sql, param) rows = cur.fetchall() except Exception as e: con.rollback() logger.error(traceback.format_exc()) logger.error("[sql]:{} [param]:{}".format(sql, param)) return simple_list(rows)
def execute(self): '''执行任务 ''' vul_results = [] try: vul_results.extend(self.execute_verify()) except Exception as e: logger.error(traceback.format_exc()) logger.error('{} verify target:{}'.format(self.source, self.target)) return vul_results
def queryone_process(con, cur, sql, param=None): """" queryone:内部调用 """ row = None try: cur.execute(sql, param) row = cur.fetchone() except Exception as e: con.rollback() logger.error(traceback.format_exc()) logger.error("[sql]:{} [param]:{}".format(sql, param)) return simple_value(row)
def run(self, options): '''执行任务 ''' try: self.prepare(options) ip_ports = self.execute() result = self.save_ip(ip_ports) result['status'] = 'success' return result except Exception as e: logger.error(traceback.format_exc()) return {'status': 'fail', 'msg': str(e)}
def login_check_view(): password = request.form.get('password') if password == ProductionConfig.WEB_PASSWORD: try: session['login'] = '******' logger.info('[login success] from {}'.format(request.remote_addr)) return redirect(url_for('index.view_index')) except Exception as e: logger.error(traceback.format_exc()) return redirect(url_for('authenticate.login_view')) else: logger.info('[login fail] from {}'.format(request.remote_addr)) return redirect(url_for('authenticate.login_view'))
def insertone_process(con, cur, sql, param): """ insertone:内部调用 """ lastrowid = 0 try: cur.execute(sql, param) con.commit() lastrowid = cur.lastrowid except Exception as e: con.rollback() logger.error(traceback.format_exc()) logger.error("[sql]:{} [param]:{}".format(sql, param)) return lastrowid
def __load_services(self): '''读取映射关系:nmap-services ''' try: with open(self.nmap_services_file) as f: for line in f: if line.startswith('#'): continue '''ms-wbt-server\t3389/tcp\t0.083904\t# Microsoft Remote Display Protocol (aka ms-term-serv, microsoft-rdp) | MS WBT Server''' datas = line.split('\t') if datas and len(datas) >= 3: service = datas[0] port = datas[1] self.port_service[port] = service except FileNotFoundError: logger.error('nmap-services file not found') except: logger.error(traceback.format_exc()) # 读取自定义的映射关系 try: with open(self.custom_service_file) as f: for line in f: txt = line.strip() if not txt: continue datas = txt.split(' ') if datas and len(datas) >= 2: port = datas[0].strip() service = datas[1].strip() self.custom_port_service[port] = service except FileNotFoundError: logger.error('custom-services file not found') except: logger.error(traceback.format_exc())
def run(self, options): '''执行任务 ''' try: self.prepare(options) self.execute(self.target) result = self.save_ip(self.target) result.update(self.save_domain(self.target)) result['status'] = 'success' return result except Exception as e: logger.error(traceback.format_exc()) return {'status': 'fail', 'msg': str(e)}
def __fill_where_by_search(self, org_id, domain, ip, color_tag, memo_content, date_delta): '''根据指定的字段,生成查询SQL语句和参数 ''' sql = [] param = [] link_word = ' where ' if org_id: sql.append(link_word) sql.append(' org_id=%s ') param.append(org_id) link_word = ' and ' if domain: sql.append(link_word) sql.append(' domain like %s') param.append('%' + domain + '%') link_word = ' and ' if ip: sql.append(link_word) sql.append( ' id in (select distinct r_id from domain_attr where tag="A" and content=%s)' ) param.append(ip) link_word = ' and ' if color_tag: sql.append(link_word) sql.append( ' id in (select r_id from domain_color_tag where color=%s)') param.append(color_tag) link_word = ' and ' if memo_content: sql.append(link_word) sql.append( ' id in (select r_id from domain_memo where content like %s)') param.append('%' + memo_content + '%') link_word = ' and ' if date_delta: try: days_span = int(date_delta) if days_span > 0: sql.append(link_word) sql.append(' update_datetime between %s and %s ') param.append(datetime.now() - timedelta(days=days_span)) param.append(datetime.now()) link_word = ' and ' except: logger.error(traceback.format_exc()) logger.error('date delta error:{}'.format(date_delta)) return sql, param
def get_iplocation(self, ip): '''获取IP地址 先查C段,如果没有C段则查B段 ''' if not check_ip_or_domain: logger.error('check ip fail:{}'.format(ip)) return '' datas = ip.split('.') c_data = '{}.0/24'.format('.'.join(datas[0:3])) location = self.ip_location_dict.get(c_data) if not location: b_data = '{}.0.0/16'.format('.'.join(datas[0:2])) location = self.ip_location_dict.get(b_data) return location if location else ''
def get_celery_workers(self): '''得到worker的统计信息 ''' api = 'api/workers?refresh=1' url = '{}/{}'.format(self.api_host, api) result = {} try: r = requests.get(url, auth=self.auth, timeout=self.timeout) result = self.__process_result(r) except Exception as e: logger.error(traceback.format_exc()) result['code'] = '-1' result['status'] = str(e) return result
def __fetch_same_domain(self, ip): '''调用chinaz查询同IP的域名 ''' url = 'http://s.tool.chinaz.com/same?s={}'.format(ip) try: r = requests.get(url, headers=self.headers, timeout=self.timeout) if r.status_code == requests.codes.ok: p = r'target=_blank>(.*?)</a>' m = re.findall(p, r.text) return m except: logger.error(traceback.format_exc()) logger.error('fetch same domain of port:{}'.format(ip)) return []
def get_task_result(self, task_id): '''获取一个task的执行结果 ''' api = 'api/task/result/{}'.format(task_id) url = '{}/{}'.format(self.api_host, api) result = {} try: r = requests.get(url, auth=self.auth, timeout=self.timeout) result = self.__process_result(r) except Exception as e: logger.error(traceback.format_exc()) result['code'] = '-1' result['status'] = str(e) return result
def __get_userinfo(self): '''获取API KEY用户信息 ''' try: url = '{url}{api}'.format(url=self.base_url, api=self.login_api_url) data = {"email": self.email, 'key': self.key} req = requests.get(url, params=data) return req.json() except requests.exceptions.ConnectionError: error_msg = {"error": True, "errmsg": "Connect error"} logger.error(traceback.format_exc()) logger.error(error_msg) return error_msg