def __fetch_iplocation_from_7188(self, ip):
'''调用www.hao7188.com的接口查询指定的IP地址
'''
url = 'http://www.hao7188.com/'
session = requests.Session()
try:
r1 = session.get(url, headers=self.headers)
if r1.status_code == requests.codes.ok:
pattern = r'name="so_token" value="(.*?)">'
m = re.findall(pattern, r1.text)
token = m[0] if m and len(m) > 0 else ''
if token != '':
url = 'http://www.hao7188.com/query.html?ip={}&so_token={}'.format(
ip, token)
r2 = session.get(url, headers=self.headers)
if r2.status_code == requests.codes.ok:
m = re.findall(
r"<script>window\.location\.href = '(.*?)'</script>",
r2.text)
new_href = m[0] if m and len(m) > 0 else ''
if new_href != '':
url_new = 'http://www.hao7188.com/{}'.format(
new_href)
r3 = session.get(url_new, headers=self.headers)
p = u'<span>(.*?)</span>'
m = re.findall(p, r3.text)
return m
except:
logger.error(traceback.format_exc())
logger.error('fetch ip location from 7188,ip:{}'.format(ip))
return None
def fetch_title(self, url):
'''调用httpx进行探测
'''
with NamedTemporaryFile('w+t') as tfile_url, NamedTemporaryFile(
'w+t') as tfile_output:
# 将目标写入文件中
tfile_url.write(url)
tfile_url.seek(0)
httpx_bin = [
os.path.join(self.BIN_PATH,
self.BIN_FILE), '-l', tfile_url.name, '-H',
'\'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/52.0.2743.116 Safari/537.36 Edge/15.15063\'',
'-status-code', '-title', '-web-server', '-follow-redirects',
'-no-color', '-content-type', '-silent', '-json', '-o',
tfile_output.name
]
# 调用httpx进行扫描
try:
child = subprocess.Popen(httpx_bin, stdout=subprocess.PIPE)
child.wait()
result = tfile_output.read()
except Exception as e2:
logger.error(traceback.format_exc())
logger.error('{} url:{}'.format(self.source, url))
result = None
print(e2)
return result
def get_tasks(self, limit=10, task_name=None, state=None):
'''获取所有task，包括已执行完成的
建议设置filter条件
state: PENDING,STARTED,SUCCESS,FAILURE,REVOKED,RETRY
#
Query Parameters:
limit – maximum number of tasks
workername – filter task by workername
taskname – filter tasks by taskname
state – filter tasks by state
'''
api = 'api/tasks'
url = '{}/{}'.format(self.api_host, api)
params = {}
if limit:
params['limit'] = limit
if state:
params['state'] = state
if task_name:
params['taskname'] = task_name
result = {}
try:
r = requests.get(url, auth=self.auth,
params=params, timeout=self.timeout)
result = self.__process_result(r)
except Exception as e:
logger.error(traceback.format_exc())
result['code'] = '-1'
result['status'] = str(e)
return result
def parse_result(self, content):
'''从httpx的返回中提取title和banner
'''
result = {}
if content and len(content) > 0:
try:
json_data = json.loads(content)
if 'title' in json_data and json_data['title']:
result.update(title=json_data['title'])
if 'webserver' in json_data and json_data['webserver']:
result.update(server=json_data['webserver'])
if 'status-code' in json_data and json_data['status-code']:
result.update(status=json_data['status-code'])
# 去除一些无谓的参数：
try:
json_data.pop('response-time')
json_data.pop('vhost')
json_data.pop('http2')
json_data.pop('content-length')
json_data.pop('method')
except:
pass
result.update(httpx=str(json_data))
except Exception as e:
logger.error(traceback.format_exc())
logger.error('{} content:{}'.format(self.source, content))
print(e)
return result
def __exe_whatweb(self, url):
'''调用nmap对指定IP和端口进行扫描
'''
with NamedTemporaryFile('w+t') as tfile_output:
whatweb_bin = [
self.whatweb_bin, '-q', '--color=never', '--log-brief',
tfile_output.name, '--max-threads',
str(self.whatweb_threads), '--open-timeout',
str(5), '--read-timeou',
str(10),
'-U=Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/52.0.2743.116 Safari/537.36 Edge/15.15063',
url
]
# 调用whatweb进行扫描
try:
child = subprocess.Popen(whatweb_bin, stdout=subprocess.PIPE)
child.wait()
result = tfile_output.read()
if result.startswith('ERROR'):
result = None
except Exception as e2:
logger.error(traceback.format_exc())
logger.error('whatweb url:{}'.format(url))
result = None
print(e2)
return result
def save_domain(self, data):
'''保存domain资产的相关结果
'''
domain_app = Domain()
doamin_attr_app = DomainAttr()
result = {'domain': len(data)}
for domain in data:
if 'domain' not in domain:
continue
if self.org_id:
domain['org_id'] = self.org_id
# 保存到domain
domain_id = domain_app.save_and_update(domain)
if domain_id > 0:
# 保存domain的属性
for attr_key in ('CNAME', 'A', 'title', 'whatweb', 'server', 'httpx'):
if attr_key in domain:
for attr_value in domain[attr_key]:
domain_attr = {'r_id': domain_id, 'source': self.source,
'tag': attr_key, 'content': attr_value[0:800]}
try:
doamin_attr_app.save_and_update(domain_attr)
except Exception as e:
logger.error(traceback.format_exc())
logger.error('save domain attr:{}-{}-{}'.format(domain['domain'], domain_attr['tag'],
domain_attr['content']))
return result
def __parse_masscan_output_file(self, output_results):
'''解析masscan的扫描结果
'''
results = []
port_service = ParsePortService()
try:
for line in output_results.split(os.linesep):
if line.strip() == '':
continue
if line.startswith('#'):
continue
data = line.strip().split(' ')
if data[0] == 'open' and data[1] == 'tcp':
ip = data[3].strip()
port = data[2].strip()
results.append({
'ip':
ip,
'status':
'alive',
'port': [{
'port': port,
'status': 'open',
'service': port_service.get_service(port)
}]
})
except Exception as e:
logger.error(traceback.format_exc())
return results
def vulnerability_list_view():
'''任务列表展示
'''
if request.method == 'GET':
return render_template('vulnerability-list.html')
vul_list = []
json_data = {}
index = 1
try:
draw = int(request.form.get('draw'))
start = int(request.form.get('start'))
length = int(request.form.get('length'))
vul_source = request.form.get('vul_source')
vul_target = request.form.get('vul_target')
vul_poc_file = request.form.get('vul_poc_file')
date_delta = request.form.get('date_delta')
vul_app = Vulnerability()
vul_results = vul_app.gets_by_search(target=vul_target,
poc_file=vul_poc_file,
source=vul_source,
date_delta=date_delta,
page=(start // length) + 1,
rows_per_page=length)
for row in vul_results:
vul = {
'id': row['id'],
'index': index,
'target': row['target'],
'url': row['url'],
'poc_file': row['poc_file'],
'source': row['source']
}
vul.update(create_datetime=row['create_datetime'].strftime(
"%Y-%m-%d %H:%M"))
vul.update(update_datetime=row['update_datetime'].strftime(
"%Y-%m-%d %H:%M"))
vul_list.append(vul)
index += 1
count = vul_app.count_by_search(
target=vul_target,
poc_file=vul_poc_file,
source=vul_source,
date_delta=date_delta,
)
json_data = {
'draw': draw,
'recordsTotal': count,
'recordsFiltered': count,
'data': vul_list
}
except Exception as e:
logger.error(traceback.format_exc())
print(e)
return jsonify(json_data)
def task_list_view():
'''任务列表展示
'''
if request.method == 'GET':
return render_template('task-list.html')
taskapi = TaskAPI()
data = []
try:
draw = int(request.form.get('draw'))
task_status = request.form.get('task_status')
task_limit = request.form.get('task_limit', default=None)
task_status = None if not task_status else task_status
task_result = taskapi.get_tasks(limit=task_limit, state=task_status)
if task_result['status'] == 'success':
for k, t in task_result['result'].items():
try:
task = {
'uuid':
t['uuid'],
'name':
t['name'].replace('nemo.core.tasks.tasks.',
'').replace('_', '-'),
'state':
t['state'],
'args':
t['args'],
'kwargs':
t['kwargs'],
'result':
t['result']
}
task.update(received=_format_datetime(t['received']))
task.update(started=_format_datetime(t['started']))
task.update(runtime=_format_runtime(t['runtime']))
# 获取worker信息
task_info = taskapi.get_task_info(t['uuid'])
if task_info['status'] == 'success':
task.update(worker=task_info['result']['worker'])
else:
task.update(worker='')
data.append(task)
except Exception as e:
print(e)
count = len(data)
json_data = {
'draw': draw,
'recordsTotal': count,
'recordsFiltered': count,
'data': data
}
return jsonify(json_data)
except Exception as e:
logger.error(traceback.format_exc())
print(e)
def get_connect_cursor():
""" get connect and cursor
"""
try:
con = connect_mysql()
cur = con.cursor()
return con, cur
except Exception as e:
logger.error(traceback.format_exc())
logger.error("cannot create mysql connect")
def __fetch_iplocation_from_qqwry(self, ip):
try:
q = QQwry()
q.load_file('nemo/common/thirdparty/qqwry/qqwry_lastest.dat')
result = q.lookup(ip)
return result
except:
logger.error(traceback.format_exc())
logger.error('fetch ip location from ipcn,ip:{}'.format(ip))
return None
def export_ips(org_id=None, domain_address=None, ip_address=None, port=None, content=None, iplocation=None,
port_status=None, color_tag=None, memo_content=None, date_delta=None):
'''导出IP为excel文件
'''
wb = load_workbook(template_file_ip)
ws = wb.active
ips = _get_ips(org_id, domain_address, ip_address,
port, content, iplocation, port_status, color_tag, memo_content, date_delta)
row_start = 2
for ip in ips:
merged_row_start = row_start
_copy_cell_style(ws, 2, row_start, 1, 9)
if ip['port_attr']:
for port in ip['port_attr']:
try:
_copy_cell_style(ws, 2, row_start, 1, 9)
ws.cell(column=5, row=row_start,
value="{0}".format(port['port']))
ws.cell(column=6, row=row_start,
value="{0}".format(port['source']))
ws.cell(column=7, row=row_start,
value="{0}".format(port['tag']))
ws.cell(column=8, row=row_start,
value="{0}".format(port['content']))
ws.cell(column=9, row=row_start,
value="{0}".format(port['update_datetime']))
except:
logger.error(traceback.format_exc())
finally:
row_start += 1
else:
row_start += 1
ws.merge_cells(start_row=merged_row_start, start_column=1,
end_row=row_start - 1, end_column=1)
ws.merge_cells(start_row=merged_row_start, start_column=2,
end_row=row_start - 1, end_column=2)
ws.merge_cells(start_row=merged_row_start, start_column=3,
end_row=row_start - 1, end_column=3)
ws.merge_cells(start_row=merged_row_start, start_column=4,
end_row=row_start - 1, end_column=4)
ws.cell(column=1, row=merged_row_start,
value="{0}".format(ip['index']))
ws.cell(column=2, row=merged_row_start, value="{0}".format(ip['ip']))
ws.cell(column=3, row=merged_row_start,
value="{0}".format('\n'.join(ip['domain'])))
ws.cell(column=4, row=merged_row_start, value="{0}".format(
ip['location'] if ip['location'] else ''))
with NamedTemporaryFile() as tmp:
wb.save(tmp.name)
tmp.seek(0)
data = tmp.read()
return data
def execute(self):
'''调用nmap执行扫描任务
'''
ip_ports = []
try:
ip_ports.extend(self.__nmap_scan(self.target, self.port))
except Exception as e:
logger.error(traceback.format_exc())
logger.error('nmap scan target:{},port:{}'.format(self.target,self.port))
return ip_ports
def close_cursor_connect(cur, con):
""" close cursor and connect
"""
try:
cur.close()
con.close()
return True
except Exception as e:
logger.error(traceback.format_exc())
logger.error("cannot close mysql connect or cusor")
return False
def __shodan_search(self, target):
'''查询FOFA
'''
api = shodan.Shodan(self.key)
try:
host = api.host(target)
return self.__parse_ip_port(host)
except Exception as ex:
logger.error(traceback.format_exc())
print(ex)
return None
def execute_process(con, cur, sql, param=None):
""" execute:内部调用
"""
cnt = 0
try:
cnt = cur.execute(sql, param)
con.commit()
except Exception as e:
con.rollback()
logger.error(traceback.format_exc())
logger.error("[sql]:{} [param]:{}".format(sql, param))
return cnt
def queryall_process(con, cur, sql, param=None):
"""" queryall:内部调用
"""
rows = None
try:
cur.execute(sql, param)
rows = cur.fetchall()
except Exception as e:
con.rollback()
logger.error(traceback.format_exc())
logger.error("[sql]:{} [param]:{}".format(sql, param))
return simple_list(rows)
def execute(self):
'''执行任务
'''
vul_results = []
try:
vul_results.extend(self.execute_verify())
except Exception as e:
logger.error(traceback.format_exc())
logger.error('{} verify target:{}'.format(self.source,
self.target))
return vul_results
def queryone_process(con, cur, sql, param=None):
"""" queryone:内部调用
"""
row = None
try:
cur.execute(sql, param)
row = cur.fetchone()
except Exception as e:
con.rollback()
logger.error(traceback.format_exc())
logger.error("[sql]:{} [param]:{}".format(sql, param))
return simple_value(row)
def run(self, options):
'''执行任务
'''
try:
self.prepare(options)
ip_ports = self.execute()
result = self.save_ip(ip_ports)
result['status'] = 'success'
return result
except Exception as e:
logger.error(traceback.format_exc())
return {'status': 'fail', 'msg': str(e)}
def login_check_view():
password = request.form.get('password')
if password == ProductionConfig.WEB_PASSWORD:
try:
session['login'] = '******'
logger.info('[login success] from {}'.format(request.remote_addr))
return redirect(url_for('index.view_index'))
except Exception as e:
logger.error(traceback.format_exc())
return redirect(url_for('authenticate.login_view'))
else:
logger.info('[login fail] from {}'.format(request.remote_addr))
return redirect(url_for('authenticate.login_view'))
def insertone_process(con, cur, sql, param):
""" insertone:内部调用
"""
lastrowid = 0
try:
cur.execute(sql, param)
con.commit()
lastrowid = cur.lastrowid
except Exception as e:
con.rollback()
logger.error(traceback.format_exc())
logger.error("[sql]:{} [param]:{}".format(sql, param))
return lastrowid
def __load_services(self):
'''读取映射关系:nmap-services
'''
try:
with open(self.nmap_services_file) as f:
for line in f:
if line.startswith('#'):
continue
'''ms-wbt-server\t3389/tcp\t0.083904\t# Microsoft Remote Display Protocol (aka ms-term-serv, microsoft-rdp) | MS WBT Server'''
datas = line.split('\t')
if datas and len(datas) >= 3:
service = datas[0]
port = datas[1]
self.port_service[port] = service
except FileNotFoundError:
logger.error('nmap-services file not found')
except:
logger.error(traceback.format_exc())
# 读取自定义的映射关系
try:
with open(self.custom_service_file) as f:
for line in f:
txt = line.strip()
if not txt:
continue
datas = txt.split(' ')
if datas and len(datas) >= 2:
port = datas[0].strip()
service = datas[1].strip()
self.custom_port_service[port] = service
except FileNotFoundError:
logger.error('custom-services file not found')
except:
logger.error(traceback.format_exc())
def run(self, options):
'''执行任务
'''
try:
self.prepare(options)
self.execute(self.target)
result = self.save_ip(self.target)
result.update(self.save_domain(self.target))
result['status'] = 'success'
return result
except Exception as e:
logger.error(traceback.format_exc())
return {'status': 'fail', 'msg': str(e)}
def __fill_where_by_search(self, org_id, domain, ip, color_tag,
memo_content, date_delta):
'''根据指定的字段，生成查询SQL语句和参数
'''
sql = []
param = []
link_word = ' where '
if org_id:
sql.append(link_word)
sql.append(' org_id=%s ')
param.append(org_id)
link_word = ' and '
if domain:
sql.append(link_word)
sql.append(' domain like %s')
param.append('%' + domain + '%')
link_word = ' and '
if ip:
sql.append(link_word)
sql.append(
' id in (select distinct r_id from domain_attr where tag="A" and content=%s)'
)
param.append(ip)
link_word = ' and '
if color_tag:
sql.append(link_word)
sql.append(
' id in (select r_id from domain_color_tag where color=%s)')
param.append(color_tag)
link_word = ' and '
if memo_content:
sql.append(link_word)
sql.append(
' id in (select r_id from domain_memo where content like %s)')
param.append('%' + memo_content + '%')
link_word = ' and '
if date_delta:
try:
days_span = int(date_delta)
if days_span > 0:
sql.append(link_word)
sql.append(' update_datetime between %s and %s ')
param.append(datetime.now() - timedelta(days=days_span))
param.append(datetime.now())
link_word = ' and '
except:
logger.error(traceback.format_exc())
logger.error('date delta error:{}'.format(date_delta))
return sql, param
def get_iplocation(self, ip):
'''获取IP地址
先查C段，如果没有C段则查B段
'''
if not check_ip_or_domain:
logger.error('check ip fail:{}'.format(ip))
return ''
datas = ip.split('.')
c_data = '{}.0/24'.format('.'.join(datas[0:3]))
location = self.ip_location_dict.get(c_data)
if not location:
b_data = '{}.0.0/16'.format('.'.join(datas[0:2]))
location = self.ip_location_dict.get(b_data)
return location if location else ''
def get_celery_workers(self):
'''得到worker的统计信息
'''
api = 'api/workers?refresh=1'
url = '{}/{}'.format(self.api_host, api)
result = {}
try:
r = requests.get(url, auth=self.auth, timeout=self.timeout)
result = self.__process_result(r)
except Exception as e:
logger.error(traceback.format_exc())
result['code'] = '-1'
result['status'] = str(e)
return result
def __fetch_same_domain(self, ip):
'''调用chinaz查询同IP的域名
'''
url = 'http://s.tool.chinaz.com/same?s={}'.format(ip)
try:
r = requests.get(url, headers=self.headers, timeout=self.timeout)
if r.status_code == requests.codes.ok:
p = r'target=_blank>(.*?)</a>'
m = re.findall(p, r.text)
return m
except:
logger.error(traceback.format_exc())
logger.error('fetch same domain of port:{}'.format(ip))
return []
def get_task_result(self, task_id):
'''获取一个task的执行结果
'''
api = 'api/task/result/{}'.format(task_id)
url = '{}/{}'.format(self.api_host, api)
result = {}
try:
r = requests.get(url, auth=self.auth, timeout=self.timeout)
result = self.__process_result(r)
except Exception as e:
logger.error(traceback.format_exc())
result['code'] = '-1'
result['status'] = str(e)
return result
def __get_userinfo(self):
'''获取API KEY用户信息
'''
try:
url = '{url}{api}'.format(url=self.base_url,
api=self.login_api_url)
data = {"email": self.email, 'key': self.key}
req = requests.get(url, params=data)
return req.json()
except requests.exceptions.ConnectionError:
error_msg = {"error": True, "errmsg": "Connect error"}
logger.error(traceback.format_exc())
logger.error(error_msg)
return error_msg
