def bbscan_parse_uri(scheme, ip, port, title, content, status_code, header,
task_name, task_id, tag_name):
"""
检查是否在白名单-> 从content解析二级路径-> 把路径和要检测的内容组合成新的url
检测白名单放在后面，更具有通用性
:param content:
:return:
"""
try:
if check_white_list(content):
data = {"ip": ip, "port": port, "vul_title": title}
redis_conn.lpush("VulScan",
json.dumps(data)) # TODO 存储BBScan白名单的结果到数据库
# return
else:
log.info("starting parse uri for %s://%s:%s", scheme, ip, port)
web = Web(scheme, ip, port, title, content, status_code, header,
task_name, task_id, tag_name)
web.init_run()
except:
log.error("celery task parse_uri error for %s://%s:%s",
scheme,
ip,
port,
exc_info=True)
def check_http_status(scheme, host, port, task_name, task_id, tag_name):
"""
检测端口的连通
状态码的有效性
压入redis，供bbscan扫描
:param scheme:
:param host:
:param port:
:return:
"""
url = "%s://%s:%s" % (scheme, host, port)
try:
# s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# s.settimeout(5.0)
# if s.connect_ex((url, int(port))) == 0:
log.info('Checking Http Status Valid: %s', url)
status_code, headers, content = http_request(url)
if status_code:
location = headers.get('Location', '')
if status_code not in HTTP.IGNORE_CODE:
if status_code in [301, 302
] and location.startswith("https://"):
scheme, host, port = get_hostname_port(location)
status_code, headers, content = http_request(location)
m = re.search('<title>(.*?)</title>', decode_text(content))
title = m.group(1) if m else ''
header = get_headers(headers)
banner = header + decode_text(content)
rdata = {
'scheme': scheme,
'port': port,
'ip': host,
'title': title,
'status_code': status_code,
"banner": banner,
'task_name': task_name,
'task_id': task_id,
'tag_name': tag_name
}
redis_conn.lpush("BBScan_First",
json.dumps(rdata)) #压入redis，bbscan来解析扫描
return True
except Exception as e:
log.error('[Warning] Get http connection failed %s:%s' % (host, port),
exc_info=True)
return False
def post(self):
parser = reqparse.RequestParser()
parser.add_argument('task_name', type=str)
parser.add_argument('task_target', type=str)
parser.add_argument('BBScan', type=bool)
parser.add_argument('port', type=str)
parser.add_argument('task_desc', type=str)
parser.add_argument('pocs', action='split')
parser.add_argument('tag_name', type=str)
args = parser.parse_args()
task_name = args.get('task_name')
task_target = args.get('task_target')
BBScan_flag = args.get('BBScan')
tag_name = args.get('tag_name')
ports = args.get('port')
task_desc = args.get('task_desc')
pocs = args.get('pocs')
task_id = str(uuid.uuid1())
task_date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
pocs = pocs.split('\n') if isinstance(pocs, str) else pocs
# print(pocs)
data = {
"task_id": task_id,
"task_name": task_name,
"task_target": task_target.split('\n'),
"BBScan_flag": BBScan_flag,
"tag_name": tag_name,
"ports": ports,
"pocs": pocs,
"task_desc": task_desc,
"task_add_date": task_date
}
db.task.insert_one(data)
for target in task_target.split('\n'):
_ = {
"task_name": task_name,
"task_id": task_id,
"target": target,
"tag_name": tag_name
}
redis_conn.lpush("before_scan", json.dumps(_))
def post(self):
try:
ramdom_id = str(uuid.uuid1())
parser = reqparse.RequestParser()
parser.add_argument('task_id', type=str)
args = parser.parse_args()
task_id = args.task_id
task_name = list(db.task.find({"task_id": task_id},{"_id":0, "task_name":1}))[0]['task_name'].split('_')[0] \
+ '_' + ramdom_id[:3]
task_target = list(db.task.find({"task_id": task_id},{"_id":0, "task_target":1}))[0]['task_target']
task_date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
BBScan_flag = list(db.task.find({"task_id": task_id},{"_id":0, "BBScan_flag":1}))[0]['BBScan_flag']
ports = list(db.task.find({"task_id": task_id},{"_id":0, "ports":1}))[0]['ports']
pocs = list(db.task.find({"task_id": task_id}, {"_id": 0, "pocs": 1}))[0]['pocs']
task_desc = list(db.task.find({"task_id": task_id},{"_id":0, "task_desc":1}))[0]['task_desc']
tag_name = list(db.task.find({"task_id": task_id}, {"_id": 0, "tag_name": 1}))[0]['tag_name']
data = {
"task_id": ramdom_id,
"task_name": task_name,
"task_target": task_target,
"tag_name": tag_name,
"BBScan_flag": BBScan_flag,
"ports": ports,
"pocs": pocs,
"task_desc": task_desc,
"task_add_date": task_date
}
db.task.insert_one(data)
for i in task_target:
_ = {
"task_name": task_name,
"task_id": ramdom_id,
"target": i,
"tag_name": tag_name
}
redis_conn.lpush("before_scan", json.dumps(_))
return jsonify({"msg": "ok"})
except Exception as ex:
print(str(ex))
return jsonify({"msg": "bad"})
def scan_init(task_name, task_id, target, tag_name):
"""
3个模块的初始化判断
1. 去除多余的空格
2. 校验是不是IP或者域名
3. 域名是否可以解析IP, IP是否可以连通
4. 如果可以连通，然后判断是否开启了端口扫描，可以的话进入到端口扫描流程
5. 判断运行的service
6. 根据运行的service，如果是http的话，进入到bbscan
7. 获取运行的service，进入到poc扫描,因为POC扫描只有两种service可识别，所以只需要判断是否是http服务就可以了。
:return:
"""
target = target.strip()
scheme, host, port = get_hostname_port(target)
target = '%s://%s:%s' % (scheme, host, port)
ips, hostname = get_host_ip(target) #获取hostname和ip地址
if ips:
ip = ips[0]
else:
return
iscdn = is_cdn(ip)
if len(ips) > 1:
log.info("Multi ip: %s", ips)
if iscdn:
db.portInfo.update_one({
"task_id": task_id,
"task_name": task_name
}, {"$set": {
"url": hostname,
"cdn": true
}})
log.info("CDN Check True: %s", target)
ports = list(db.task.find({"task_id": task_id}, {
"_id": 0,
"ports": 1
}))[0]['ports']
if ports and not iscdn:
_ = {
"task_name": task_name,
"task_id": task_id,
"tag_name": tag_name,
"ip": ip,
"hostname": hostname,
"ports": ports
}
log.info("Task Port Scan Begin %s", hostname)
redis_conn.lpush("Task_Port_Scan", json.dumps(_))
# service = None
# alive = None
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(5.0)
try:
if s.connect_ex((host, int(port))) == 0:
log.info("Port Open %s:%s", host, port)
if port == 443:
target = 'https://{}'.format(hostname)
service = 'https'
elif port == 80:
target = 'http://{}'.format(hostname)
service = 'http'
else:
service, content = http_detect(
host, port) # 端口开放的时候，判断运行的是什么服务，用来丢给POC扫描流程
if service == 'http':
if 'The plain HTTP request was sent to HTTPS port' in content:
service = 'https'
status_code, headers, content = http_request(target)
if status_code in (301, 302) and headers['Location']:
_ = {
"task_name": task_name,
"task_id": task_id,
"target": headers['Location'],
"tag_name": tag_name
}
log.info("Http %s redirect to %s", target, headers['Location'])
redis_conn.lpush("before_scan", json.dumps(_))
alive = True
# content = content.eocode() if content is isinstance(content, str) else content
if check_white_list(content): # 白名单储存到数据库
log.error("White List Check True")
db.bbscan.update_one(
{
"task_id": task_id,
"task_name": task_name,
"tag_name": tag_name
}, {"$set": {
"vul_Type": "white list",
"url": target
}})
# return
if service in ('http', 'https'): # http端口保存到资产数据库
m = re.search('<title>(.*?)</title>', content)
title = m.group(1) if m else ''
db.portInfo.update_one(
{
"task_id": task_id,
"task_name": task_name,
"tag_name": tag_name,
"ip": ip,
"port": port
}, {
"$set": {
"server": service,
"banner": content,
"title": title,
"hostname": hostname,
"url": target
}
},
upsert=True)
else:
log.info("Port Closed %s:%s", host, port)
service = False
alive = False
except Exception:
log.error('[Warning] Fail to connect to %s:%s' % (host, port),
exc_info=True)
return
finally:
s.close()
pocs = list(db.task.find({"task_id": task_id}, {
"_id": 0,
"pocs": 1
}))[0]['pocs']
if alive: # 主机是否存活
log.info("host %s is alive, Check Scan options", target)
if pocs and service: # service表示是否开启运行了http还是unknown
log.info("Begin POC Scan %s", target)
_ = {
"task_name": task_name,
"task_id": task_id,
"hostname": host,
"port": port,
"pocs": pocs,
"tag_name": tag_name,
"service": service
}
redis_conn.lpush("Task_Poc_Scan", json.dumps(_))
elif list(
db.task.find({"task_id": task_id}, {
"_id": 0,
"BBScan_flag": 1
}))[0]['BBScan_flag'] and service in ('http', 'https'):
_ = {
"task_name": task_name,
"task_id": task_id,
"target": target,
"tag_name": tag_name
}
redis_conn.lpush("BBScan_init", json.dumps(_))
else:
log.info("[Warning]: No POC Selected: %s", target)
pass