def log_failed_message(listid, srctype, src, msg, err):
try:
msgid = msg.msgid
except:
msgid = "<unknown>"
log.error("Failed to load message (msgid %s) from %s, spec %s: %s" % (msgid.encode('us-ascii', 'replace'), srctype, src, str(str(err), 'us-ascii', 'replace')))
# We also put the data in the db. This happens in the main transaction
# so if the whole script dies, it goes away...
conn.cursor().execute("INSERT INTO loaderrors (listid, msgid, srctype, src, err) VALUES (%(listid)s, %(msgid)s, %(srctype)s, %(src)s, %(err)s)", {
'listid': listid,
'msgid': msgid,
'srctype': srctype,
'src': src,
'err': str(str(err), 'us-ascii', 'replace'),
})
def check_http_status(scheme, host, port, task_name, task_id, tag_name):
"""
检测端口的连通
状态码的有效性
压入redis，供bbscan扫描
:param scheme:
:param host:
:param port:
:return:
"""
url = "%s://%s:%s" % (scheme, host, port)
try:
# s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# s.settimeout(5.0)
# if s.connect_ex((url, int(port))) == 0:
log.info('Checking Http Status Valid: %s', url)
status_code, headers, content = http_request(url)
if status_code:
location = headers.get('Location', '')
if status_code not in HTTP.IGNORE_CODE:
if status_code in [301, 302
] and location.startswith("https://"):
scheme, host, port = get_hostname_port(location)
status_code, headers, content = http_request(location)
m = re.search('<title>(.*?)</title>', decode_text(content))
title = m.group(1) if m else ''
header = get_headers(headers)
banner = header + decode_text(content)
rdata = {
'scheme': scheme,
'port': port,
'ip': host,
'title': title,
'status_code': status_code,
"banner": banner,
'task_name': task_name,
'task_id': task_id,
'tag_name': tag_name
}
redis_conn.lpush("BBScan_First",
json.dumps(rdata)) #压入redis，bbscan来解析扫描
return True
except Exception as e:
log.error('[Warning] Get http connection failed %s:%s' % (host, port),
exc_info=True)
return False
def reverse_host(host):
"""
解析域名获取IP
:param host: hostname
:return: list
"""
ret = []
try:
resolver = dns.resolver.Resolver()
resolver.nameservers = NAMESERVERS
answer = resolver.query(host, 'A')
for i in answer:
ret.append(i.address)
except Exception:
log.error("[Warning]: resolving hostname failed: %s", host)
return
return ret
def setup_callback():
global serversocket, callback_ip, callback_port, local_port, portmap_port
# Don't try to map ports if we have a public IP
callback_ip = callback_port = None
using_portmap = False
local_ip = network.get_local_ip()
if not network.is_private_ip(local_ip):
log.info("Host has a public IP...")
callback_ip = local_ip
elif config.get("port"):
log.info("Host has a private IP, port specified, configure forward " +
"manually...")
callback_ip = network.get_public_ip()
else:
log.info("Host has a private IP, trying upnp port mapping...")
network.portmap_setup()
network.portmap_search()
callback_ip = network.get_public_ip()
using_portmap = True
if not callback_ip:
log.error("No public IP found, not registering callback.")
return
if not serversocket:
serversocket, local_port = bind_port()
log.info("Listening on port {0}...".format(local_port))
serversocket.listen(5) # max incoming calls queued
if not using_portmap:
callback_port = local_port
else:
portmap_port = network.portmap_add(portmap_port, local_port)
if not portmap_port:
log.error("Failed to map port, not registering callback.")
return
callback_port = portmap_port
sync_obj.populate()
for acc in sync_obj.get_bunq_accounts():
url = "https://{}:{}/bunq2ynab-autosync".format(
callback_ip, callback_port)
bunq_api.add_callback(acc["bunq_user_id"], acc["bunq_account_id"],
"bunq2ynab-autosync", url)
def portmap_add(try_port, local_port):
if not upnp:
return
if not try_port:
try_port = local_port
log.info("Adding upnp port mapping...")
for i in range(0, 128):
try:
upnp.addportmapping(try_port, 'TCP', upnp.lanaddr, local_port,
'bynq2ynab-autosync', '')
return try_port
except Exception as e:
if "ConflictInMappingEntry" not in str(e):
log.error("Failed to map port: {}".format(e))
return
log.info("Port {} is already mapped, trying next port...".format(
try_port))
try_port = next_port(try_port)
def task(self, hostname):
"""
执行采集器，拿到采集结果汇报给API
1. 执行所有的采集器拿到info
2. 汇报info给api
:param hostname:
:return:
"""
try:
info = get_server_info(self, hostname)
r1 = requests.post(url=self.asset_api,
data=json.dumps(info).encode('utf-8'),
headers={'Content-Type': 'application/json'})
print(r1)
except Exception as e:
msg = traceback.format_exc()
log.error(msg)
print(msg)
def log_failed_message(listid, srctype, src, msg, err):
try:
msgid = msg.msgid
except Exception:
msgid = "<unknown>"
log.error("Failed to load message (msgid %s) from %s, spec %s: %s" %
(msgid, srctype, src, err))
# We also put the data in the db. This happens in the main transaction
# so if the whole script dies, it goes away...
conn.cursor().execute(
"INSERT INTO loaderrors (listid, msgid, srctype, src, err) VALUES (%(listid)s, %(msgid)s, %(srctype)s, %(src)s, %(err)s)",
{
'listid': listid,
'msgid': msgid,
'srctype': srctype,
'src': src,
'err': str(err),
})
def init_run(self):
"""
增加301跳转的判断
:return:
"""
try:
status_code, headers, content = http_request(self.url)
location = headers.get('Location', '')
if status_code in [301, 302] and location.startwith("https://"):
scheme, host, port = get_hostname_port(location)
self.url = "%s://%s:%s" % (scheme, host, port)
status_code, headers, content = http_request(self.url)
m = re.search('<title>(.*?)</title>', content)
title = m.group(1) if m else ''
self.header = headers
self.title = title
self.content = content
self.get_all_uri()
except:
log.error("running bbscan failed", exc_info=True)
def check_port_open(host, port):
"""
检测端口连通性
:param host:
:param port:
:return:
"""
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(5.0)
try:
if s.connect_ex((host, int(port))) == 0:
log.info("Check port open %s:%s is Open", host, port)
return True
except Exception:
log.error('[Warning] Fail to connect to %s:%s' % (host, port),
exc_info=True)
finally:
# s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
s.close()
log.info("Check port Close %s:%s is Close", host, port)
return False
def _check_404_status(self):
try:
try:
c = requests.get(self.url +
'/fjkgofdjgosdjfjd-404-existence-check',
verify=False,
timeout=HTTP_TIMEOUT,
allow_redirects=False)
self._404_status = c.status_code
content = c.content
except:
log.error('[Warning] HTTP 404 check failed <%s>',
self.url,
exc_info=True)
self._404_status, content = -1, ''
if self._404_status == 404:
self.has_status_404 = True
else:
self.has_status_404 = False
self.len_404_doc = len(content)
except:
log.error('[Check_404] Exception %s', self.url, exc_info=True)
def scheduler_bbscan_scan_first():
while redis_conn.llen("BBScan_First"):
try:
http_object = json.loads(
redis_conn.lpop("BBScan_First")) # 获取第一步待扫描的内容
scheme = http_object['scheme']
ip = http_object['ip']
port = http_object['port']
header = http_object['banner'].split("\r\n\r\n\r\n")[0]
content = http_object['banner'].split("\r\n\r\n\r\n")[1]
status_code = http_object['status_code']
task_name = http_object['task_name']
task_id = http_object['task_id']
tag_name = http_object['tag_name']
m = re.search('<title>(.*?)</title>', content)
title = m.group(1) if m else ''
bbscan_parse_uri.delay(scheme, ip, port, title, content,
status_code, header, task_name, task_id,
tag_name)
except:
log.error("scheduler_bbscan_scan_first", exc_info=True)
def linux(self, handler, hostname):
response = BaseResponse()
try:
if self.debug:
ret = {
'os_platform': 'linux',
'os_version': '6.5',
'hostname': 'c1.com'
}
else:
ret = {
'os_platform': self.os_platform(handler, hostname),
'os_version': self.os_version(handler, hostname),
'hostname': self.os_hostname(handler, hostname),
}
response.data = ret
except Exception as e:
msg = traceback.format_exc()
response.status = False
response.error = msg
log.error(msg)
return response.dict
def save_res_to_redis(self, url):
try:
if url == "/":
for k in redis_conn_byte.scan_iter("bbscan_rules*"):
rule_list = redis_conn_byte.get(k).decode('utf-8').split(
',')
rule_list[0] = self.url + rule_list[0]
ret = ','.join(rule_list) + ',' + str(self._404_status) + ',' + str(self.len_404_doc)+','+ \
self.task_name + ',' + self.task_id + ',' + self.tag_name
# log.info("store bbscan root request to redis %s", ret)
redis_conn_byte.lpush('BBScan_Second', ret)
else:
for k in redis_conn_byte.scan_iter("bbscan_rules_common::*"):
rule_list = redis_conn_byte.get(k).decode('utf-8').split(
',')
rule_list[0] = self.url + url.rstrip('/') + rule_list[0]
ret = ','.join(rule_list) + ',' + str(self._404_status) + ',' + str(self.len_404_doc)+','+ \
self.task_name + ',' + self.task_id + ',' + self.tag_name
# log.info("store bbscan common request to redis %s", ret)
redis_conn_byte.lpush('BBScan_Second', ret)
except:
log.error("save to redis error", exc_info=True)
pass
def get_iban_from_event(event):
body_str = event.get("body")
if not body_str:
log.info("No request body found")
return
try:
body = json.loads(body_str)
except json.JSONDecodeError as e:
log.error("Error decoding quest body as JSON: {}".format(e))
return
nu = body.get("NotificationUrl", {})
category = nu.get("category")
if category != "MUTATION":
log.error("Category is not MUTATION but {}".format(e))
return
iban = nu.get("object", {}).get("Payment", {}).get("alias", {}).get("iban")
if not iban:
log.error("No IBAN found in request body")
return
log.info("Found IBAN {} in request body".format(iban))
return iban
# We could do this "properly" by reordering operations and using ON CONFLICT,
# but concurrency is not that important and this is easier...
try:
curs.execute("SET statement_timeout='30s'")
curs.execute("SELECT pg_advisory_xact_lock(8059944559669076)")
except Exception as e:
print(("Failed to wait on advisory lock: %s" % e))
sys.exit(1)
# Get the listid we're working on
curs.execute("SELECT listid FROM lists WHERE listname=%(list)s", {
'list': opt.list
})
r = curs.fetchall()
if len(r) != 1:
log.error("List %s not found" % opt.list)
conn.close()
sys.exit(1)
listid = r[0][0]
purges = set()
if opt.directory:
# Parse all files in directory
for x in os.listdir(opt.directory):
log.status("Parsing file %s" % x)
with open(os.path.join(opt.directory, x)) as f:
ap = ArchivesParserStorage()
ap.parse(f)
if opt.filter_msgid and not ap.is_msgid(opt.filter_msgid):
continue
try:
network.portmap_remove(portmap_port)
except Exception as e:
log.error("Error removing upnp port mapping: {}".format(e))
# ----- Main loop
try:
while True:
try:
sync_obj = sync.Sync()
setup_callback()
log.info("Starting periodic synchronization...")
synchronize()
if callback_ip and callback_port:
wait_for_callback()
else:
log.warning("No callback, waiting for {} minutes...".format(
refresh_nocallback_minutes))
time.sleep(refresh_nocallback_minutes * 60)
except Exception as e:
log.error("Error: {}".format(e))
log.error(traceback.format_exc())
log.error("Error occured, waiting 10 seconds.")
time.sleep(10)
finally:
teardown_callback()
# We could do this "properly" by reordering operations and using ON CONFLICT,
# but concurrency is not that important and this is easier...
try:
curs.execute("SET statement_timeout='30s'")
curs.execute("SELECT pg_advisory_xact_lock(8059944559669076)")
except Exception as e:
print(("Failed to wait on advisory lock: %s" % e))
sys.exit(1)
# Get the listid we're working on
curs.execute("SELECT listid FROM lists WHERE listname=%(list)s", {
'list': opt.list
})
r = curs.fetchall()
if len(r) != 1:
log.error("List %s not found" % opt.list)
conn.close()
sys.exit(1)
listid = r[0][0]
purges = set()
if opt.directory:
# Parse all files in directory
for x in os.listdir(opt.directory):
log.status("Parsing file %s" % x)
with open(os.path.join(opt.directory, x)) as f:
ap = ArchivesParserStorage()
ap.parse(f)
if opt.filter_msgid and not ap.is_msgid(opt.filter_msgid):
continue
def scan_init(task_name, task_id, target, tag_name):
"""
3个模块的初始化判断
1. 去除多余的空格
2. 校验是不是IP或者域名
3. 域名是否可以解析IP, IP是否可以连通
4. 如果可以连通，然后判断是否开启了端口扫描，可以的话进入到端口扫描流程
5. 判断运行的service
6. 根据运行的service，如果是http的话，进入到bbscan
7. 获取运行的service，进入到poc扫描,因为POC扫描只有两种service可识别，所以只需要判断是否是http服务就可以了。
:return:
"""
target = target.strip()
scheme, host, port = get_hostname_port(target)
target = '%s://%s:%s' % (scheme, host, port)
ips, hostname = get_host_ip(target) #获取hostname和ip地址
if ips:
ip = ips[0]
else:
return
iscdn = is_cdn(ip)
if len(ips) > 1:
log.info("Multi ip: %s", ips)
if iscdn:
db.portInfo.update_one({
"task_id": task_id,
"task_name": task_name
}, {"$set": {
"url": hostname,
"cdn": true
}})
log.info("CDN Check True: %s", target)
ports = list(db.task.find({"task_id": task_id}, {
"_id": 0,
"ports": 1
}))[0]['ports']
if ports and not iscdn:
_ = {
"task_name": task_name,
"task_id": task_id,
"tag_name": tag_name,
"ip": ip,
"hostname": hostname,
"ports": ports
}
log.info("Task Port Scan Begin %s", hostname)
redis_conn.lpush("Task_Port_Scan", json.dumps(_))
# service = None
# alive = None
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(5.0)
try:
if s.connect_ex((host, int(port))) == 0:
log.info("Port Open %s:%s", host, port)
if port == 443:
target = 'https://{}'.format(hostname)
service = 'https'
elif port == 80:
target = 'http://{}'.format(hostname)
service = 'http'
else:
service, content = http_detect(
host, port) # 端口开放的时候，判断运行的是什么服务，用来丢给POC扫描流程
if service == 'http':
if 'The plain HTTP request was sent to HTTPS port' in content:
service = 'https'
status_code, headers, content = http_request(target)
if status_code in (301, 302) and headers['Location']:
_ = {
"task_name": task_name,
"task_id": task_id,
"target": headers['Location'],
"tag_name": tag_name
}
log.info("Http %s redirect to %s", target, headers['Location'])
redis_conn.lpush("before_scan", json.dumps(_))
alive = True
# content = content.eocode() if content is isinstance(content, str) else content
if check_white_list(content): # 白名单储存到数据库
log.error("White List Check True")
db.bbscan.update_one(
{
"task_id": task_id,
"task_name": task_name,
"tag_name": tag_name
}, {"$set": {
"vul_Type": "white list",
"url": target
}})
# return
if service in ('http', 'https'): # http端口保存到资产数据库
m = re.search('<title>(.*?)</title>', content)
title = m.group(1) if m else ''
db.portInfo.update_one(
{
"task_id": task_id,
"task_name": task_name,
"tag_name": tag_name,
"ip": ip,
"port": port
}, {
"$set": {
"server": service,
"banner": content,
"title": title,
"hostname": hostname,
"url": target
}
},
upsert=True)
else:
log.info("Port Closed %s:%s", host, port)
service = False
alive = False
except Exception:
log.error('[Warning] Fail to connect to %s:%s' % (host, port),
exc_info=True)
return
finally:
s.close()
pocs = list(db.task.find({"task_id": task_id}, {
"_id": 0,
"pocs": 1
}))[0]['pocs']
if alive: # 主机是否存活
log.info("host %s is alive, Check Scan options", target)
if pocs and service: # service表示是否开启运行了http还是unknown
log.info("Begin POC Scan %s", target)
_ = {
"task_name": task_name,
"task_id": task_id,
"hostname": host,
"port": port,
"pocs": pocs,
"tag_name": tag_name,
"service": service
}
redis_conn.lpush("Task_Poc_Scan", json.dumps(_))
elif list(
db.task.find({"task_id": task_id}, {
"_id": 0,
"BBScan_flag": 1
}))[0]['BBScan_flag'] and service in ('http', 'https'):
_ = {
"task_name": task_name,
"task_id": task_id,
"target": target,
"tag_name": tag_name
}
redis_conn.lpush("BBScan_init", json.dumps(_))
else:
log.info("[Warning]: No POC Selected: %s", target)
pass
def bbscan(url, tag, status_to_match, content_type, content_type_no, vul_type,
status_404, len_404_content, task_name, task_id, tag_name):
status_to_match = int(status_to_match)
status_404 = int(status_404)
try:
status_code, headers, content = http_request(url)
cur_content_type = headers['Content-Type']
status = status_code
content = decode_text(content)
cur_content_length = len(content)
if check_black_list(content): # 在黑名单的的url返回
return
if 0 <= int(cur_content_length) <= 10: # text too short
return
if cur_content_type.find('image/') >= 0: # exclude image
return
if content_type != 'application/json' and cur_content_type.find('application/json') >= 0 and \
not url.endswith('.json'): # invalid json
return
if content_type and cur_content_type.find(content_type) < 0 \
or content_type_no and cur_content_type.find(content_type_no) >= 0:
return # content type mismatch
if tag and content.find(tag) < 0:
return # tag mismatch
if check_white_list(content):
valid_item = True
else:
if status_to_match == 206 and status != 206:
return
if status_to_match in (200, 206) and status in (200, 206):
valid_item = True
elif status_to_match and status != status_to_match:
return
elif status in (403, 404) and status != status_to_match:
return
else:
valid_item = True
if status == status_404 and url != '/':
len_doc = len(content)
len_sum = int(len_404_content) + len_doc
# print("bool is %s" % bool(0.4 <= float(len_doc) / len_sum <= 0.6))
if len_sum == 0 or (0.4 <= float(len_doc) / len_sum <= 0.6):
return
if valid_item:
vul_type = vul_type.replace('_', ' ')
m = re.search('<title>(.*?)</title>', content)
title = m.group(1) if m else ''
scheme, host, port = get_hostname_port(url)
vul_url = "%s://%s:%s" % (scheme, host, port)
first_find_date = datetime.datetime.now().strftime(
"%Y-%m-%d %H:%M:%S")
# log.info(bool(db.bbscan.find_one({"task_id": task_id, "url": url})))
if db.bbscan.find_one({
"task_id": task_id,
"tag_name": tag_name,
"vul_url": url
}): # 以task_id和url为主键查询条件
log.info("Get Vul Repeat %s", {"task_id": task_id, "url": url})
db.bbscan.update({
"task_id": task_id,
"url": url
}, {"$set": {
"last_find_date": first_find_date
}})
else:
log.info("Get Vul Success %s", {
"task_id": task_id,
"url": url
})
result = {
"task_name": task_name,
"task_id": task_id,
"tag_name": tag_name,
"vul_url": url,
"url": vul_url,
"vul_Type": vul_type,
"status": status,
"title": title,
"first_find_date": first_find_date,
"last_find_date": first_find_date
}
db.bbscan.insert(result)
except TypeError:
pass
except KeyError:
pass
except:
log.error("BBScan::process_request error %s", url, exc_info=True)
