def _get_dbip_geoinfo(self, root: IP, task: IscoutTask, level,
obj: ScoutFeedBackBase):
"""
dbip获取geoinfo
:param task:
:param level:
:param obj:
:return:
"""
ip = obj.value
try:
db = DbipMmdb()
gobj, org, isp = db.get_ip_mmdbinfo(level, ip)
root.org = org
root.isp = isp
if gobj is not None:
task.success_count()
self.__set_value(root, gobj)
root = self.__segment_output(root, level, ip)
yield gobj
else:
task.fail_count()
# for gobj in db.get_ip_geoinfo(level, ip):
# self.__set_value(root, gobj)
# root = self.__segment_output(root, level, ip)
# yield gobj
except:
self._logger.error(
"Get dbip geoinfo error:\ntaskid:{}\nbatchid:{}\nrootobject:{}\nobjtype:{}"
.format(task.taskid, task.batchid, obj.value,
obj._objtype.name))
def __sonar_get_email(
self, root: Domain, task: IscoutTask, level, domain, reason=None
):
"""
email只返回email
:param root:
:param task:
:param level:
:param domain:
:param reason:
:return:
"""
try:
for data in SonarApi.domain_whois(task, level, domain, reason):
# email插件只要email
if isinstance(data, Email):
# self.__set_value(root, data)
root.set_email(data)
root = self.__segment_output(root, level, domain)
yield data
task.success_count()
except Exception:
task.fail_count()
self._logger.error(
f"Get email from sonar whois error, err:{traceback.format_exc()}"
)
def _bing_search_side_site(
self, root: Domain, task: IscoutTask, level, domain, reason
):
"""
bing 搜索一个domain的旁站
:param root:
:param task:
:param level:
:param domain:
:param reason:
:return:
"""
try:
ss = BingSideDetect()
for ssdata in ss.bing_domain_side_site(domain, level, reason):
if isinstance(ssdata, SideSite):
root.set_side_site(ssdata)
root = self.__segment_output(root, level, domain)
yield ssdata
task.success_count()
except:
task.fail_count()
self._logger.error(
f"Bing Search side site error, err:{traceback.format_exc()}"
)
def _get_mailserver(
self, root: Domain, task: IscoutTask, level, obj: ScoutFeedBackBase, reason=None
) -> iter:
"""
使用域名去查询邮箱域名的mx记录
:param root:
:param task:
:param level:
:param obj:
:return:
"""
if not task.cmd.stratagyscout.cmddomain.enabled_mail_server:
return
self._logger.debug("DOMIAN:Start getting mail server.")
domain = obj.value
log = f"开始解析目标{domain} {self.dtools.mail_server}信息"
self._outprglog(log)
count = 0
try:
mxq = DomainMXQuery(task)
for ms in mxq.get_mail_server(level, domain, reason):
self.__set_value(root, ms)
root = self.__segment_output(root, level, domain)
count += 1
yield ms
task.success_count()
except:
task.fail_count()
self._logger.error(
f"Get domain mx maiserver error, err:{traceback.format_exc()}"
)
finally:
log = f"获取到目标{domain}未经处理的{count}条{self.dtools.mail_server}数据"
self._outprglog(log)
def _get_whois_sonar(
self, root: Domain, task: IscoutTask, level, obj: ScoutFeedBackBase, reason
) -> iter:
domain: str = tld.get_tld(
obj.value, fail_silently=True, as_object=True, fix_protocol=True
).fld
try:
for data in SonarApi.domain_whoishistory(task, level, domain, reason):
if isinstance(data, Whois):
# self.__set_value(root, data)
self._logger.info(
"Got a whois data: {}".format(
data._registrar + data._registtime + data.infotime
)
) # 取到一条whois信息
root.set_whois(data)
root = self.__segment_output(root, level, domain)
yield data
task.success_count()
except Exception:
task.fail_count()
self._logger.error(
"Get sonar whoishistory data error:\ntaskid:{}\nbatchid:{}\nrootobject:{}\nobjtype:{}".format(
task.taskid, task.batchid, obj.value, obj._objtype.name
)
)
def _mx_domain_detect(self,
root: IP,
task: IscoutTask,
level,
ip,
reason=None):
"""
nslookup查询ip的域名
:param root:
:param task:
:param level:
:param ip:
:param reason:
:return:
"""
try:
mdd = IpMxDomain(task)
reverse_domain = mdd.get_ip_reverse_domain(level, ip, reason)
if reverse_domain is not None:
# self.__set_value(root, reverse_domain)
root.set_ipreverse(reverse_domain)
root = self.__segment_output(root, level, ip)
yield reverse_domain
task.success_count()
except:
task.fail_count()
self._logger.error(
f"Get mx ip reverse error, err:{traceback.format_exc()}")
def _sonarapi_get_email(self, root: Phone, task: IscoutTask, level, phone,
reason):
"""
sonar api 先去查whoisr，然后使用查到的domain，再去domain whois那边拿phone
:param root:
:param task:
:param level:
:param phone:
:param reason:
:return:
"""
try:
for ew in SonarApi.phone_whoisr(task, level, phone):
domain = ew._domain
self._logger.debug(f"Sonar search a domain:{domain}.")
for data in SonarApi.domain_whois(task, level, domain, reason):
if isinstance(data, Email):
root.set_email(data)
root = self.__segment_output(root, level, domain)
yield data
task.success_count()
except:
task.fail_count()
self._logger.error(
f"Get email from sonar api error, err:{traceback.format_exc()}"
)
def _get_rangec_detect(self,
root: IP,
task: IscoutTask,
level,
obj: ScoutFeedBackBase,
reason=None):
"""
获取ip所在的C段主机的存活情况
:param root:
:param task:
:param level:
:param obj:
:return:
"""
if not task.cmd.stratagyscout.cmdip.enabled_rangec_detect:
return
count = 0
try:
log = f"开始探测目标{obj.value} {self.dtools.rangec_detect}信息"
self._outprglog(log)
self._logger.debug("IP:Start getting rangec detect.")
# ports = task.cmd.stratagyscout.default_ports
# if isinstance(
# self.task.cmd.stratagyscout.cmdip.ports,
# list) and len(self.task.cmd.stratagyscout.cmdip.ports) > 0:
# ports = self.task.cmd.stratagyscout.cmdip.ports
ip: IPy.IP = IPy.IP(obj.value)
ipcrange = ip.make_net("255.255.255.0")
nmap = Nmap()
for host in nmap.scan_alive_hosts(
self.task,
level,
[ipcrange.__str__()],
):
if not isinstance(host, RangeCHost):
continue
root.set_rangec(host)
count += 1
root = self.__segment_output(root, level, obj.value)
yield host
task.success_count("rangec_detect", level=level)
except Exception:
task.fail_count("rangec_detect", level=level)
self._logger.error(
"Range C detect error:\ntaskid:{}\nbatchid:{}\nobj{}\nerror:{}"
.format(
task.taskid,
task.batchid,
obj.value,
traceback.format_exc(),
))
finally:
log = f"获取到目标{obj.value}未经处理的{count}条{self.dtools.rangec_detect}数据"
self._outprglog(log)
def get_iplog(task: IscoutTask, level, domain):
"""
去获取iplog
:param domain_name:
:return:
"""
# pagelimit = 3000
# 限制下，目前只去拿500条
pagelimit = 500
url = f'{scouter_config.sonarapi}/dbs/dns'
querystring = {"domainName": domain, "pageSize": pagelimit}
headers = {'Accept': 'application/json'}
errortimes = 0
while True:
try:
response = requests.request("GET",
url,
headers=headers,
params=querystring,
timeout=10)
res_text = response.text
res_dict = json.loads(res_text)
data = res_dict.get('data', [])
data_num = len(data)
if data_num == 0:
return
for el in data:
unix_logtime = el.get('timestamp')
ip_list = el.get('value')
date_logtime = datetime.datetime.fromtimestamp(
int(unix_logtime)).strftime('%Y-%m-%d %H:%M:%S')
for ip in ip_list:
ipobj = IP(task, level, ip)
ipobj.logtime = date_logtime
yield ipobj
# 目前因为数据量很大，所以限制下只返回500条
task.success_count()
break
# 这里判断下要不要继续翻页拿数据
# if data_num < pagelimit:
# break
# else:
# 这里是拿最后一个data_one的timestamp,python嘛，动态语言就是这么玩的
# querystring['start'] = unix_logtime
except:
if errortimes > 3:
task.fail_count()
logger.error(
f"Sonarapi get iplog error, please check sonar api connect, error:{traceback.format_exc()}"
)
break
errortimes += 1
continue
finally:
time.sleep(1)
def _get_waf_detect(
self, root: Domain, task: IscoutTask, level, obj: ScoutFeedBackBase, reason=None
):
"""
目标防护探测数据，探测当前URL在服务器上的web防火墙
:param root:
:param task:
:param level:
:param obj:
:return:
"""
if not task.cmd.stratagyscout.cmddomain.enabled_waf_detect:
return
log = f"开始收集目标{obj.value} {self.dtools.waf_detect}信息"
self._outprglog(log)
self._logger.debug("DOMIAN: Start getting waf_detect info.")
count_dict = {}
try:
wd = WafDetect(task)
if "http://" in obj.value or "https://" in obj.value:
self._logger.error("http:// or https:// should not in obj.value!")
return
# 探测目标是HTTP协议
for waf in wd.waf_detect("http://" + obj.value):
if not isinstance(waf, str):
continue
root.set_waf(waf)
count_dict[waf] = 1
root = self.__segment_output(root, level, obj.value)
yield waf
task.success_count()
# 探测目标是HTTPS协议
for waf in wd.waf_detect("https://" + obj.value):
if not isinstance(waf, str):
continue
count_dict[waf] = 1
root.set_waf(waf)
root = self.__segment_output(root, level, obj.value)
yield waf
task.success_count()
except Exception:
task.fail_count()
self._logger.error(
"Waf detect error:\ntaskid:{}\nbatchid:{}\nobj:{}\nerror:{}".format(
self.task.taskid,
self.task.batchid,
obj.value,
traceback.format_exc(),
)
)
finally:
log = f"获取到目标{obj.value}未经处理的{count_dict.__len__()}条{self.dtools.waf_detect}数据"
self._outprglog(log)
def _get_side_site_detect(self,
root: IP,
task: IscoutTask,
level,
obj: ScoutFeedBackBase,
reason=None):
"""
探测当前IP所在的服务器上的其他网站
:param root:
:param task:
:param level:
:param obj:
:return:
"""
if not task.cmd.stratagyscout.cmdip.enabled_side_site_detect:
return
ip = obj.value
count_dict = {}
try:
self._logger.debug("IP: Start getting side_site detect info")
log = f"开始探测目标{ip} {self.dtools.side_site_detect}信息"
self._outprglog(log)
# sidesite api
sd = SideSiteDetect(task)
for side_site in sd.side_site_detect(ip, level):
if not isinstance(side_site, SideSite):
continue
root.set_side_site(side_site)
count_dict[side_site.host + side_site.ip +
str(side_site.port)] = 1
root = self.__segment_output(root, level, ip)
yield side_site
# bing api
for data in self._bing_search_side_site(root, task, level, ip,
reason):
count_dict[data.host + data.ip + str(data.port)] = 1
yield data
task.success_count("side_site_detect", level)
except Exception:
task.fail_count("side_site_detect", level)
self._logger.error(
"Sidesite detect error:\ntaskid:{}\nbatchid:{}\nobj:{}\nerror:{}"
.format(
task.taskid,
task.batchid,
obj.value,
traceback.format_exc(),
))
finally:
log = f"获取到到目标{ip}未经处理的{count_dict.__len__()}条{self.dtools.side_site_detect}信息"
self._outprglog(log)
def get_subdomain(task: IscoutTask, level, domian_name: str):
"""
获取子域名
:param domian_name:
:return:
"""
pagelimit = 500
errortimes = 0
url = f'{scouter_config.sonarapi}/dbs/domains/subdomains'
headers = {'Accept': 'application/json'}
querystring = {"domainName": domian_name, "pageSize": pagelimit}
while True:
try:
response = requests.request("GET",
url,
headers=headers,
params=querystring,
timeout=10)
res_text = response.text
res_dict = json.loads(res_text)
data = res_dict.get('data')
data_num = len(data)
if data_num == 0:
return
for domain_one in data:
dobj = Domain(task, level, domain_one)
yield dobj
# 限制500
# 走到这里算成功
task.success_count()
break
# 这里判断下要不要继续翻页拿数据
# if data_num < pagelimit:
# break
# else:
# 这里是拿最后一个data_one的domain,python嘛，动态语言就是这么玩的
# querystring['start'] = domain_one
except:
if errortimes > 3:
# 连续出错3次表示失败
task.fail_count()
logger.error(
f"Sonarapi get subdomain error, please check sonar api connect, err:{traceback.format_exc()}"
)
break
errortimes += 1
continue
finally:
time.sleep(1)
def get_ipwhois_history(self, task: IscoutTask, ip: str, reason,
level: int) -> iter:
"""get ipwhois"""
try:
try:
IPy.IP(ip)
except Exception:
task.fail_count('ipwhois', level=level)
self._logger.error(
"Invalid ip for getting ipwhois error: ip:{}, error: {}".
format(ip, traceback.format_exc()))
return
# 查历史记录时，应做路由并查询
# 目前暂不支持路由，
# 且只有apnic做了查历史的，
# 所以查一下apnic的历史，并查一下arin的最新记录
# apnic 查亚洲
for iw in self._apnic.get_ipwhois_history(ip, reason):
yield iw
iw = self._apnic.get_ipwhois(ip, reason)
yield iw
# arin 查美洲
iw = self._arin.get_ipwhois(ip, reason)
yield iw
# afrinic 非洲
iw = self._afrinic.get_ipwhois(ip, reason)
yield iw
# ripe 欧洲、中东和中亚
# ripe 很强大，东西很多，暂时做不过来，
# 上面的可以先用着，后面再加
# iw = self._ripe.get_ipwhois(ip, reason)
# yield iw
# lacnic 拉丁美洲和一些加勒比岛屿
task.success_count('ipwhois', level=level)
except Exception:
task.fail_count('ipwhois', level=level)
self._logger.error(
"Get ipwhois history error: ip:{}, error: {}".format(
ip, traceback.format_exc()))
def __get_sonar_whoisr(self, root: Email, task: IscoutTask, level, email):
"""
sonar的api
:param root:
:param task:
:param level:
:param email:
:return:
"""
try:
for ew in SonarApi.email_whoisr(task, level, email):
self.__set_value(root, ew)
root = self.__segment_output(root, level, email)
yield ew
task.success_count()
except:
task.fail_count()
self._logger.error(
f"Get Sonar email whoisr error, err:{traceback.format_exc()}")
def _get_sonar_geoinfo(self, root: IP, task: IscoutTask, level,
obj: ScoutFeedBackBase):
"""
sonar获取geoinfo
:return:
"""
ip = obj.value
try:
for gobj in SonarApi.geoinfo(task, level, ip):
self.__set_value(root, gobj)
root = self.__segment_output(root, level, ip)
yield gobj
task.success_count()
except:
task.fail_count()
self._logger.error(
"Get sonar geoinfo error:\ntaskid:{}\nbatchid:{}\nrootobject:{}\nobjtype:{}"
.format(task.taskid, task.batchid, obj.value,
obj._objtype.name))
def __get_mx_email_server(self, root: Email, task: IscoutTask, level,
email):
"""
根据mx记录查询邮服地址
:param root:
:param task:
:param level:
:param email:
:return:
"""
try:
mxq = MXQuery(task)
for ms in mxq.get_mail_server(level, email):
self.__set_value(root, ms)
root = self.__segment_output(root, level, email)
yield ms
task.success_count()
except:
task.fail_count()
self._logger.error(
f"Get mx maiserver error, err:{traceback.format_exc()}")
def _get_ipwhois(self,
root: IP,
task: IscoutTask,
level,
obj: ScoutFeedBackBase,
reason=None):
"""get ipwhois"""
if not task.cmd.stratagyscout.cmdip.enabled_ipwhois:
return
log = f"开始获取目标{obj.value}的{self.dtools.ipwhois}信息"
self._outprglog(log)
self._logger.debug("IP:Start getting ipwhois.")
plgipwhois = IPWhois()
count_dict = {}
try:
for iw in plgipwhois.get_ipwhois_history(task, obj.value, reason,
level):
if iw is not None:
self.__set_value(root, iw)
count_dict[iw._handle + iw._md5] = 1
root = self.__segment_output(root, level, obj.value)
yield iw
task.success_count("ipwhois", level=level)
except Exception:
task.fail_count("ipwhois", level=level)
self._logger.error(
"Get IPWhois error:\ntaskid:{}\nbatchid:{}\nrootobject:{}\nobjtype:{}\nerror:{}"
.format(
task.taskid,
task.batchid,
obj.value,
obj._objtype.name,
traceback.format_exc(),
))
finally:
log = f"获取到目标{obj.value}未经处理的{count_dict.__len__()}条{self.dtools.ipwhois}数据"
self._outprglog(log)
def _landing_instagram(self, root: NetworkId, task: IscoutTask, level,
obj: ScoutFeedBackBase, reason) -> iter:
"""get instagram profiles by search username"""
ins: Instagram = Instagram(task)
networkprofiles: NetworkProfiles = NetworkProfiles(self.task)
try:
if len(self.task.cmd.stratagyscout.cmdnetworkid.netidinfo) <= 0:
if ins.judgment_url:
profile = ins.judgment_url(obj.value, level, reason)
if profile is not None:
yield profile
task.success_count()
return
else:
pass
if ins.judgment_keyword:
for profile in ins.judgment_keyword(
obj.value, level, reason):
yield profile
task.success_count()
except Exception:
task.fail_count()
self._logger.error(
"Get instagram profiles by search name error:\ntaskid:{}\nbatchid:{}\nobj:{}\nerror:{}"
.format(
self.task.taskid,
self.task.batchid,
obj._objtype,
traceback.format_exc(),
))
finally:
# 最后输出
if networkprofiles is not None and len(networkprofiles) > 0:
self.outputdata(networkprofiles.get_outputdict(),
networkprofiles._suffix)
def _get_home_info(self,
root: URL,
task: IscoutTask,
level,
obj: ScoutFeedBackBase,
reason=None):
"""
这个方法用来获取首页的信息，如果不能在一个方法内获取完成，那么就再拆分
:param root:
:param task:
:param level:
:param obj:
:return:
"""
# task.cmd.stratagyscout.cmdurl.enabled_homepage_code
# task.cmd.stratagyscout.cmdurl.enabled_summary
# task.cmd.stratagyscout.cmdurl.enabled_homepage_screenshot
self._logger.debug("URL:Start getting homeinfo.")
url: str = obj.value
log = f"开始收集目标{url}的{self.dtools.urlInfo}相关信息"
self._outprglog(log)
# 一个方法可以将3个东西获取完成
count = 0
u_info = None
try:
u_info = UrlInfo(task)
for data in u_info.visit_url(url, level, reason):
if data is None:
continue
data_type = data[-1]
try:
# home page code
if (data_type == "homecode" and
task.cmd.stratagyscout.cmdurl.enabled_homepage_code
):
root.set_homepage_code(data[0])
count += 1
# title meta
elif (data_type == "summary"
and task.cmd.stratagyscout.cmdurl.enabled_summary):
root.set_homepage_summary(data[0], data[1])
count += 1
# screen shot
elif (data_type == "screenshot" and task.cmd.stratagyscout.
cmdurl.enabled_homepage_screenshot):
# 这个数据是直接输出的
OutputManagement.output(data[0])
count += 1
yield data
task.success_count()
except:
task.fail_count()
self._logger.error(
f"Set data error,datatype:{data_type}, error:{traceback.format_exc()}"
)
continue
except:
self._logger.error(
f"Get homeinfo error, err:{traceback.format_exc()}")
finally:
log = f"获取到目标{url}未经处理的{count}条{self.dtools.urlInfo}数据"
self._outprglog(log)
# 手动关闭下浏览器
if u_info is not None:
del u_info
def _landing_twitter(self, root: NetworkId, task: IscoutTask, level, email,
reason) -> iter:
"""
去调用下载器下载profile,
可能会回来其他的数据
:param task:
:param level:
:param email:
:return:
"""
tw = LandingTwitter(task)
# twitter get profile，这个profile是单独输出的
networkprofiles: NetworkProfiles = NetworkProfiles(self.task)
try:
# 这里要先看看有没有指定需要获取的land信息
got: bool = False
if len(task.cmd.stratagyscout.cmdnetworkid.netidinfo) > 0:
for (
netidinfo_tw
) in task.cmd.stratagyscout.cmdnetworkid.netidinfo.values():
netidinfo_tw: NetidInfo = netidinfo_tw
if (not isinstance(netidinfo_tw._source, str)
or netidinfo_tw._source != "twitter"):
continue
if netidinfo_tw._userid is not None:
self._logger.debug(
f"Get task cmd twitter userid, userid:{netidinfo_tw._userid}"
)
profile: NetworkProfile = tw.landing_userid(
level, netidinfo_tw._userid, reason)
# 大哥你这里面是不是也应该吧profile加到networkprofiles里面？
if isinstance(profile, NetworkProfile):
profile.reason = reason
networkprofiles.set_profile(profile)
got = True
yield profile
# -----------------------------------上面是去拿特定的，可能会拿到没有指定的，下面才是去search
if not got:
self._logger.debug(f"Get twitter object value:{email}")
idxstart: int = 0
idxstop: int = 10
if (task.cmd.stratagyscout.cmdnetworkid.searchindex.
landing_twitter is not None):
si = task.cmd.stratagyscout.cmdnetworkid.searchindex.landing_twitter
idxstart = si.start
idxstop = si.stop
for tp in tw.landing(level,
email,
reason,
idxstart=idxstart,
idxstop=idxstop):
if isinstance(tp, NetworkProfile):
tp.reason = reason
networkprofiles.set_profile(tp)
networkprofiles = self.__segment_output_profiles(
networkprofiles)
yield tp
task.success_count()
except:
task.fail_count()
self._logger.error(
f"Get profile from twitter error\nerr:{traceback.format_exc()}"
)
finally:
# 最后输出,最后剩下没有输出的，一定要输出，不管拿到多少个
if len(networkprofiles) > 0:
self.outputdata(networkprofiles.get_outputdict(),
networkprofiles._suffix)
def _scan_application_protocol(
self,
task: IscoutTask,
level: int,
obj: ScoutFeedBackBase,
port: PortInfo,
**kwargs,
):
"""根据 portinfo 的协议类型，扫描其应用层协议"""
try:
zgrab2 = Zgrab2()
outlog = kwargs.get("outlog")
# all application protocol need to support TLS banner grab
# protocols supported TLS:
# http
# mongodb
# redis
# 绝对不用扫描ssl证书的端口：
# if port._port != 23 or \
# port._port != 25 or \
# port._port != 80 or \
# port._port != 110 or \
# port._port != 143 or \
# port._port != 993:
if port._port != 80:
zgrab2.get_tlsinfo(task, level, port)
# 先简单匹配 banner抓取器 了，，后面需要单独出匹配banner 抓取器了
if port.service == "ftp" or port._port == 21:
zgrab2.get_ftp_info(task, level, port)
elif port.service == "ssh" or port._port == 22:
zgrab2.get_ssh_info(task, level, port)
elif port.service == "telnet" or port._port == 23:
zgrab2.get_telnet_info(task, level, port)
elif port.service == "smtp" or port._port == 25 or port._port == 465:
zgrab2.get_smtp_info(task, level, port)
elif port.service == "http" or port._port == 80 or port._port == 443:
zgrab2.get_siteinfo(task, level, port)
elif port.service == "pop3" or port._port == 110 or port._port == 995:
zgrab2.get_pop3_info(task, level, port)
elif port.service == "ntp" or port._port == 123:
zgrab2.get_ntp_info(task, level, port)
elif port.service == "imap" or port._port == 143 or port._port == 993:
zgrab2.get_imap_info(task, level, port)
elif port.service == "mssql" or port._port == 1433:
zgrab2.get_mssql_info(task, level, port)
elif port.service == "redis" or port._port == 6379:
zgrab2.get_redis_info(task, level, port)
elif port.service == "mongodb" or port._port == 27017:
zgrab2.get_mongodb_info(task, level, port)
elif port.service == "mysql" or port._port == 3306:
zgrab2.get_mysql_info(task, level, port)
elif port.service == "oracle" or port._port == 1521:
zgrab2.get_oracle_info(task, level, port)
log = f"目标:{port._host},端口:{port._port},协议:{port.service},协议详情探测完成"
outlog(log)
except Exception:
task.fail_count("service", level=level)
self._logger.error(
"Scout ip port application protocol error:\ntaskid:{}\nbatchid:{}\nrootobject:{}\nobjtype:{}"
.format(task.taskid, task.batchid, obj.value,
obj._objtype.name))
def _get_waf_detect(self,
root: URL,
task: IscoutTask,
level,
obj: ScoutFeedBackBase,
reason=None):
"""
目标防护探测，探测web应用防火墙指纹，识别WAF类型
:param root:
:param task:
:param level:
:param obj:
:return:
"""
if not task.cmd.stratagyscout.cmdurl.enabled_waf_detect:
return
self._logger.debug("URL: Start getting waf_detect info.")
domain: str = obj.value
log = f"开始收集目标{domain} {self.dtools.waf_detect}信息"
self._outprglog(log)
count_dict = {}
try:
if domain.startswith("http"):
url = parse.urlparse(domain)
domain = url.netloc
wd = WafDetect(task)
if "http://" in domain or "https://" in domain:
self._logger.error("http:// or https:// should not in domain!")
return
# 探测目标是HTTP协议
for waf in wd.waf_detect("http://" + domain):
if not isinstance(waf, str):
continue
root.set_waf(waf)
count_dict[waf] = 1
root = self.__segment_output(root, level, obj.value)
yield waf
task.success_count()
# 探测目标是HTTPS协议
for waf in wd.waf_detect("https://" + domain):
if not isinstance(waf, str):
continue
root.set_waf(waf)
count_dict[waf] = 1
root = self.__segment_output(root, level, obj.value)
yield waf
task.success_count()
except Exception:
task.fail_count()
self._logger.error(
"Waf detect error:\ntaskid:{}\nbatchid:{}\nobj:{}\nerror:{}".
format(
self.task.taskid,
self.task.batchid,
obj.value,
traceback.format_exc(),
))
finally:
log = f"获取到目标{domain}未经处理的{count_dict.__len__()}条{self.dtools.waf_detect}数据"
self._outprglog(log)
def _get_port_service(self,
root: IP,
task: IscoutTask,
level,
obj: ScoutFeedBackBase,
reason=None) -> iter:
"""扫描端口服务详情"""
if not task.cmd.stratagyscout.cmdip.enabled_service:
return
count = 0
try:
log = f"开始获取目标{obj.value} {self.dtools.service}信息"
self._outprglog(log)
self._logger.debug("IP:Start getting port service.")
ports = task.cmd.stratagyscout.default_ports
if (isinstance(task.cmd.stratagyscout.cmdip.ports, list)
and len(task.cmd.stratagyscout.cmdip.ports) > 0):
ports = task.cmd.stratagyscout.cmdip.ports
vuls = task.cmd.stratagyscout.cmdip.vuls
log = f"开始探测目标{obj.value}的端口，一共将会探测{len(ports)}个端口"
self._outprglog(log)
# here should use nmap,
# for service type/version and os version scan
nmap = Nmap()
# 这里来的肯定是只有 一个域名
for port in nmap.scan_open_ports(
task,
level,
[obj.value],
ports,
outlog=self._outprglog,
):
# 这里出来就是端口基本信息
# 必有字段：
# host
# port
# transport protocol
# service
if not isinstance(port, PortInfo):
continue
port: PortInfo = port
self._scan_application_protocol(task,
level,
obj,
port,
outlog=self._outprglog)
self._logicalgrabber.grabbanner(port, [],
flag="iscout",
outlog=self._outprglog)
self.__set_value(root, port)
count += 1
root = self.__segment_output(root, level, obj.value)
yield port
task.success_count("service", level=level)
except Exception:
task.fail_count("service", level=level)
self._logger.error(
"Scan IP port service error:\ntaskid:{}\nbatchid:{}\nrootobject:{}\nobjtype:{}\nerror:{}"
.format(
task.taskid,
task.batchid,
obj.value,
obj._objtype.name,
traceback.format_exc(),
))
finally:
log = f"获取到目标{obj.value}未经处理的{count}条{self.dtools.service}数据"
self._outprglog(log)
