def _mx_domain_detect(self,
root: IP,
task: IscoutTask,
level,
ip,
reason=None):
"""
nslookup查询ip的域名
:param root:
:param task:
:param level:
:param ip:
:param reason:
:return:
"""
try:
mdd = IpMxDomain(task)
reverse_domain = mdd.get_ip_reverse_domain(level, ip, reason)
if reverse_domain is not None:
# self.__set_value(root, reverse_domain)
root.set_ipreverse(reverse_domain)
root = self.__segment_output(root, level, ip)
yield reverse_domain
task.success_count()
except:
task.fail_count()
self._logger.error(
f"Get mx ip reverse error, err:{traceback.format_exc()}")
def _generate_task_back_percent(self, task: IscoutTask) -> bool:
"""返回总任务百分比"""
res: bool = False
try:
currpercent = math.floor(task.batchcompletecount /
task.batchtotalcount * 100)
lastprogress = task.progress * 100
if task.batchtotalcount > 100:
# 子任务数大于100个的，每1%返回一次
if currpercent - lastprogress < 1:
res = True
elif 50 < task.batchtotalcount <= 100:
# 50<x<=100个子任务的，总共返回25个百分比文件，每%4返回一次
if currpercent - lastprogress < 4:
res = True
else:
# 0<x<=50个子任务的，每%10左右返回一次
if currpercent - lastprogress < 10:
res = True
if res:
return res
# 最新总任务Progress更新到数据库并发送回馈数据
task.progress = currpercent / 100
task.cmdstatus = ECommandStatus.Progress
if not self._dbmanager.update_iscout_task2(task):
self._logger.error(
"Update IScoutTask with progress failed: taskid={} objecttype={} progress={}"
.format(task.taskid, task._objecttype.name, task.progress))
return res
taskback: IscoutTaskBack = IscoutTaskBack.create_from_task(
task,
ECommandStatus.Progress,
recvmsg='{}%'.format(currpercent),
batchcompletecount=task.batchtotalcount)
if not OutputManagement.output(taskback):
res = False
self._logger.error(
"Output IscoutTaskBack progress failed:\ntaskid:{}\ncmdstatus:{}\nprogress:{}"
.format(task.taskid, task.cmdstatus.name, currpercent))
return res
res = True
self._logger.info(
"IscoutTaskBack generated [Progress {}]:\ntaskid={}".format(
task.progress, task.taskid))
except Exception:
res = False
self._logger.error(
"Generate IscoutTaskBack for batch complete percent error:\ntaskid:{}\nerror:{}"
.format(task.taskid, traceback.format_exc()))
finally:
pass
return res
def _sonarapi_get_email(self, root: Phone, task: IscoutTask, level, phone,
reason):
"""
sonar api 先去查whoisr，然后使用查到的domain，再去domain whois那边拿phone
:param root:
:param task:
:param level:
:param phone:
:param reason:
:return:
"""
try:
for ew in SonarApi.phone_whoisr(task, level, phone):
domain = ew._domain
self._logger.debug(f"Sonar search a domain:{domain}.")
for data in SonarApi.domain_whois(task, level, domain, reason):
if isinstance(data, Email):
root.set_email(data)
root = self.__segment_output(root, level, domain)
yield data
task.success_count()
except:
task.fail_count()
self._logger.error(
f"Get email from sonar api error, err:{traceback.format_exc()}"
)
def __sonar_get_email(
self, root: Domain, task: IscoutTask, level, domain, reason=None
):
"""
email只返回email
:param root:
:param task:
:param level:
:param domain:
:param reason:
:return:
"""
try:
for data in SonarApi.domain_whois(task, level, domain, reason):
# email插件只要email
if isinstance(data, Email):
# self.__set_value(root, data)
root.set_email(data)
root = self.__segment_output(root, level, domain)
yield data
task.success_count()
except Exception:
task.fail_count()
self._logger.error(
f"Get email from sonar whois error, err:{traceback.format_exc()}"
)
def _get_mailserver(
self, root: Domain, task: IscoutTask, level, obj: ScoutFeedBackBase, reason=None
) -> iter:
"""
使用域名去查询邮箱域名的mx记录
:param root:
:param task:
:param level:
:param obj:
:return:
"""
if not task.cmd.stratagyscout.cmddomain.enabled_mail_server:
return
self._logger.debug("DOMIAN:Start getting mail server.")
domain = obj.value
log = f"开始解析目标{domain} {self.dtools.mail_server}信息"
self._outprglog(log)
count = 0
try:
mxq = DomainMXQuery(task)
for ms in mxq.get_mail_server(level, domain, reason):
self.__set_value(root, ms)
root = self.__segment_output(root, level, domain)
count += 1
yield ms
task.success_count()
except:
task.fail_count()
self._logger.error(
f"Get domain mx maiserver error, err:{traceback.format_exc()}"
)
finally:
log = f"获取到目标{domain}未经处理的{count}条{self.dtools.mail_server}数据"
self._outprglog(log)
def _bing_search_side_site(
self, root: Domain, task: IscoutTask, level, domain, reason
):
"""
bing 搜索一个domain的旁站
:param root:
:param task:
:param level:
:param domain:
:param reason:
:return:
"""
try:
ss = BingSideDetect()
for ssdata in ss.bing_domain_side_site(domain, level, reason):
if isinstance(ssdata, SideSite):
root.set_side_site(ssdata)
root = self.__segment_output(root, level, domain)
yield ssdata
task.success_count()
except:
task.fail_count()
self._logger.error(
f"Bing Search side site error, err:{traceback.format_exc()}"
)
def _get_whois_sonar(
self, root: Domain, task: IscoutTask, level, obj: ScoutFeedBackBase, reason
) -> iter:
domain: str = tld.get_tld(
obj.value, fail_silently=True, as_object=True, fix_protocol=True
).fld
try:
for data in SonarApi.domain_whoishistory(task, level, domain, reason):
if isinstance(data, Whois):
# self.__set_value(root, data)
self._logger.info(
"Got a whois data: {}".format(
data._registrar + data._registtime + data.infotime
)
) # 取到一条whois信息
root.set_whois(data)
root = self.__segment_output(root, level, domain)
yield data
task.success_count()
except Exception:
task.fail_count()
self._logger.error(
"Get sonar whoishistory data error:\ntaskid:{}\nbatchid:{}\nrootobject:{}\nobjtype:{}".format(
task.taskid, task.batchid, obj.value, obj._objtype.name
)
)
def _get_dbip_geoinfo(self, root: IP, task: IscoutTask, level,
obj: ScoutFeedBackBase):
"""
dbip获取geoinfo
:param task:
:param level:
:param obj:
:return:
"""
ip = obj.value
try:
db = DbipMmdb()
gobj, org, isp = db.get_ip_mmdbinfo(level, ip)
root.org = org
root.isp = isp
if gobj is not None:
task.success_count()
self.__set_value(root, gobj)
root = self.__segment_output(root, level, ip)
yield gobj
else:
task.fail_count()
# for gobj in db.get_ip_geoinfo(level, ip):
# self.__set_value(root, gobj)
# root = self.__segment_output(root, level, ip)
# yield gobj
except:
self._logger.error(
"Get dbip geoinfo error:\ntaskid:{}\nbatchid:{}\nrootobject:{}\nobjtype:{}"
.format(task.taskid, task.batchid, obj.value,
obj._objtype.name))
def _get_rangec_detect(self,
root: IP,
task: IscoutTask,
level,
obj: ScoutFeedBackBase,
reason=None):
"""
获取ip所在的C段主机的存活情况
:param root:
:param task:
:param level:
:param obj:
:return:
"""
if not task.cmd.stratagyscout.cmdip.enabled_rangec_detect:
return
count = 0
try:
log = f"开始探测目标{obj.value} {self.dtools.rangec_detect}信息"
self._outprglog(log)
self._logger.debug("IP:Start getting rangec detect.")
# ports = task.cmd.stratagyscout.default_ports
# if isinstance(
# self.task.cmd.stratagyscout.cmdip.ports,
# list) and len(self.task.cmd.stratagyscout.cmdip.ports) > 0:
# ports = self.task.cmd.stratagyscout.cmdip.ports
ip: IPy.IP = IPy.IP(obj.value)
ipcrange = ip.make_net("255.255.255.0")
nmap = Nmap()
for host in nmap.scan_alive_hosts(
self.task,
level,
[ipcrange.__str__()],
):
if not isinstance(host, RangeCHost):
continue
root.set_rangec(host)
count += 1
root = self.__segment_output(root, level, obj.value)
yield host
task.success_count("rangec_detect", level=level)
except Exception:
task.fail_count("rangec_detect", level=level)
self._logger.error(
"Range C detect error:\ntaskid:{}\nbatchid:{}\nobj{}\nerror:{}"
.format(
task.taskid,
task.batchid,
obj.value,
traceback.format_exc(),
))
finally:
log = f"获取到目标{obj.value}未经处理的{count}条{self.dtools.rangec_detect}数据"
self._outprglog(log)
def get_iplog(task: IscoutTask, level, domain):
"""
去获取iplog
:param domain_name:
:return:
"""
# pagelimit = 3000
# 限制下，目前只去拿500条
pagelimit = 500
url = f'{scouter_config.sonarapi}/dbs/dns'
querystring = {"domainName": domain, "pageSize": pagelimit}
headers = {'Accept': 'application/json'}
errortimes = 0
while True:
try:
response = requests.request("GET",
url,
headers=headers,
params=querystring,
timeout=10)
res_text = response.text
res_dict = json.loads(res_text)
data = res_dict.get('data', [])
data_num = len(data)
if data_num == 0:
return
for el in data:
unix_logtime = el.get('timestamp')
ip_list = el.get('value')
date_logtime = datetime.datetime.fromtimestamp(
int(unix_logtime)).strftime('%Y-%m-%d %H:%M:%S')
for ip in ip_list:
ipobj = IP(task, level, ip)
ipobj.logtime = date_logtime
yield ipobj
# 目前因为数据量很大，所以限制下只返回500条
task.success_count()
break
# 这里判断下要不要继续翻页拿数据
# if data_num < pagelimit:
# break
# else:
# 这里是拿最后一个data_one的timestamp,python嘛，动态语言就是这么玩的
# querystring['start'] = unix_logtime
except:
if errortimes > 3:
task.fail_count()
logger.error(
f"Sonarapi get iplog error, please check sonar api connect, error:{traceback.format_exc()}"
)
break
errortimes += 1
continue
finally:
time.sleep(1)
def _merge_task_status(self, task: IscoutTask):
"""合并处理子任务状态"""
try:
# 当前模块为 任务发送管理器，只负责 等待发送的任务及其状态管理。
# 合并子任务状态，并将总任务状态从 等待发送更新为正在执行。
# 只要有一个子任务发送成功，则总任务更新为发送成功
# 若全部子任务发送失败，整个任务才算发送失败，
# 貌似这样才能在非实时性的任务状态中没什么错
## 先看是否还有尚未发送的子任务，有的话先不要乱动。。
waitforsendcount: int = self._dbmanager.get_iscoutbtask_count_by_cmdstatus(
task._platform, task.taskid, ECommandStatus.WaitForSend)
if waitforsendcount > 0:
return
## 再看有没有发送成功的，有的话直接总任务发送成功，正在执行
sendsucccount: int = self._dbmanager.get_iscoutbtask_count_by_cmdstatus(
task._platform, task.taskid, ECommandStatus.Dealing)
if sendsucccount > 0:
task.cmdstatus = ECommandStatus.Dealing
self._logger.info("IScoutTask all sent, taskid={}".format(
task.taskid))
# 只要有一个子任务发送成功，则更新总任务为正在执行（不需要返回回馈数据）
if not self._dbmanager.update_iscouttask_status(
task._platform, task.taskid, ECommandStatus.Dealing):
self._logger.error(
"Update IScoutTask cmdstatus to {} faled: taskid:{}".
format(ECommandStatus.Dealing.name, task.taskid))
else:
task.cmdstatus = ECommandStatus.Failed
self._logger.error(
"IScoutTask all sent failed, taskid={} objtype={} obj={}".
format(task.taskid, task._objecttype, task._object))
# 若全部子任务都已发送（不管发成功还是失败），且子任务没有发送成功的，
# 则更新总任务为失败，并返回回馈数据
if not self._dbmanager.update_iscouttask_status(
task._platform, task.taskid, ECommandStatus.Failed):
self._logger.error(
"Update IScoutTask cmdstatus to {} faled: taskid:{}".
format(ECommandStatus.Failed.name, task.taskid))
# 失败，返回总任务回馈数据
tb: IscoutTaskBack = IscoutTaskBack.create_from_task(
task, ECommandStatus.Failed, "任务执行失败，发送到采集端失败")
if not OutputManagement.output(tb):
self._logger.error(
"Write IscoutTaskBack failed:\ntaskid:{}".format(
task.taskid))
except Exception as ex:
self._logger.error(
"Merge IScoutTask status error:\nplatform:{}\ntaskid:{}\nerror:{}"
.format(task._platform, task.taskid, ex.args))
self._dbmanager.update_iscouttask_status(task._platform,
task.taskid,
ECommandStatus.Failed)
def _get_side_site_detect(self,
root: IP,
task: IscoutTask,
level,
obj: ScoutFeedBackBase,
reason=None):
"""
探测当前IP所在的服务器上的其他网站
:param root:
:param task:
:param level:
:param obj:
:return:
"""
if not task.cmd.stratagyscout.cmdip.enabled_side_site_detect:
return
ip = obj.value
count_dict = {}
try:
self._logger.debug("IP: Start getting side_site detect info")
log = f"开始探测目标{ip} {self.dtools.side_site_detect}信息"
self._outprglog(log)
# sidesite api
sd = SideSiteDetect(task)
for side_site in sd.side_site_detect(ip, level):
if not isinstance(side_site, SideSite):
continue
root.set_side_site(side_site)
count_dict[side_site.host + side_site.ip +
str(side_site.port)] = 1
root = self.__segment_output(root, level, ip)
yield side_site
# bing api
for data in self._bing_search_side_site(root, task, level, ip,
reason):
count_dict[data.host + data.ip + str(data.port)] = 1
yield data
task.success_count("side_site_detect", level)
except Exception:
task.fail_count("side_site_detect", level)
self._logger.error(
"Sidesite detect error:\ntaskid:{}\nbatchid:{}\nobj:{}\nerror:{}"
.format(
task.taskid,
task.batchid,
obj.value,
traceback.format_exc(),
))
finally:
log = f"获取到到目标{ip}未经处理的{count_dict.__len__()}条{self.dtools.side_site_detect}信息"
self._outprglog(log)
def get_subdomain(task: IscoutTask, level, domian_name: str):
"""
获取子域名
:param domian_name:
:return:
"""
pagelimit = 500
errortimes = 0
url = f'{scouter_config.sonarapi}/dbs/domains/subdomains'
headers = {'Accept': 'application/json'}
querystring = {"domainName": domian_name, "pageSize": pagelimit}
while True:
try:
response = requests.request("GET",
url,
headers=headers,
params=querystring,
timeout=10)
res_text = response.text
res_dict = json.loads(res_text)
data = res_dict.get('data')
data_num = len(data)
if data_num == 0:
return
for domain_one in data:
dobj = Domain(task, level, domain_one)
yield dobj
# 限制500
# 走到这里算成功
task.success_count()
break
# 这里判断下要不要继续翻页拿数据
# if data_num < pagelimit:
# break
# else:
# 这里是拿最后一个data_one的domain,python嘛，动态语言就是这么玩的
# querystring['start'] = domain_one
except:
if errortimes > 3:
# 连续出错3次表示失败
task.fail_count()
logger.error(
f"Sonarapi get subdomain error, please check sonar api connect, err:{traceback.format_exc()}"
)
break
errortimes += 1
continue
finally:
time.sleep(1)
def __get_sonar_whoisr(self, root: Email, task: IscoutTask, level, email):
"""
sonar的api
:param root:
:param task:
:param level:
:param email:
:return:
"""
try:
for ew in SonarApi.email_whoisr(task, level, email):
self.__set_value(root, ew)
root = self.__segment_output(root, level, email)
yield ew
task.success_count()
except:
task.fail_count()
self._logger.error(
f"Get Sonar email whoisr error, err:{traceback.format_exc()}")
def _get_sonar_geoinfo(self, root: IP, task: IscoutTask, level,
obj: ScoutFeedBackBase):
"""
sonar获取geoinfo
:return:
"""
ip = obj.value
try:
for gobj in SonarApi.geoinfo(task, level, ip):
self.__set_value(root, gobj)
root = self.__segment_output(root, level, ip)
yield gobj
task.success_count()
except:
task.fail_count()
self._logger.error(
"Get sonar geoinfo error:\ntaskid:{}\nbatchid:{}\nrootobject:{}\nobjtype:{}"
.format(task.taskid, task.batchid, obj.value,
obj._objtype.name))
def _calc_task_elapsed(self, task: IscoutTask):
"""calc task elapsed"""
try:
elapsed: float = self._dbmanager.get_iscouttask_elapsed(
task._platform, task.taskid)
task.elapsed = task.elapsed + elapsed
except Exception:
self._logger.error(
"Calc IscoutTask elapsed error:\ntaskid:{}\nerror:{}".format(
task.taskid, traceback.format_exc()))
def _choose_client(self,
task: IscoutTask) -> Tuple[bool, bool, Client, str]:
"""选择最优采集端，返回配置的采集端ip。
返回<是否选择采集端成功，是否更新到数据库，选择的采集端>"""
succ: bool = True
needdispatch: bool = True
res: Client = None
msg: str = None
try:
currentclients: dict = StatusMantainer.get_clientstatus_id_sorted()
# 检查是否存在已有的子任务，如果是完全重复的taskid+batchid（直接手动重新搞的），则找到已经被分配过的采集端
existtask: IscoutTask = self._dbmanager.get_iscout_batch_task(
task._platform, task.taskid, task.batchid)
if not isinstance(existtask, IscoutTask) or not isinstance(
existtask._clientid, str
) or existtask._clientid == "" or not currentclients.__contains__(
existtask._clientid):
# 如果目标采集端未上线，或查询到任务未分配过采集端，则新分配一个，并需要更新到数据库。
# 这种情况会导致同一个账号分布在不同的采集端，将导致退出登陆等任务无法兼顾到所有
# 采集端。所以如果在任务分配到新的采集端后，旧的采集端又上限了，那么在更新到数据库
# 之前，需要搞一个零时任务，后台有一个线程循环执行，
# 当收到任何来自此账号所属的原采集端的关于此账号的任何信息时，触发零时任务，并用此零时
# 任务告知原有采集端取消此账号的自动下载，并将此账号下线（删除即可，不注销）（触发性重分配告知）。
succ, res, msg = self._strategymngr.dispatch(
task, currentclients)
else:
# 这里的周期可能会有问题by judy 2020/09/1
# 走到这里，目标采集端必然存在并在线
# 如果已存在的任务状态为正在执行等状态，则直接返回一个回馈文件说正在执行
# 否则重新下发到目标采集端
res = currentclients[existtask._clientid]
if existtask.cmdstatus == ECommandStatus.Failed or \
existtask.cmdstatus == ECommandStatus.Succeed or \
existtask.cmdstatus == ECommandStatus.Cancelled or \
existtask.cmdstatus == ECommandStatus.Timeout:
# 重新下发
if task._is_period:
task.periodnum = existtask.periodnum
else:
# 返回回馈数据，说正在执行
needdispatch = False
# 这种情况暂时不返回任何回馈数据
except Exception:
self._logger.error("Choose IScoutTask client error: %s" %
traceback.format_exc())
res = None
succ = False
msg = '内部错误500'
return (succ, needdispatch, res, msg)
def __get_mx_email_server(self, root: Email, task: IscoutTask, level,
email):
"""
根据mx记录查询邮服地址
:param root:
:param task:
:param level:
:param email:
:return:
"""
try:
mxq = MXQuery(task)
for ms in mxq.get_mail_server(level, email):
self.__set_value(root, ms)
root = self.__segment_output(root, level, email)
yield ms
task.success_count()
except:
task.fail_count()
self._logger.error(
f"Get mx maiserver error, err:{traceback.format_exc()}")
def _get_ipwhois(self,
root: IP,
task: IscoutTask,
level,
obj: ScoutFeedBackBase,
reason=None):
"""get ipwhois"""
if not task.cmd.stratagyscout.cmdip.enabled_ipwhois:
return
log = f"开始获取目标{obj.value}的{self.dtools.ipwhois}信息"
self._outprglog(log)
self._logger.debug("IP:Start getting ipwhois.")
plgipwhois = IPWhois()
count_dict = {}
try:
for iw in plgipwhois.get_ipwhois_history(task, obj.value, reason,
level):
if iw is not None:
self.__set_value(root, iw)
count_dict[iw._handle + iw._md5] = 1
root = self.__segment_output(root, level, obj.value)
yield iw
task.success_count("ipwhois", level=level)
except Exception:
task.fail_count("ipwhois", level=level)
self._logger.error(
"Get IPWhois error:\ntaskid:{}\nbatchid:{}\nrootobject:{}\nobjtype:{}\nerror:{}"
.format(
task.taskid,
task.batchid,
obj.value,
obj._objtype.name,
traceback.format_exc(),
))
finally:
log = f"获取到目标{obj.value}未经处理的{count_dict.__len__()}条{self.dtools.ipwhois}数据"
self._outprglog(log)
def _construct_task(self, filedata: dict) -> IscoutTask:
"""
单独构造iscout task的函数，单独使用
:param filedata:
:return:
"""
# 加载设置,包括补齐默认设置,已经证明了这个代码完全没用
# 不一定，现在先这样用着吧
# if tsk.cmd_id is None:
# tsk.cmd: IdownCmd = self.d_cmd
# else:
# tsk.cmd.fill_defcmd(self.d_cmd)
tsk: IscoutTask = IscoutTask(filedata)
tsk.priority = tsk.cmd.stratagy.priority
tsk.on_complete = self.on_task_complete
return tsk
def _get_waf_detect(
self, root: Domain, task: IscoutTask, level, obj: ScoutFeedBackBase, reason=None
):
"""
目标防护探测数据，探测当前URL在服务器上的web防火墙
:param root:
:param task:
:param level:
:param obj:
:return:
"""
if not task.cmd.stratagyscout.cmddomain.enabled_waf_detect:
return
log = f"开始收集目标{obj.value} {self.dtools.waf_detect}信息"
self._outprglog(log)
self._logger.debug("DOMIAN: Start getting waf_detect info.")
count_dict = {}
try:
wd = WafDetect(task)
if "http://" in obj.value or "https://" in obj.value:
self._logger.error("http:// or https:// should not in obj.value!")
return
# 探测目标是HTTP协议
for waf in wd.waf_detect("http://" + obj.value):
if not isinstance(waf, str):
continue
root.set_waf(waf)
count_dict[waf] = 1
root = self.__segment_output(root, level, obj.value)
yield waf
task.success_count()
# 探测目标是HTTPS协议
for waf in wd.waf_detect("https://" + obj.value):
if not isinstance(waf, str):
continue
count_dict[waf] = 1
root.set_waf(waf)
root = self.__segment_output(root, level, obj.value)
yield waf
task.success_count()
except Exception:
task.fail_count()
self._logger.error(
"Waf detect error:\ntaskid:{}\nbatchid:{}\nobj:{}\nerror:{}".format(
self.task.taskid,
self.task.batchid,
obj.value,
traceback.format_exc(),
)
)
finally:
log = f"获取到目标{obj.value}未经处理的{count_dict.__len__()}条{self.dtools.waf_detect}数据"
self._outprglog(log)
def _convert(self, data: InputData) -> iter:
"""将中心下发的任务转换为自有的通用任务结构Task体枚举（一个文件可能有多个任务段）"""
try:
if data.stream is None or not data.stream.readable():
self._logger.error(
"Data stream is None when trying to convert to standard Task: %s"
% data._source)
return
for dicseg in self._get_segments(data):
if dicseg is None or len(dicseg._fields) < 1:
continue
try:
# 创建任务对象
task: IscoutTask = IscoutTask.create_from_dict(
dicseg._fields, data._platform)
task.segindex = dicseg.segindex
task.segline = dicseg.segline
if task is None or not isinstance(task, IscoutTask):
self._logger.error("Parse IscoutTask failed.")
continue
yield task
except Exception:
self._logger.error(
"Generate IscoutTask from dic fields error:\ndata:%s\nex:%s"
% (data._source, traceback.format_exc()))
if not data is None:
data.on_complete(False)
except Exception:
self._logger.error(
"Convert data to IscoutTask error:\ndata:%s\nex:%s" %
(data._source, traceback.format_exc()))
if not data is None:
data.on_complete(False)
def get_ipwhois_history(self, task: IscoutTask, ip: str, reason,
level: int) -> iter:
"""get ipwhois"""
try:
try:
IPy.IP(ip)
except Exception:
task.fail_count('ipwhois', level=level)
self._logger.error(
"Invalid ip for getting ipwhois error: ip:{}, error: {}".
format(ip, traceback.format_exc()))
return
# 查历史记录时，应做路由并查询
# 目前暂不支持路由，
# 且只有apnic做了查历史的，
# 所以查一下apnic的历史，并查一下arin的最新记录
# apnic 查亚洲
for iw in self._apnic.get_ipwhois_history(ip, reason):
yield iw
iw = self._apnic.get_ipwhois(ip, reason)
yield iw
# arin 查美洲
iw = self._arin.get_ipwhois(ip, reason)
yield iw
# afrinic 非洲
iw = self._afrinic.get_ipwhois(ip, reason)
yield iw
# ripe 欧洲、中东和中亚
# ripe 很强大，东西很多，暂时做不过来，
# 上面的可以先用着，后面再加
# iw = self._ripe.get_ipwhois(ip, reason)
# yield iw
# lacnic 拉丁美洲和一些加勒比岛屿
task.success_count('ipwhois', level=level)
except Exception:
task.fail_count('ipwhois', level=level)
self._logger.error(
"Get ipwhois history error: ip:{}, error: {}".format(
ip, traceback.format_exc()))
def _taskparse(self, data: dict, file_suffix: str):
"""
用于判断不同的任务类型
目前有：idowntask
idowncmd
后面应该根据文件的后缀来判断文件类型
所以后面这个方法要改
add by judy 2019/06/11
:param data:
:return:
"""
if file_suffix is None:
raise Exception(
"To distinguish file types, file_suffix can not be None.")
# 单独处理an_dns的数据 add by judy 2020/03/04
# if file_suffix == 'an_dns_client':
# return DnsData(data)
# 初始化数据要增加clientid,这样无论是task，和idowncmd就会有clientid了
data['clientid'] = basic_client_config.clientid
if file_suffix == 'idown_task':
return Task(data)
elif file_suffix == 'idown_cmd':
return IdownCmd.parse_from_dict(data)
elif file_suffix == 'iscan_task':
return IscanTask(data)
elif file_suffix == 'iscout_task':
return IscoutTask(data)
elif file_suffix == 'automated_task':
return AutomatedTask.create_from_dict(data)
# -------------------------------------这些东西目前改了，用后缀来判断任务类型，这样更准确些
# if data.get('taskid') is not None:
# taskid 不为空目前一定是task
# return Task(data)
# elif data.get('taskid') is None and data.get('cmdid') is not None:
# 没有taskid但是有cmdid
# return IdownCmd.parse_from_dict(data)
else:
# 希望别走到这，走到这就说明这个任务解析错了
self._logger.error('Unkown task type')
def _landing_instagram(self, root: NetworkId, task: IscoutTask, level,
obj: ScoutFeedBackBase, reason) -> iter:
"""get instagram profiles by search username"""
ins: Instagram = Instagram(task)
networkprofiles: NetworkProfiles = NetworkProfiles(self.task)
try:
if len(self.task.cmd.stratagyscout.cmdnetworkid.netidinfo) <= 0:
if ins.judgment_url:
profile = ins.judgment_url(obj.value, level, reason)
if profile is not None:
yield profile
task.success_count()
return
else:
pass
if ins.judgment_keyword:
for profile in ins.judgment_keyword(
obj.value, level, reason):
yield profile
task.success_count()
except Exception:
task.fail_count()
self._logger.error(
"Get instagram profiles by search name error:\ntaskid:{}\nbatchid:{}\nobj:{}\nerror:{}"
.format(
self.task.taskid,
self.task.batchid,
obj._objtype,
traceback.format_exc(),
))
finally:
# 最后输出
if networkprofiles is not None and len(networkprofiles) > 0:
self.outputdata(networkprofiles.get_outputdict(),
networkprofiles._suffix)
def _get_waf_detect(self,
root: URL,
task: IscoutTask,
level,
obj: ScoutFeedBackBase,
reason=None):
"""
目标防护探测，探测web应用防火墙指纹，识别WAF类型
:param root:
:param task:
:param level:
:param obj:
:return:
"""
if not task.cmd.stratagyscout.cmdurl.enabled_waf_detect:
return
self._logger.debug("URL: Start getting waf_detect info.")
domain: str = obj.value
log = f"开始收集目标{domain} {self.dtools.waf_detect}信息"
self._outprglog(log)
count_dict = {}
try:
if domain.startswith("http"):
url = parse.urlparse(domain)
domain = url.netloc
wd = WafDetect(task)
if "http://" in domain or "https://" in domain:
self._logger.error("http:// or https:// should not in domain!")
return
# 探测目标是HTTP协议
for waf in wd.waf_detect("http://" + domain):
if not isinstance(waf, str):
continue
root.set_waf(waf)
count_dict[waf] = 1
root = self.__segment_output(root, level, obj.value)
yield waf
task.success_count()
# 探测目标是HTTPS协议
for waf in wd.waf_detect("https://" + domain):
if not isinstance(waf, str):
continue
root.set_waf(waf)
count_dict[waf] = 1
root = self.__segment_output(root, level, obj.value)
yield waf
task.success_count()
except Exception:
task.fail_count()
self._logger.error(
"Waf detect error:\ntaskid:{}\nbatchid:{}\nobj:{}\nerror:{}".
format(
self.task.taskid,
self.task.batchid,
obj.value,
traceback.format_exc(),
))
finally:
log = f"获取到目标{domain}未经处理的{count_dict.__len__()}条{self.dtools.waf_detect}数据"
self._outprglog(log)
def _get_home_info(self,
root: URL,
task: IscoutTask,
level,
obj: ScoutFeedBackBase,
reason=None):
"""
这个方法用来获取首页的信息，如果不能在一个方法内获取完成，那么就再拆分
:param root:
:param task:
:param level:
:param obj:
:return:
"""
# task.cmd.stratagyscout.cmdurl.enabled_homepage_code
# task.cmd.stratagyscout.cmdurl.enabled_summary
# task.cmd.stratagyscout.cmdurl.enabled_homepage_screenshot
self._logger.debug("URL:Start getting homeinfo.")
url: str = obj.value
log = f"开始收集目标{url}的{self.dtools.urlInfo}相关信息"
self._outprglog(log)
# 一个方法可以将3个东西获取完成
count = 0
u_info = None
try:
u_info = UrlInfo(task)
for data in u_info.visit_url(url, level, reason):
if data is None:
continue
data_type = data[-1]
try:
# home page code
if (data_type == "homecode" and
task.cmd.stratagyscout.cmdurl.enabled_homepage_code
):
root.set_homepage_code(data[0])
count += 1
# title meta
elif (data_type == "summary"
and task.cmd.stratagyscout.cmdurl.enabled_summary):
root.set_homepage_summary(data[0], data[1])
count += 1
# screen shot
elif (data_type == "screenshot" and task.cmd.stratagyscout.
cmdurl.enabled_homepage_screenshot):
# 这个数据是直接输出的
OutputManagement.output(data[0])
count += 1
yield data
task.success_count()
except:
task.fail_count()
self._logger.error(
f"Set data error,datatype:{data_type}, error:{traceback.format_exc()}"
)
continue
except:
self._logger.error(
f"Get homeinfo error, err:{traceback.format_exc()}")
finally:
log = f"获取到目标{url}未经处理的{count}条{self.dtools.urlInfo}数据"
self._outprglog(log)
# 手动关闭下浏览器
if u_info is not None:
del u_info
def _landing_twitter(self, root: NetworkId, task: IscoutTask, level, email,
reason) -> iter:
"""
去调用下载器下载profile,
可能会回来其他的数据
:param task:
:param level:
:param email:
:return:
"""
tw = LandingTwitter(task)
# twitter get profile，这个profile是单独输出的
networkprofiles: NetworkProfiles = NetworkProfiles(self.task)
try:
# 这里要先看看有没有指定需要获取的land信息
got: bool = False
if len(task.cmd.stratagyscout.cmdnetworkid.netidinfo) > 0:
for (
netidinfo_tw
) in task.cmd.stratagyscout.cmdnetworkid.netidinfo.values():
netidinfo_tw: NetidInfo = netidinfo_tw
if (not isinstance(netidinfo_tw._source, str)
or netidinfo_tw._source != "twitter"):
continue
if netidinfo_tw._userid is not None:
self._logger.debug(
f"Get task cmd twitter userid, userid:{netidinfo_tw._userid}"
)
profile: NetworkProfile = tw.landing_userid(
level, netidinfo_tw._userid, reason)
# 大哥你这里面是不是也应该吧profile加到networkprofiles里面？
if isinstance(profile, NetworkProfile):
profile.reason = reason
networkprofiles.set_profile(profile)
got = True
yield profile
# -----------------------------------上面是去拿特定的，可能会拿到没有指定的，下面才是去search
if not got:
self._logger.debug(f"Get twitter object value:{email}")
idxstart: int = 0
idxstop: int = 10
if (task.cmd.stratagyscout.cmdnetworkid.searchindex.
landing_twitter is not None):
si = task.cmd.stratagyscout.cmdnetworkid.searchindex.landing_twitter
idxstart = si.start
idxstop = si.stop
for tp in tw.landing(level,
email,
reason,
idxstart=idxstart,
idxstop=idxstop):
if isinstance(tp, NetworkProfile):
tp.reason = reason
networkprofiles.set_profile(tp)
networkprofiles = self.__segment_output_profiles(
networkprofiles)
yield tp
task.success_count()
except:
task.fail_count()
self._logger.error(
f"Get profile from twitter error\nerr:{traceback.format_exc()}"
)
finally:
# 最后输出,最后剩下没有输出的，一定要输出，不管拿到多少个
if len(networkprofiles) > 0:
self.outputdata(networkprofiles.get_outputdict(),
networkprofiles._suffix)
def _get_port_service(self,
root: IP,
task: IscoutTask,
level,
obj: ScoutFeedBackBase,
reason=None) -> iter:
"""扫描端口服务详情"""
if not task.cmd.stratagyscout.cmdip.enabled_service:
return
count = 0
try:
log = f"开始获取目标{obj.value} {self.dtools.service}信息"
self._outprglog(log)
self._logger.debug("IP:Start getting port service.")
ports = task.cmd.stratagyscout.default_ports
if (isinstance(task.cmd.stratagyscout.cmdip.ports, list)
and len(task.cmd.stratagyscout.cmdip.ports) > 0):
ports = task.cmd.stratagyscout.cmdip.ports
vuls = task.cmd.stratagyscout.cmdip.vuls
log = f"开始探测目标{obj.value}的端口，一共将会探测{len(ports)}个端口"
self._outprglog(log)
# here should use nmap,
# for service type/version and os version scan
nmap = Nmap()
# 这里来的肯定是只有 一个域名
for port in nmap.scan_open_ports(
task,
level,
[obj.value],
ports,
outlog=self._outprglog,
):
# 这里出来就是端口基本信息
# 必有字段：
# host
# port
# transport protocol
# service
if not isinstance(port, PortInfo):
continue
port: PortInfo = port
self._scan_application_protocol(task,
level,
obj,
port,
outlog=self._outprglog)
self._logicalgrabber.grabbanner(port, [],
flag="iscout",
outlog=self._outprglog)
self.__set_value(root, port)
count += 1
root = self.__segment_output(root, level, obj.value)
yield port
task.success_count("service", level=level)
except Exception:
task.fail_count("service", level=level)
self._logger.error(
"Scan IP port service error:\ntaskid:{}\nbatchid:{}\nrootobject:{}\nobjtype:{}\nerror:{}"
.format(
task.taskid,
task.batchid,
obj.value,
obj._objtype.name,
traceback.format_exc(),
))
finally:
log = f"获取到目标{obj.value}未经处理的{count}条{self.dtools.service}数据"
self._outprglog(log)
def _scan_application_protocol(
self,
task: IscoutTask,
level: int,
obj: ScoutFeedBackBase,
port: PortInfo,
**kwargs,
):
"""根据 portinfo 的协议类型，扫描其应用层协议"""
try:
zgrab2 = Zgrab2()
outlog = kwargs.get("outlog")
# all application protocol need to support TLS banner grab
# protocols supported TLS:
# http
# mongodb
# redis
# 绝对不用扫描ssl证书的端口：
# if port._port != 23 or \
# port._port != 25 or \
# port._port != 80 or \
# port._port != 110 or \
# port._port != 143 or \
# port._port != 993:
if port._port != 80:
zgrab2.get_tlsinfo(task, level, port)
# 先简单匹配 banner抓取器 了，，后面需要单独出匹配banner 抓取器了
if port.service == "ftp" or port._port == 21:
zgrab2.get_ftp_info(task, level, port)
elif port.service == "ssh" or port._port == 22:
zgrab2.get_ssh_info(task, level, port)
elif port.service == "telnet" or port._port == 23:
zgrab2.get_telnet_info(task, level, port)
elif port.service == "smtp" or port._port == 25 or port._port == 465:
zgrab2.get_smtp_info(task, level, port)
elif port.service == "http" or port._port == 80 or port._port == 443:
zgrab2.get_siteinfo(task, level, port)
elif port.service == "pop3" or port._port == 110 or port._port == 995:
zgrab2.get_pop3_info(task, level, port)
elif port.service == "ntp" or port._port == 123:
zgrab2.get_ntp_info(task, level, port)
elif port.service == "imap" or port._port == 143 or port._port == 993:
zgrab2.get_imap_info(task, level, port)
elif port.service == "mssql" or port._port == 1433:
zgrab2.get_mssql_info(task, level, port)
elif port.service == "redis" or port._port == 6379:
zgrab2.get_redis_info(task, level, port)
elif port.service == "mongodb" or port._port == 27017:
zgrab2.get_mongodb_info(task, level, port)
elif port.service == "mysql" or port._port == 3306:
zgrab2.get_mysql_info(task, level, port)
elif port.service == "oracle" or port._port == 1521:
zgrab2.get_oracle_info(task, level, port)
log = f"目标:{port._host},端口:{port._port},协议:{port.service},协议详情探测完成"
outlog(log)
except Exception:
task.fail_count("service", level=level)
self._logger.error(
"Scout ip port application protocol error:\ntaskid:{}\nbatchid:{}\nrootobject:{}\nobjtype:{}"
.format(task.taskid, task.batchid, obj.value,
obj._objtype.name))