def destory_by_wid(wid):
try:
models = WebsiteCDNModel.objects.filter(wid=wid)
models.delete()
return True
except Exception as E:
logger.error(E)
return False
def call_task(kind, kwargs, expires=3600):
"""用于建立task任务"""
logger.info("建立任务,kind: {} kwargs:{}".format(kind, kwargs))
try:
task_instance = CeleryTask._get_celery_task_instance(kind)
result = task_instance.apply_async(kwargs=kwargs, expires=expires)
return result.task_id
except Exception as E:
logger.error("未找到对应的任务实例,kind: {} Except:{}".format(kind, E))
return None
def from_db_value(self, value, expression, connection):
if not value:
value = []
if isinstance(value, dict):
return value
# 直接将字符串转换成python内置的list
try:
return ast.literal_eval(value)
except Exception as E:
logger.error(value)
return {}
def destory_by_pid(pid):
try:
models = IPaddressModel.objects.filter(pid=pid)
for model in models:
Port.destory_by_ipid(model.id)
models.delete()
return True
except Exception as E:
logger.error(E)
return False
def _sub_domain_store(pid, retval):
logger.info("subdomain 存储结果: {} {}".format(pid, retval))
domain = retval.get('domain')
subdomains = retval.get('subdomain')
try:
for website in subdomains:
Website.update_website(pid=pid, domain=domain, website=website)
except Exception as E:
logger.error("存储结果失败,异常: {}".format(E))
return False
return True
def from_db_value(value, expression, connection):
if not value:
value = []
if isinstance(value, list):
return value
# 直接将字符串转换成python内置的list
try:
return ast.literal_eval(value)
except Exception as E:
from Core.lib import logger
logger.exception(E)
logger.error(value)
return []
def destory_by_pid(pid):
try:
models = WebsiteModel.objects.filter(pid=pid)
# 删除关联表
for model in models:
id = model.id
WebsiteTech.destory_by_wid(id)
WebsiteTech.destory_by_wid(id)
WebsiteWaf.destory_by_wid(id)
WebsiteCDN.destory_by_wid(id)
models.delete()
return True
except Exception as E:
logger.error(E)
return False
def _port_scan_store(pid, retval):
logger.info("portscan 存储结果: {} {}".format(pid, retval))
format_result = {}
scanResult = retval.get('result')
domain = retval.get('domain')
try:
for one in scanResult:
if one.get('data').get('error') is None:
if one.get('data').get('versioninfo') is None:
info = {}
else:
info = one.get('data').get('versioninfo')
if format_result.get(one.get('ipaddress')) is None:
format_result[one.get('ipaddress')] = [{'port': one.get('port'),
'service': one.get('data').get('service'),
'info': info
}]
else:
format_result[one.get('ipaddress')].append({'port': one.get('port'),
'service': one.get('data').get('service'),
'info': info
})
else: # 扫描中包含错误
pass
for ipaddress in format_result:
ipaddress_update_result = IPaddress.update_ipaddress(pid=pid, ipaddress=ipaddress, domain=domain)
ipid = ipaddress_update_result.get('id')
ports = format_result.get(ipaddress)
for port in ports:
port_update_result = Port.update_port(ipid=ipid,
port=port.get('port'),
service=port.get('service'),
info=port.get('info'))
# 更新
webServiceList = ['http', 'https']
if port.get('service') in webServiceList:
# 更新web服务到website
websiteWithIPaddress = "{}:{}".format(ipaddress, port.get('port'))
Website.update_website(pid=pid, domain=domain, website=websiteWithIPaddress)
except Exception as E:
logger.error("存储结果失败,异常: {}".format(E))
return False
return True
def remove_x00(result):
for one in result:
try:
if one.get('info').get('hostname') is not None and len(
one.get('info').get('hostname')) > 0:
one['info']['hostname'] = [
one['info']['hostname'][0].replace('\x00', '')
]
if one.get('info').get('info') is not None and len(
one.get('info').get('info')) > 0:
one['info']['info'] = [
one['info']['info'][0].replace('\x00', '')
]
except Exception as E:
logger.error(E)
continue
return result
def result_callback(pid, kind, retval):
"""调用定义好的回调函数"""
try:
if kind == PORTSCAN:
flag = CeleryTask._port_scan_store(pid, retval)
return flag
elif kind == SUBDOMAIN:
flag = CeleryTask._sub_domain_store(pid, retval)
return flag
elif kind == WEBCHECK:
flag = CeleryTask._webcheck_store(pid, retval)
return flag
else:
logger.error("未找到对应的回调函数,kind: {}".format(kind))
return False
except Exception as E:
logger.error("存储结果失败,异常信息为:{}".format(E))
return False
def update_port(ipid, port, service, info):
defaultDict = {
'ipid': ipid,
'port': port,
'service': service,
'info': info,
'update_time': int(time.time()),
} # 没有该主机数据时新建
model, created = PortModel.objects.get_or_create(ipid=ipid,
port=port,
defaults=defaultDict)
if created is True:
result = PortSerializer(model, many=False).data
with transaction.atomic():
try:
model = PortModel.objects.select_for_update().get(id=model.id)
model.service = service
model.info = info
model.update_time = int(time.time())
model.save()
result = PortSerializer(model, many=False).data
except Exception as E:
logger.error(E)
result = PortSerializer(model, many=False).data
# 解析OS信息及hostname
try:
if result.get('info').get('operatingsystem') is not None and \
len(result.get('info').get('operatingsystem')) > 0:
IPaddress.update_ipaddress_os(
ipid,
result.get('info').get('operatingsystem')[0])
if result.get('info').get('hostname') is not None and \
len(result.get('info').get('hostname')) > 0:
hostname = result.get('info').get('hostname')[0].replace(
'\x00', '')
IPaddress.update_ipaddress_hostname(ipid, hostname)
except Exception as E:
logger.error(E)
return result
def load_all_modules_config():
def _sort_by_moduletype(module_config=None):
return TAG2CH.get_moduletype_order(module_config.get('MODULETYPE'))
all_modules_config = []
# viper 内置模块
viper_module_count = 0
modulenames = os.listdir(os.path.join(settings.BASE_DIR, 'MODULES'))
for modulename in modulenames:
modulename = modulename.split(".")[0]
if modulename == "__init__" or modulename == "__pycache__": # __init__.py的特殊处理
continue
class_intent = importlib.import_module(
'MODULES.{}'.format(modulename))
try:
if isinstance(class_intent.PostModule.ATTCK, str):
attck = [class_intent.PostModule.ATTCK]
elif isinstance(class_intent.PostModule.ATTCK, list):
attck = [class_intent.PostModule.ATTCK]
else:
attck = []
one_module_config = {
"BROKER": class_intent.PostModule.MODULE_BROKER, # 处理器
"NAME": class_intent.PostModule.NAME,
"DESC": class_intent.PostModule.DESC,
"WARN": class_intent.PostModule.WARN,
"AUTHOR": class_intent.PostModule.AUTHOR,
"REFERENCES": class_intent.PostModule.REFERENCES,
"MODULETYPE": class_intent.PostModule.MODULETYPE,
"OPTIONS": class_intent.PostModule.OPTIONS,
"loadpath": 'MODULES.{}'.format(modulename),
# post类配置
"REQUIRE_SESSION": class_intent.PostModule.REQUIRE_SESSION,
"PLATFORM": class_intent.PostModule.PLATFORM,
"PERMISSIONS": class_intent.PostModule.PERMISSIONS,
"ATTCK": attck,
# bot类配置
"SEARCH": class_intent.PostModule.SEARCH,
}
all_modules_config.append(one_module_config)
viper_module_count += 1
except Exception as E:
logger.error(E)
continue
logger.warning("内置模块加载完成,加载{}个模块".format(viper_module_count))
Notices.send_success(f"内置模块加载完成,加载{viper_module_count}个模块")
# 自定义模块
diy_module_count = 0
modulenames = os.listdir(
os.path.join(settings.BASE_DIR, 'Docker', "module"))
for modulename in modulenames:
modulename = modulename.split(".")[0]
if modulename == "__init__" or modulename == "__pycache__": # __init__.py的特殊处理
continue
class_intent = importlib.import_module(
'Docker.module.{}'.format(modulename))
importlib.reload(class_intent)
try:
if isinstance(class_intent.PostModule.ATTCK, str):
attck = [class_intent.PostModule.ATTCK]
elif isinstance(class_intent.PostModule.ATTCK, list):
attck = [class_intent.PostModule.ATTCK]
else:
attck = []
one_module_config = {
"BROKER": class_intent.PostModule.MODULE_BROKER, # 处理器
"NAME": class_intent.PostModule.NAME,
"DESC": class_intent.PostModule.DESC,
"WARN": class_intent.PostModule.WARN,
"AUTHOR": class_intent.PostModule.AUTHOR,
"REFERENCES": class_intent.PostModule.REFERENCES,
"MODULETYPE": class_intent.PostModule.MODULETYPE,
"OPTIONS": class_intent.PostModule.OPTIONS,
"loadpath": 'Docker.module.{}'.format(modulename),
# post类配置
"REQUIRE_SESSION": class_intent.PostModule.REQUIRE_SESSION,
"PLATFORM": class_intent.PostModule.PLATFORM,
"PERMISSIONS": class_intent.PostModule.PERMISSIONS,
"ATTCK": attck,
# bot类配置
"SEARCH": class_intent.PostModule.SEARCH,
}
all_modules_config.append(one_module_config)
diy_module_count += 1
except Exception as E:
logger.error(E)
continue
logger.warning("自定义模块加载完成,加载{}个模块".format(diy_module_count))
Notices.send_success(f"自定义模块加载完成,加载{diy_module_count}个模块")
all_modules_config.sort(key=_sort_by_moduletype)
if Xcache.update_moduleconfigs(all_modules_config):
return len(all_modules_config)
else:
return 0
