def validate(self, val):
try:
check_type = val["check_type"]
except KeyError:
return val
# disk checks
# make sure no duplicate diskchecks exist for an agent/policy
if check_type == "diskspace" and not self.instance: # only on create
checks = (Check.objects.filter(**self.context).filter(
check_type="diskspace").exclude(managed_by_policy=True))
if checks:
for check in checks:
if val["disk"] in check.disk:
raise serializers.ValidationError(
f"A disk check for Drive {val['disk']} already exists!"
)
# ping checks
if check_type == "ping":
if (not _v.ipv4(val["ip"]) and not _v.ipv6(val["ip"])
and not _v.domain(val["ip"])):
raise serializers.ValidationError(
"Please enter a valid IP address or domain name")
return val
def getAnonymizedIpOrDomain(value):
if validators.ipv4(value) or validators.ipv6(value):
return "<IPADDRESS:%s>" % hashlib.md5(value).hexdigest()
elif validators.domain(value):
return "<DOMAIN:%s>" % hashlib.md5(value).hexdigest()
else:
return "<NOTIPADDRESSORDOMAIN:%s>" % hashlib.md5(value).hexdigest()
def add(request):
msg = vmsg.msg()
all_instances = perms.instance_getall_by_group(request)
hostname_default = perms.get_hostname_prefered(request)
is_superuser = perms.get_is_superuser(request.user)
if 'server' in request.POST:
if validators.ipv6(request.POST['server'].strip()) or validators.ipv4(
request.POST['server'].strip()) or validators.domain(
request.POST['server'].strip()):
v = vapi.set_ntp(hostname_default, request.POST['server'].strip())
if v.success == False:
msg.add_error("NTP server add fail - " + v.reason)
else:
msg.add_success("NTP server added")
else:
msg.add_error(
"ntp server add fail - insert only domains or IPv4 or IPv6")
context = {
'instances': all_instances,
'hostname_default': hostname_default,
'is_superuser': is_superuser,
'username': request.user,
'msg': msg.get_all(),
}
return render(request, 'ntp/add.html', context)
def read_iocs(cb, file=sys.stdin):
iocs = defaultdict(list)
report_id = hashlib.md5()
report_id.update(str(time.time()).encode("utf-8"))
for idx, line in enumerate(sys.stdin):
line = line.rstrip("\r\n")
report_id.update(line.encode("utf-8"))
if validators.md5(line):
iocs["md5"].append(line)
elif validators.sha256(line):
eprint("line {}: sha256 provided but not yet supported by backend".format(idx + 1))
iocs["sha256"].append(line)
elif validators.ipv4(line):
iocs["ipv4"].append(line)
elif validators.ipv6(line):
iocs["ipv6"].append(line)
elif validators.domain(line):
iocs["dns"].append(line)
else:
if cb.validate_query(line):
query_ioc = {"search_query": line}
iocs["query"].append(query_ioc)
else:
eprint("line {}: invalid query".format(idx + 1))
return (report_id.hexdigest(), dict(iocs))
def getAnonymizedIpOrDomain(value):
if validators.ipv4(value) or validators.ipv6(value):
return "<IPADDRESS:%s>" % hashlib.md5(value).hexdigest()
elif validators.domain(value):
return "<DOMAIN:%s>" % hashlib.md5(value).hexdigest()
else:
return "<NOTIPADDRESSORDOMAIN:%s>" % hashlib.md5(value).hexdigest()
def __do_convert_url_pattern(self, result: Result, s: str,
is_allow: bool) -> None:
t = s
is_domain_suffix = True
if t.startswith("||"):
t = t[2:]
is_domain_suffix = True
if t.startswith("|"):
t = t[1:]
is_domain_suffix = False
if t.startswith("."):
t = t[1:]
is_domain_suffix = True
t = re.sub(r"^https?://", "", t)
t = t.lstrip("*.").rstrip("^/*")
ur = urllib.parse.urlparse("http://" + t)
if ur.fragment:
return
if ur.path and not self.ignore_path:
return
t = ur.netloc
t = re.sub(r":\d{2,5}$", "", t)
if validators.ipv4(t) or validators.ipv4_cidr(t):
result.add(result.ipv4_cidr, t, remove=is_allow)
elif validators.ipv6(t) or validators.ipv6_cidr(t):
result.add(result.ipv6_cidr, t, remove=is_allow)
else:
if validators.domain(t) and tld.get_fld(
t, fail_silently=True, fix_protocol=True):
if is_domain_suffix:
result.add(result.domain_suffix, t, remove=is_allow)
else:
result.add(result.domain, t, remove=is_allow)
def test_public_ip(agent):
error = []
if not validators.ipv4(agent.get_public_ip()) and not validators.ipv6(
agent.get_public_ip()):
error.append("not ipv4 or ipv6")
assert not error
def read_iocs(cb, file=sys.stdin):
iocs = defaultdict(list)
report_id = hashlib.md5()
report_id.update(str(time.time()).encode("utf-8"))
for idx, line in enumerate(sys.stdin):
line = line.rstrip("\r\n")
report_id.update(line.encode("utf-8"))
if validators.md5(line):
iocs["md5"].append(line)
elif validators.sha256(line):
eprint("line {}: sha256 provided but not yet supported by backend".
format(idx + 1))
iocs["sha256"].append(line)
elif validators.ipv4(line):
iocs["ipv4"].append(line)
elif validators.ipv6(line):
iocs["ipv6"].append(line)
elif validators.domain(line):
iocs["dns"].append(line)
else:
if cb.validate_query(line):
query_ioc = {"search_query": line}
iocs["query"].append(query_ioc)
else:
eprint("line {}: invalid query".format(idx + 1))
return (report_id.hexdigest(), dict(iocs))
def checkType(self, argument):
"""
Identify observable type
"""
if len(argument.strip()) == 0:
return None
elif argument[0] is '#':
return None
elif validators.url(argument):
return "URL"
elif validators.md5(argument):
return "MD5"
elif validators.sha1(argument):
return "SHA1"
elif validators.sha256(argument):
return "SHA256"
elif validators.sha512(argument):
return "SHA512"
elif validators.ipv4(argument):
return "IPv4"
elif validators.ipv6(argument):
return "IPv6"
elif validators.domain(argument):
return "domain"
else:
mod.display("MAIN", argument, "ERROR", "Unable to retrieve observable type")
return None
def is_mavlink_endpoint(cls: Type["Endpoint"], values: Any) -> Any:
connection_type, place, argument = (values.get("connection_type"),
values.get("place"),
values.get("argument"))
if connection_type in [
EndpointType.UDPServer,
EndpointType.UDPClient,
EndpointType.TCPServer,
EndpointType.TCPClient,
]:
if not (validators.domain(place) or validators.ipv4(place)
or validators.ipv6(place)):
raise ValueError(f"Invalid network address: {place}")
if not argument in range(1, 65536):
raise ValueError(
f"Ports must be in the range 1:65535. Received {argument}."
)
return values
if connection_type == EndpointType.Serial.value:
if not place.startswith("/") or place.endswith("/"):
raise ValueError(
f"Bad serial address: {place}. Make sure to use an absolute path."
)
if not argument in VALID_SERIAL_BAUDRATES:
raise ValueError(
f"Invalid serial baudrate: {argument}. Valid option are {VALID_SERIAL_BAUDRATES}."
)
return values
raise ValueError(
f"Invalid connection_type: {connection_type}. Valid types are: {[e.value for e in EndpointType]}."
)
def checkType(self, argument):
"""
Identify observable type
"""
if not argument or len(argument.strip()) == 0:
return None
elif argument[0] is '#':
return None
elif validators.url(argument):
return "URL"
elif validators.md5(argument):
return "MD5"
elif validators.sha1(argument):
return "SHA1"
elif validators.sha256(argument):
return "SHA256"
elif validators.sha512(argument):
return "SHA512"
elif validators.ipv4(argument):
return "IPv4"
elif validators.ipv6(argument):
return "IPv6"
elif validators.domain(argument):
return "domain"
else:
return None
def get_public_ip(self):
try:
ifconfig = requests.get("https://ifconfig.co/ip", timeout=5).text.strip()
if not validators.ipv4(ifconfig) and not validators.ipv6(ifconfig):
icanhaz = requests.get("https://icanhazip.com", timeout=7).text.strip()
if not validators.ipv4(icanhaz) and not validators.ipv6(icanhaz):
return "error"
else:
return icanhaz
else:
return ifconfig
except Exception as e:
self.logger.debug(e)
return "error"
def check_endpoint(self, endpoint):
is_ipv4 = validators.ipv4(endpoint)
is_ipv6 = validators.ipv6(endpoint)
if is_ipv4 or is_ipv6:
return False
if validators.domain(endpoint):
return True
raise CheckThirdpartEndpointFailed(msg="invalid endpoint")
def validate(self, val):
# hack: :: is a valid IPv6 address
if val == '::':
return True
try:
return validators.ipv4(val) or (validators.ipv6(val))
except validators.ValidationFailure:
pass
def url_from_host(host):
"""Get a URL with given host.
:param host: a host value to be used in the URL
:returns: a URL to be used during testing
"""
if ipv6(host):
host = '['+host+']'
return 'http://'+host
def _get_indicator_type(indicator):
if validators.ipv4(indicator):
return "IPv4"
elif validators.ipv6(indicator):
return "IPv6"
elif validators.url(indicator):
return "URL"
return "domain"
def url_from_host(host):
"""Get a URL with given host.
:param host: a host value to be used in the URL
:returns: a URL to be used during testing
"""
if ipv6(host):
host = '[' + host + ']'
return 'http://' + host
def do_print_report(self, line):
'''Prints to screen a readable output of the table as IP DOMAIN '''
invalid = []
master = target.master
for ip in sorted(master.keys()):
for fqdn in master[ip]['fqdns']:
if validators.ipv6(ip) or validators.ipv4(
ip) or 'arpa' in fqdn:
invalid.append('{}\t{}'.format(ip, fqdn))
else:
print("{}\t{}".format(str(ip), str(fqdn)))
def query(target):
if validators.ipv4(target) or validators.ipv6(target):
ip_whois = IPWhois(target)
return ip_whois.lookup_rdap(asn_methods=["dns", "whois", "http"],
inc_nir=True)
elif validators.domain(target):
result = whois.whois(target)
return result
else:
logging.error(f"[whois query] not valid target: {target}")
return None
def get_domain(self, var, default=NOTSET):
# type: (str, Union[str, NotSetType]) -> str
value = self.get(var, default=default)
if value == default:
return value
if not (validators.ipv4(value) or validators.ipv6(value) or validators.domain(value)):
raise VariableTypeError('Environment variable {var} with value {val} is not a valid hostname.'
.format(var=var, val=value))
return value
def get_ip(self, var, default=NOTSET):
# type: (str, Union[str, NotSetType]) -> str
value = self.get(var, default=default)
if value == default:
return value
if not (validators.ipv4(value) or validators.ipv6(value)):
raise VariableTypeError('Environment variable {var} with value {val} could not be converted into ipv4/ipv6.'
.format(var=var, val=value))
return value
def _get_kind(address: str):
if validators.ipv4(address):
return "IPv4Address"
elif validators.ipv6(address):
return "IPv6Address"
elif validators.ipv4_cidr(address):
return "IPv4Network"
raise PluginException(
cause=f"Kind not detected from address: {address}",
assistance="Check address input and try again. Allowed kind are: " "IPv4Address, IPv6Address, IPv4Network.",
)
def ipv6_address(value):
"""
Validate whether or not the given string is a valid IP version 6 address.
Args:
value (str): the value to validate.
Raises:
ValidationError: when the value is not a valid IP version 6 address.
"""
if not validators.ipv6(value):
raise ValidationError('{!r} is not a valid IPv6 address'.format(value))
def determine_resource_type(value):
if validators.ipv4(value) or validators.ipv6(value):
return 'ip'
if validators.domain(value):
return 'domain'
if validators.email(value):
return 'email'
if validators.mac_address(value):
return 'mac_address'
def _determine_address_type(address: str) -> str:
if validators.ipv4(address):
return "ipv4"
if validators.ipv6(address):
return "ipv6"
if validators.domain(address):
return "fqdn"
if re.search('/', address):
return "cidr"
raise PluginException(
cause="Unknown address type provided.",
assistance=f"{address} is not one of the following: IPv4, IPv6, CIDR or domain name."
)
def remove(request, server):
vinfo = viewinfo.prepare(request)
dnsresolver_srv = vapi.get_dnsresolver(vinfo.hostname_default)
if len(dnsresolver_srv.data['name-server']) == 0:
return redirect('dnsresolver:dnsresolver-list')
if dnsresolver_srv.success:
if 'name-server' in dnsresolver_srv.data:
if validators.ipv6(server.strip()) or validators.ipv4(server.strip()):
v = vapi.delete_dnsresolver(vinfo.hostname_default, server.strip())
return redirect('dnsresolver:dnsresolver-list')
def extract_urls(msg: str) -> list:
urls = re.findall(
r"(?m)\b(?:http(?:s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~:\/?#[\]@!\$&'\(\)\*\+,;=.]+\b",
msg.lower()
)
normalized_urls = []
for url in urls:
if '@' in url or validators.ipv6(url) or validators.ipv4(url):
continue
normalized_urls.append(url)
return normalized_urls
def ip_in_url(urls):
for url in urls: #http://ufpel.edu.br/portal
try:
if(validators.url(url)): #if url is valid
netlocal = get_domain(url) #ufpel.edu.br
#return on first occurrence of ip in URL
if validators.ipv4(netlocal):
return True
if validators.ipv6(netlocal):
return True
except:
continue
return False
def get_data_type(indicator):
if validators.ipv4(indicator) or validators.ipv6(indicator):
return "IP"
elif validators.url(indicator):
return "URL"
elif validators.domain(indicator):
return "DOMAIN"
elif validators.sha1(indicator):
return "FILE_SHA1"
raise PluginException(
cause="Invalid indicator input provided.",
assistance="Supported indicators are IP, URL, domain and SHA1 hash."
)
def validate_url(value: str) -> bool:
if (validatorslib.url(value) or
validatorslib.domain(value) or
validatorslib.ipv4(value) or
validatorslib.ipv6(value) or
validatorslib.ipv4_cidr(value) or
validatorslib.ipv6_cidr(value) or
validatorslib.slug(value)):
return True
if DOMAIN_PORT_REGEX.match(value):
return True
return False
def parse_items(items: Optional[str]) -> List[Attr]:
parsed_items: List[Attr] = []
if not items:
return parsed_items
for item in items.split('\n'):
if item:
item = ''.join(item.split()) # Normalize whitespace
item = urllib.parse.unquote_plus(item)
if domain(item):
typ = AttrType.DOMAIN
search_types = [AttrType.DOMAIN]
report_types = [AttrType.DOMAIN]
elif url(item):
typ = AttrType.URL
search_types = [AttrType.URL]
report_types = [AttrType.URL]
# Remove arguments from URLs
item = get_canonical_url(item)
elif defanged_url(item):
typ = AttrType.URL
search_types = [AttrType.URL]
report_types = [AttrType.URL]
# MISP wants a correct URL, so replace hxx with htt
item = item.replace('hxx', 'htt', 1)
elif ipv4(item) or ipv6(item):
typ = AttrType.IP_SRC
search_types = [
AttrType.DOMAIN_IP,
AttrType.IP_SRC,
AttrType.IP_SRC_PORT,
AttrType.IP_DST,
AttrType.IP_DST_PORT,
]
report_types = [AttrType.IP_SRC]
elif md5(item):
typ = AttrType.MD5
search_types = [AttrType.MD5, AttrType.FILENAME_MD5]
report_types = [AttrType.MD5]
elif sha1(item):
typ = AttrType.SHA1
search_types = [AttrType.SHA1, AttrType.FILENAME_SHA1]
report_types = [AttrType.SHA1]
elif sha256(item):
typ = AttrType.SHA256
search_types = [AttrType.SHA256, AttrType.FILENAME_SHA256]
report_types = [AttrType.SHA256]
else:
raise ParseException(f'Could not parse {item}')
parsed_items.append(Attr(value=item, type=typ, search_types=search_types, report_types=report_types))
return parsed_items
def cheacking(inp):
k = validators.domain(inp)
if k is True:
return "Domain"
else:
k = validators.ipv4(inp)
h = validators.ipv6(inp)
if k is True or h is True:
return "IP"
else:
k = validators.url(inp)
if k is True:
return "URL"
else:
return "File_Hash"
def remove(request, server):
hostname_default = perms.get_hostname_prefered(request)
ntp_servers = vapi.get_ntp(hostname_default)
if len(ntp_servers.data['name-server']) == 0:
return redirect('ntp:ntp-list')
if ntp_servers.success:
if 'server' in ntp_servers.data:
if validators.ipv6(server.strip()) or validators.ipv4(
server.strip()) or validators.domain(server.strip()):
v = vapi.delete_ntp(hostname_default, server.strip())
return redirect('ntp:ntp-list')
def do_print_domains(self, line):
domains = []
# master = target.master.keys()
for ip in target.master.keys():
for domain in target.master[ip]['fqdns']:
if validators.ipv6(domain) or validators.ipv4(
domain) or 'arpa' in domain:
pass
else:
domains.append(domain)
unique_domains = set(domains)
for i in unique_domains:
print(i)
print("Total:\t{}").format(len(unique_domains))
def add_new_ip(self, ipaddr, source=""):
# global target
ip = ipaddr
# if is_ipv4(ip):
if validators.ipv6(ip) or validators.ipv4(ip):
if ip not in target.master.keys():
target.master[ip] = {}
target.master[ip]['source'] = ['{}'.format(source)]
target.master[ip]['fqdns'] = []
target.master[ip]['ports'] = []
target.master[ip]['cidr'] = []
target.master[ip]['netname'] = []
else:
print("{} already exists in the table".format(ip))
else:
print("{} is not a valid IPv4 or IPv6 address".format(ip))
def test_returns_true_on_valid_ipv4_address(address):
assert ipv6(address)
def isipv6(value):
try:
validators.ipv6(value)
except validators.ValidationFailure:
return False
return True
def test_returns_failed_validation_on_invalid_ipv6_address(address):
assert isinstance(ipv6(address), ValidationFailure)
def arg_validator(arg, vlist=None):
"""
检查数据，可对同一个数据进行多种检查
arg ： 字符串，要验证的参数
vlist ： 列表，验证列表，每个元素包含两项.
第一个项是检查类型（字符串），第二项是可选参数字典,类似：
[
("检查类型",{"可选参数1":参数值,"可选参数2":参数值...})，
("检查类型",{"可选参数1":参数值,"可选参数2":参数值...})，
...
]
返回： 双元组,第一项为True 或 False,第二项为验证失败的类型(第一项为True的话第二项就留空)
注意：
vlist 列表的第一项可以是字符串 "required"，用于表示是必填项：
如果第一项不是，而且要验证的 arg 为空,会直接返回 True,不是的继续验证。
如果第一项是，就完全按照 vlist[1:] 的要求验证
vlist 的元素如果是验证整数/小数/email等不需要附加参数的可以直接传入验证类型字符串即可
用例（使用args_validator函数的，看这里vdict每个键值对的形式）：
vdict = {
"token": ["required", "uuid"],
"username": ["required", ("length", {"min": 4, "max": 30}), "safe_str"],
"password": ["required", ("length", {"min": 4, "max": 20}), "safe_str"],
"captcha": ["required", ("length", {"min": 4, "max": 8}), "safe_str"],
}
form = args_validator(self.request.arguments, vdict)
"""
if not any((isinstance(vlist, list), isinstance(vlist, tuple))):
einfo = "不支持的数据类型！应使用列表或元组,但输入的是{}".format(type(vlist))
raise ValueError(einfo)
if vlist[0] == "required":
# 第一项不是 required 的,把第一项的 "required" 去掉
vlist = vlist[1:]
else:
# 第一项不是 required 的，如果 arg 是空的，直接返回 True,不是的继续验证
if not arg:
return True, None
# 待返回的验证结果
verification = None
failed_type = None # 验证失败的类型
# 开始检查，有一个不通过就返回 False
for i in vlist:
local_verification = None
if isinstance(i, str): # 如果是字符串的话就改为元组
i = (i, {})
if len(i) == 1: # 只有一个元素的，添加一个空字典
i = (i[0], {})
vtype = i[0] # 检查类型是第一项
vdict = i[1] # 检查类型所需的可选参数字典
# 在 validators 之外添加的
# 没有空白
if vtype == "no_space":
if not re.search(r"\s", arg):
local_verification = True
# 安全字符串，只包含 0-9a-zA-Z-空格和下划线
elif vtype == "safe_str":
if re.match(r"^[0-9a-zA-Z-_ ]+$", arg, flags=re.U):
local_verification = True
# 是否包含
elif vtype == "in":
# 迭代 vdict 的键值(所以键名无所谓)
for v in vdict.values():
if arg not in v:
local_verification = False
break
elif vtype == "not_in":
# 迭代 vdict 的键值(所以键名无所谓)
for v in vdict.values():
if arg in v:
local_verification = False
break
# 字符串形式的数字
elif vtype == "str_number":
if re.match(r"[+-]?\d+$", arg, flags=re.U) or \
re.match(r"[+-]?\d+\.\d+$", arg, flags=re.U):
local_verification = True
elif vtype == "str_int":
if re.match(r"[+-]?\d+$", arg, flags=re.U):
local_verification = True
elif vtype == "str_float":
if re.match(r"[+-]?\d+\.\d+$", arg, flags=re.U):
local_verification = True
# 数字
elif vtype == "number": # 整数或浮点数都可以
local_verification = isinstance(arg, int) or isinstance(arg, float)
elif vtype == "int":
local_verification = isinstance(arg, int)
elif vtype == "float":
local_verification = isinstance(arg, float)
# 直接调用 validators的
elif vtype == "length":
local_verification = validators.length(arg, **vdict)
elif vtype == "url":
local_verification = validators.url(arg, **vdict)
elif vtype == "email":
local_verification = validators.email(arg, **vdict)
elif vtype == "ip": # ipv4 或 ipv6都可以
local_verification = any((validators.ipv4(arg, **vdict),
validators.ipv6(arg, **vdict)))
elif vtype == "between":
local_verification = validators.between(arg, **vdict)
elif vtype == "uuid":
local_verification = validators.uuid(arg, **vdict)
elif vtype == "ipv4":
local_verification = validators.ipv4(arg, **vdict)
elif vtype == "ipv6":
local_verification = validators.ipv6(arg, **vdict)
elif vtype == "mac_address":
local_verification = validators.mac_address(arg, **vdict)
elif vtype == "iban":
local_verification = validators.iban(arg, **vdict)
elif vtype == "slug":
local_verification = validators.slug(arg, **vdict)
elif vtype == "truthy":
local_verification = validators.truthy(arg, **vdict)
# 对于验证不为真或没有验证的
# 不为真的时候返回的是: ValidationFailure(......)
if not local_verification:
verification = False
failed_type = vtype
break # 有一条不为 True， 直接返回 False
else:
verification = True
# 处理返回值
if verification not in(False, True):
verification = False
if not verification:
return verification, failed_type
else:
return True, None
