def address(self, value):
ip = IPAddress(self.ipformat(value))
if ip.is_loopback():
raise ValidationError("You cannot use a loopback address")
if ip.is_multicast():
raise ValidationError("You cannot use a multicast address")
self._address = value
def validate_ipv4_address(cls, _, value):
"""
Ensures the :attr:`ip` address is valid. This checks to ensure
that the value provided is:
* not a hostmask
* not link local (:rfc:`3927`)
* not used for multicast (:rfc:`1112`)
* not a netmask (:rfc:`4632`)
* not reserved (:rfc:`6052`)
* a private address (:rfc:`1918`)
"""
if value is None:
return value
try:
address = IPAddress(value)
except (AddrFormatError, ValueError) as e:
raise ValueError(
"%s is not a valid address format: %s" % (value, e))
if ALLOW_AGENT_LOOPBACK:
loopback = lambda: False
else:
loopback = address.is_loopback
if any([address.is_hostmask(), address.is_link_local(),
loopback(), address.is_multicast(),
address.is_netmask(), address.is_reserved()]):
raise ValueError("%s is not a valid address type" % value)
return value
def address(self, value):
ip = IPAddress(self.ipformat(value))
if ip.is_loopback():
raise ValidationError("You cannot use a loopback address")
if ip.is_multicast():
raise ValidationError("You cannot use a multicast address")
self._address = value
def is_valid_address(address):
''' Validate whether the address provided is routable unicast address '''
addr = IPAddress(address)
if addr.is_loopback() or addr.is_reserved() or addr.is_private()\
or addr.is_link_local() or addr.is_multicast():
return False
return True
def call(self, url, context):
if self.url_can_resolve(url):
try:
ip = yield self.resolver.get_host_by_name(url.domain)
ip = IPAddress(ip)
except Exception:
# context["event"].target.respond(
# u'[Error] Failed to handle URL: {}'.format(
# url.to_string()
# )
# )
self.plugin.logger.exception("Error while checking DNS")
returnValue(STOP_HANDLING)
return
if ip.is_loopback() or ip.is_private() or ip.is_link_local() \
or ip.is_multicast():
self.plugin.logger.warn(
"Prevented connection to private/internal address"
)
returnValue(STOP_HANDLING)
return
headers = {}
if url.domain in context["config"]["spoofing"]:
user_agent = context["config"]["spoofing"][url.domain]
if user_agent:
headers["User-Agent"] = user_agent
else:
headers["User-Agent"] = context["config"].get(
"default_user_agent",
"Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 "
"Firefox/36.0"
)
domain_langs = context.get("config") \
.get("accept_language", {}) \
.get("domains", {})
if url.domain in domain_langs:
headers["Accept-Language"] = domain_langs.get(url.domain)
else:
headers["Accept-Language"] = context.get("config") \
.get("accept_language", {}) \
.get("default", "en")
session = self.get_session(url, context)
session.get(unicode(url), headers=headers, stream=True,
background_callback=self.background_callback) \
.addCallback(self.callback, url, context, session) \
.addErrback(self.errback, url, context, session)
returnValue(STOP_HANDLING)
def validate(url,
schemes=None,
tlds=None,
private=None,
local=None,
credentials=None):
'''
Validate and normalize an URL
:param str url: The URL to validate and normalize
:return str: The normalized URL
:raises ValidationError: when URL does not validate
'''
url = url.strip()
private = config_for(private, 'URLS_ALLOW_PRIVATE')
local = config_for(local, 'URLS_ALLOW_LOCAL')
credentials = config_for(credentials, 'URLS_ALLOW_CREDENTIALS')
schemes = config_for(schemes, 'URLS_ALLOWED_SCHEMES')
tlds = config_for(tlds, 'URLS_ALLOWED_TLDS')
match = URL_REGEX.match(url)
if not match:
error(url)
scheme = (match.group('scheme') or '').lower()
if scheme and scheme not in schemes:
error(url, 'Invalid scheme {0}'.format(scheme))
if not credentials and match.group('credentials'):
error(url, 'Credentials in URL are not allowed')
tld = match.group('tld')
if tld and tld not in tlds and idna(tld) not in tlds:
error(url, 'Invalid TLD {0}'.format(tld))
ip = match.group('ipv6') or match.group('ipv4')
if ip:
try:
ip = IPAddress(ip)
except AddrFormatError:
error(url)
if ip.is_multicast():
error(url, '{0} is a multicast IP'.format(ip))
elif not ip.is_loopback() and ip.is_hostmask() or ip.is_netmask():
error(url, '{0} is a mask IP'.format(ip))
if not local:
if ip and ip.is_loopback() or match.group('localhost'):
error(url, 'is a local URL')
if not private and ip and ip.is_private():
error(url, 'is a private URL')
return url
def is_valid_gateway(ip_addr):
"""Valid the format of gateway"""
invalid_ip_prefix = ['0', '224', '169', '127']
try:
# Check if ip_addr is an IP address and not start with 0
ip_addr_prefix = ip_addr.split('.')[0]
if is_valid_ip(ip_addr) and ip_addr_prefix not in invalid_ip_prefix:
ip_address = IPAddress(ip_addr)
if not ip_address.is_multicast():
# Check if ip_addr is not multicast and reserved IP
return True
return False
except Exception:
return False
def is_valid_gateway(ip_addr):
"""Valid the format of gateway"""
invalid_ip_prefix = ['0', '224', '169', '127']
try:
# Check if ip_addr is an IP address and not start with 0
ip_addr_prefix = ip_addr.split('.')[0]
if is_valid_ip(ip_addr) and ip_addr_prefix not in invalid_ip_prefix:
ip_address = IPAddress(ip_addr)
if not ip_address.is_multicast():
# Check if ip_addr is not multicast and reserved IP
return True
return False
except Exception:
return False
def check_ip_address(ipaddr):
ip_attributes = []
ipaddress = IPAddress(ipaddr)
if ipaddress.is_private():
ip_attributes.append("IP Address is Private")
else:
ip_attributes.append("IP Address is public")
if ipaddress.is_unicast():
ip_attributes.append("IP Address is unicast")
elif ipaddress.is_multicast():
ip_attributes.append("IP Address is multicast")
if ipaddress.is_loopback():
ip_attributes.append("IP Address is loopback")
return "\n".join(ip_attributes)
def ip_check_routable(item):
ip_addr = IPAddress(item)
# This prevents netaddr allowing shortened ip addresses
if not str(ip_addr) == item:
raise AddrFormatError("IP Malformed {}".format(item))
# Check for reserved IP addresses
if any([
ip_addr.is_multicast(),
ip_addr.is_private(),
ip_addr.is_loopback(),
ip_addr.is_link_local(),
ip_addr.is_reserved()
]):
raise AddrFormatError("IP is reserved {}".format(item))
# Check to see if IP is IPv4
# elif ip_addr.version is not 4:
# raise AddrFormatError("IP is not IPv4")
return True
def test_is_multicast():
ip = IPAddress('239.192.0.1')
assert ip.is_multicast()
def callback(self, result, url, context, session):
response = result[0]
content = result[1]
self.plugin.logger.trace(
"Headers: {0}", list(response.headers)
)
self.plugin.logger.trace("HTTP code: {0}", response.status_code)
new_url = urlparse.urlparse(response.url)
if self.url_can_resolve(url):
try:
ip = yield self.resolver.get_host_by_name(new_url.hostname)
ip = IPAddress(ip)
except Exception:
# context["event"].target.respond(
# u'[Error] Failed to handle URL: {}'.format(
# url.to_string()
# )
# )
self.plugin.logger.exception("Error while checking DNS")
returnValue(STOP_HANDLING)
return
if ip.is_loopback() or ip.is_private() or ip.is_link_local() \
or ip.is_multicast():
self.plugin.logger.warn(
"Prevented connection to private/internal address"
)
returnValue(STOP_HANDLING)
return
if content is None:
self.plugin.logger.debug("No content returned")
return
soup = BeautifulSoup(content)
if soup.title and soup.title.text:
title = soup.title.text.strip()
title = re.sub("[\n\s]+", " ", title)
title = to_unicode(title)
title_limit = self.urls_plugin.config.get("max_title_length", 150)
if len(title) > title_limit:
title = title[:title_limit - 15] + u"... (truncated)"
if response.status_code == requests.codes.ok:
context["event"].target.respond(
u'"{0}" at {1}'.format(
title, new_url.hostname
)
)
else:
context["event"].target.respond(
u'[HTTP {0}] "{1}" at {2}'.format(
response.status_code,
title, new_url.hostname
)
)
else:
if response.status_code != requests.codes.ok:
context["event"].target.respond(
u'HTTP Error {0}: "{1}" at {2}'.format(
response.status_code,
STATUS_CODES.get(response.status_code, "Unknown"),
new_url.hostname
)
)
else:
self.plugin.logger.debug("No title")
self.save_session(session)
def reserved_ip_check(ip_string):
"""determine if IP address in RFC1918 or reserved"""
# IP details for invalid IP addresses
invalid_ip_details = {
"country": "INVALID",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "INVALID",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# IP details for MULTICAST IP addresses
multicast_ip_details = {
"country": "MULTICAST",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "MULTICAST",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# IP details for PRIVATE IP addresses
private_ip_details = {
"country": "PRIVATE",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "PRIVATE",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# IP details for RESERVED IP addresses
reserved_ip_details = {
"country": "RESERVED",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "RESERVED",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# IP details for NETMASK IP addresses
netmask_ip_details = {
"country": "NETMASK",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "NETMASK",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# IP details for HOSTMASK IP addresses
hostmask_ip_details = {
"country": "HOSTMASK",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "HOSTMASK",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# IP details for LOOPBACK IP addresses
loopback_ip_details = {
"country": "LOOPBACK",
"location": RESERVED_IP_COORDINATES,
"subdivisions": "LOOPBACK",
"dch_company": "",
"asn_number": "",
"asn_name": ""
}
# Check to see if IP matches a reserved category
try:
ip_address = IPAddress(ip_string)
except AddrFormatError:
return invalid_ip_details
if ip_address.is_multicast():
return multicast_ip_details
elif ip_address.is_private():
return private_ip_details
elif ip_address.is_reserved():
return reserved_ip_details
elif ip_address.is_netmask():
return netmask_ip_details
elif ip_address.is_hostmask():
return hostmask_ip_details
elif ip_address.is_loopback():
return loopback_ip_details
elif ip_address.is_unicast() and not ip_address.is_private():
# Boolean to be returned if IP is Public
ip_reserved = False
return ip_reserved
else:
return invalid_ip_details
def test_is_multicast():
ip = IPAddress('239.192.0.1')
assert ip.is_multicast()
import netaddr
import socket
from netaddr import IPAddress
# Allow user input for ip
print("what is the ip?")
addr = input()
# reverse DNS
dns = socket.gethostbyaddr(addr)
# Define ip
ip = IPAddress(addr)
# Print info
print("IP version -", ip.version)
print("private -", ip.is_private())
print("unicast -", ip.is_unicast())
print("multicast -", ip.is_multicast())
print("IP in bits -", ip.bits())
print("reverse dns -", dns)
