def test_some_proxies(self):
"""
After :py:meth:`INetwork.route.create_proxy_to` is used to create
several proxies, :py:meth:`INetwork.enumerate_proxies` returns a
:py:class:`list` including an object for each of those proxies.
"""
ip = IPAddress("10.1.2.3")
port = 4567
proxy_one = self.network.create_proxy_to(ip, port)
proxy_two = self.network.create_proxy_to(ip, port + 1)
self.assertEqual(set([proxy_one, proxy_two]),
set(self.network.enumerate_proxies()))
def port_available(host, port):
"""
test to see if a port is available.
"""
if isinstance(IPAddress(host), IPv6Address):
addrclass = socket.AF_INET6
else:
addrclass = socket.AF_INET
s = socket.socket(addrclass)
try:
s.bind((str(host), port))
except socket.error, e:
return False
def getAddresses():
from scapy.all import get_if_addr, get_if_list
from ipaddr import IPAddress
addresses = set()
for i in get_if_list():
try:
addresses.add(get_if_addr(i))
except:
pass
if '0.0.0.0' in addresses:
addresses.remove('0.0.0.0')
return [IPAddress(addr) for addr in addresses]
def test_environment_setup_rackspace(self):
"""
Uses environment variables for cluster configuration if option missing.
This test checks a typical Rackspace configuration.
"""
self.environ['FLOCKER_ACCEPTANCE_HOSTNAME_TO_PUBLIC_ADDRESS'] = '{}'
self.environ['FLOCKER_ACCEPTANCE_VOLUME_BACKEND'] = 'openstack'
options = BenchmarkOptions()
options.parseOptions([])
cluster = get_cluster(options, self.environ)
self.assertEqual(cluster.control_node_address(),
IPAddress(_ENV_CONTROL_SERVICE_ADDRESS))
def test_scenario_throws_exception_when_already_started(self, _logger):
"""
start method in the ``RequestLoadScenario`` throws a
``RequestScenarioAlreadyStarted`` if the scenario is already started.
"""
c = Clock()
node1 = Node(uuid=uuid4(), public_address=IPAddress('10.0.0.1'))
node2 = Node(uuid=uuid4(), public_address=IPAddress('10.0.0.2'))
cluster = BenchmarkCluster(
node1.public_address,
lambda reactor: FakeFlockerClient([node1, node2]),
{node1.public_address, node2.public_address},
default_volume_size=DEFAULT_VOLUME_SIZE
)
sample_size = 5
s = read_request_load_scenario(c, cluster, sample_size=sample_size)
s.start()
self.assertRaises(RequestScenarioAlreadyStarted, s.start)
def initialize(cls, path, authority, hostname, begin=None):
"""
Generate a certificate signed by the supplied root certificate.
:param FilePath path: Directory where the certificate will be stored.
:param RootCredential authority: The certificate authority with
which this certificate will be signed.
:param datetime begin: The datetime from which the generated
certificate should be valid.
:param bytes hostname: The hostname of the node where the control
service will be running.
"""
# The common name for the control service certificate. This is
# used to distinguish between control service and node
# certificates. In practice it gets overridden for validation
# purposes by the subjectAltName, so we add record there too.
name = b"control-service"
# The organizational unit is set to the organizational_unit of the
# authority, which in our case is the cluster UUID.
organizational_unit = authority.organizational_unit
dn = DistinguishedName(
commonName=name, organizationalUnitName=organizational_unit
)
keypair = flocker_keypair()
request = keypair.keypair.requestObject(dn)
serial = os.urandom(16).encode(b"hex")
serial = int(serial, 16)
try:
IPAddress(hostname)
except ValueError:
alt_name = b"DNS:" + hostname
else:
alt_name = b"IP:" + hostname
cert = sign_certificate_request(
authority.credential.keypair.keypair,
authority.credential.certificate.original.get_subject(), request,
serial, EXPIRY_20_YEARS, 'sha256', start=begin,
additional_extensions=[
# subjectAltName overrides common name for validation
# purposes, and we want to be able to validate against
# "control-service", so we include it too.
crypto.X509Extension(b"subjectAltName", False,
b"DNS:control-service," + alt_name),
])
credential = FlockerCredential(
path=path, keypair=keypair, certificate=cert)
credential.write_credential_files(
b"control-{}.key".format(hostname),
b"control-{}.crt".format(hostname))
instance = cls(credential=credential)
return instance
def checkIP(addr):
IPS = (
IPNetwork('59.66.0.0/16'),
IPNetwork('166.111.0.0/16'),
IPNetwork('101.5.0.0/16'),
IPNetwork('219.223.160.0/19'),
# private address
IPNetwork('127.0.0.0/8'),
IPNetwork('10.0.0.0/8'),
IPNetwork('192.168.0.0/16'),
)
if BlockIpModel.objects.filter(ip=addr).count() > 0:
return False
return any([IPAddress(addr) in x for x in IPS])
def test_get_node_init_process_name(self):
"""
Success results in output of string containing name of process 1.
"""
d = get_node_init_process_name(
_LocalRunner(),
Node(uuid=uuid4(), public_address=IPAddress('10.0.0.1')),
)
def check(result):
self.assertEqual(result, _pid_1_name)
d.addCallback(check)
return d
def to_python(self, value):
if not value:
return ""
if isinstance(value, _IPAddrBase):
return value
if value == "None":
return ""
else:
try:
return IPAddress(value.encode('latin-1'), version=6)
except Exception, e:
raise ValidationError("Invalid IPv6 address: %s" % e)
def test_no_such_process(self):
"""
If processes do not exist, only wallclock time is returned.
"""
d = get_node_cpu_times(
Clock(),
_LocalRunner(),
Node(uuid=uuid4(), public_address=IPAddress('10.0.0.1')),
['n0n-exist'],
)
d.addCallback(self.assertEqual, {WALLCLOCK_LABEL: 0.0})
return d
def to_python(self, value):
if not value:
return ""
if isinstance(value, _IPAddrBase):
return value
if value == "None":
return ""
else:
try:
return IPAddress(value)
except Exception as e:
raise ValidationError("Invalid IP address: %s" % e)
def is_valid_ip_address(process_ip):
ip = None
if py_version[0] == 2:
# python 2
ip = IPAddress(process_ip)
elif py_version[0] == 3:
# python 3
ip = ip_address(process_ip)
if ip is None or ip.is_reserved or ip.is_private or \
ip.is_loopback or ip.is_unspecified or \
ip.is_multicast or ip.is_link_local:
return False
return True
def _parse_netip_and_netmask(self, netip, netmask):
""" validates net ip and network """
if '/' in netip:
netip, netmask = netip.split("/", 1)
checkip(netip)
ipobj = IPAddress(ipunwrap_strip(netip))
try:
netmask = int(netmask)
except (TypeError, ValueError):
netmask = None
biggest_netmask = ipobj.max_prefixlen if ipobj.version == 4 else 64 + 1
netmask = netmask if netmask > 0 and netmask < biggest_netmask else None
return netip, netmask, ipobj.version
def ScopedIPAddress(*args, **kwargs):
zone_index = None
if kwargs.get("version") == 6 and kwargs.pop("allow_zone_index", False):
value = args[0]
if value.count("%") == 1:
value, zone_index = value.split("%")
args = (value, ) + args[1:]
result = IPAddress(*args, **kwargs)
result.zone_index = zone_index
result._string_from_ip_int = lambda ip_int: result.__class__._string_from_ip_int(
result, ip_int) + (("%" + result.zone_index)
if result.zone_index is not None else "")
return result
def checkFirewall(clientip):
try:
clientinrange = any([
IPAddress(clientip) in IPNetwork(i)
for i in AceConfig.firewallnetranges
])
except:
logger.error('Check firewall netranges settings !')
return False
if (AceConfig.firewallblacklistmode
and clientinrange) or (not AceConfig.firewallblacklistmode
and not clientinrange):
return False
return True
def postOptions(self):
# Mandatory parameters
# Validate image
if self['image'] is None:
raise usage.UsageError(
"image parameter must be provided"
)
# Validate mountpoint
if self['mountpoint'] is None:
raise usage.UsageError("mountpoint is a mandatory parameter")
else:
try:
FilePath(self['mountpoint'])
except ValueError:
raise usage.UsageError("mountpoint has to be an absolute path")
# Validate app per node
if self['apps-per-node'] is None:
raise usage.UsageError("apps-per-node is a mandatory parameter")
else:
try:
self['apps-per-node'] = int(self['apps-per-node'])
except ValueError:
raise usage.UsageError("apps-per-node has to be an integer")
# Validate control node
if self['control-node'] is None:
raise usage.UsageError("control-node is a mandatory parameter")
else:
try:
IPAddress(self['control-node'])
except ValueError:
raise usage.UsageError("control-node has to be an IP address")
# Validate certificate directory
if self['cert-directory'] is None:
raise usage.UsageError("cert-directory is a mandatory parameter")
# Validate optional parameters
# Note that we don't check if those parameters are None, because
# all of them have default value and can't be none. If they are,
# and exception will be raised
try:
self['max-size'] = int(self['max-size'])
except ValueError:
raise usage.UsageError(
"The max-size timeout must be an integer.")
try:
self['wait'] = timedelta(seconds=int(self['wait']))
except ValueError:
raise usage.UsageError("The wait timeout must be an integer.")
def parse_iptables_options(argv):
"""
Parse a single line of iptables-save(8) output from the NAT table section.
:param argv: A :py:class:`list` of :py:class:`bytes` instances like an
iptables argv (not including ``b"iptables"`` as ``argv[0]``).
:return: A :py:class:`RuleOptions` instance holding the values taken from
``argv``.
"""
# "Parsing" things like this:
#
# -A PREROUTING -p tcp -m tcp --dport 4567 -m addrtype --dst-type LOCAL
# -m comment --comment flocker -j DNAT --to-destination 10.1.2.3
#
# -A OUTPUT -p tcp -m tcp --dport 4567 -m addrtype --dst-type LOCAL -j DNAT
# --to-destination 10.1.2.3
#
# -A POSTROUTING -p tcp -m tcp --dport 4567 -j MASQUERADE
#
# To avoid having to know about every single possible current and future
# iptables option, don't try to parse the whole line. Just look for things
# we expect and recognize.
comment = None
destination_port = None
to_destination = None
try:
destination_port_index = argv.index(b"--dport")
destination_port = int(argv[destination_port_index + 1])
except (IndexError, ValueError):
destination_port = None
try:
to_destination_index = argv.index(b"--to-destination")
to_destination = IPAddress(argv[to_destination_index + 1])
except (IndexError, ValueError):
to_destination = None
try:
comment_index = argv.index(b"--comment")
comment = argv[comment_index + 1]
except (IndexError, ValueError):
comment = None
return RuleOptions(
comment=comment,
destination_port=destination_port,
to_destination=to_destination)
def whois(host,
index=None,
sourcetype="whois",
source="whois_search_command",
logger=None):
"""
Performs a whois request. If the host is a domain-name then a normal DNS whois will be
performed. If the name is an IP address, then an IP whois will be done.
"""
# See if this is an IP address. If so, do an IP whois.
try:
# The following will throw a ValueError exception indicating that this is not an IP address
IPAddress(host)
whois_object = IPWhois(host)
results_orig = whois_object.lookup_rdap(depth=1)
except ValueError:
# Since this isn't an IP address, run a domain whois
results_orig = get_whois(host)
if 'query' not in results_orig:
results_orig['query'] = host
result = flatten(results_orig, ignore_blanks=True)
# Pull out raw so that we can put it at the end. This is done in case the raw field contains
# things that might mess up the extractions.
raw = result.get('raw', None)
try:
del result['raw']
result['raw'] = raw
except KeyError:
pass # Ok, raw didn't exist
# Write the event as a stash new file
if index is not None:
writer = StashNewWriter(index=index,
source_name=source,
sourcetype=sourcetype,
file_extension=".stash_output")
# Log that we performed the whois request
if logger:
logger.debug("Wrote stash file=%s", writer.write_event(result))
return result
def test_run_probe_timeout(self):
"""
CreateContainer probe times-out if probe.run runs too long.
"""
clock = Clock()
node_id = uuid4()
node = Node(uuid=node_id, public_address=IPAddress('10.0.0.1'))
control_service = FakeFlockerClient([node], node_id)
cluster = BenchmarkCluster(
IPAddress('10.0.0.1'),
lambda reactor: control_service,
{},
None,
)
operation = CreateContainer(clock, cluster)
d = operation.get_probe()
control_service.synchronize_state() # creation of pull container
clock.advance(1)
control_service.synchronize_state() # deletion of pull container
clock.advance(1)
# get_probe has completed successfully
probe = self.successResultOf(d)
d = probe.run()
clock.advance(DEFAULT_TIMEOUT.total_seconds())
# No control_service.synchronize_state() call, so cluster state
# never shows container is created.
# The Deferred fails if container not created within 10 minutes.
self.failureResultOf(d)
def test_yaml_setup(self):
"""
Uses YAML file for cluster configuration if option given.
This is true even if the environment contains a valid configuration.
"""
tmpdir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, tmpdir)
with open(os.path.join(tmpdir, 'cluster.yml'), 'wt') as f:
f.write(_CLUSTER_YAML_CONTENTS)
options = BenchmarkOptions()
options.parseOptions(['--cluster', tmpdir])
cluster = get_cluster(options, self.environ)
self.assertEqual(cluster.control_node_address(),
IPAddress(_YAML_CONTROL_SERVICE_ADDRESS))
def handle_icmp_response(response):
logging.debug('Handling response in new thread.')
response = Bits(bytes=response)
source_ip = response[12 * 8:][:4 * 8]
source_ip = IPAddress(source_ip.uint)
response = response[20 * 8:] # ignore IP header
typ = response[:8]
if typ.uint != 11:
logging.debug('Not time exceed packet, ignore.')
return
logging.info('Got response from %s' % source_ip.compressed)
udpsock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
udpsock.bind(('0.0.0.0', SERVER_PORT))
udpsock.connect((source_ip.compressed, CLIENT_PORT))
udpsock.send(UDP_HELLO_MSG)
def ipaddress_from_string(ip_address_string):
"""
Parse an IPv4 or IPv6 address string and return an
IPAddress instance.
Remove the "embedded scope id" from IPv6 addresses (if there is
one).
:param str ip_address_string: The IP address string to be parsed.
:returns: An ``ipaddr.IPAddress`` instance.
"""
# There may be an embedded scope id in an IPv6 address. Discard
# it. Eg fe80::f816:3eff:fe11:ca54%eth0
parts = ip_address_string.rsplit('%', 1)
ip_address_string = parts[0]
return IPAddress(ip_address_string)
def edit_switch(switch, name, management_ip, room, processor):
if switch.host.name != name:
log_room_event("Changed name of '{}' to '{}'.".format(switch.host.name, name), processor, switch.host.room)
switch.host.name = name
if switch.host.room != room:
log_room_event("Moved switch '{}' from {} to {}.".format(switch.host.name, switch.host.room, room), processor, switch.host.room)
switch.host.room = room
if switch.management_ip != IPAddress(management_ip):
log_room_event("Changed management IP of switch '{}' from {} to {}."
.format(switch.host.name, switch.management_ip, management_ip), processor, switch.host.room)
switch.management_ip = management_ip
def get_lpm_routes(self, route_db, dst_addr):
''' find the routes to the longest prefix matches of dst. '''
max_prefix_len = 0
lpm_routes = []
for route in route_db.routes:
if IPNetwork(utils.sprint_prefix(route.prefix)).Contains(
IPAddress(dst_addr)):
next_hop_prefix_len = route.prefix.prefixLength
if next_hop_prefix_len == max_prefix_len:
lpm_routes.append(route)
elif next_hop_prefix_len > max_prefix_len:
lpm_routes = [route]
max_prefix_len = next_hop_prefix_len
return lpm_routes
def test_empty_cluster(self):
"""
CreateDataset fails if no nodes in cluster.
"""
control_service = FakeFlockerClient()
cluster = BenchmarkCluster(
IPAddress('10.0.0.1'),
lambda reactor: control_service,
{},
None,
)
d = CreateDataset(Clock(), cluster).get_probe()
self.failureResultOf(d, EmptyClusterError)
def get_lpm_route(self, route_db, dst_addr):
''' find the routes to the longest prefix matches of dst. '''
max_prefix_len = -1
lpm_route = None
for route in route_db.routes:
if IPNetwork(utils.sprint_prefix(route.prefix)).Contains(
IPAddress(dst_addr)):
next_hop_prefix_len = route.prefix.prefixLength
if next_hop_prefix_len == max_prefix_len:
raise Exception('Duplicate prefix found in routing table {}'
.format(utils.sprint_prefix(route.prefix)))
elif next_hop_prefix_len > max_prefix_len:
lpm_route = route
max_prefix_len = next_hop_prefix_len
return lpm_route
def __init__(self, base_path):
"""
:param FilePath base_path: The path beneath which all of the temporary
SSH server-related files will be created. An ``ssh`` directory
will be created as a child of this directory to hold the key pair
that is generated. An ``sshd`` directory will also be created here
to hold the generated host key. A ``home`` directory is also
created here and used as the home directory for shell logins to the
server.
"""
self.home = base_path.child(b"home")
self.home.makedirs()
ssh_path = base_path.child(b"ssh")
ssh_path.makedirs()
self.key_path = ssh_path.child(b"key")
check_call(
[b"ssh-keygen",
# Specify the path where the generated key is written.
b"-f", self.key_path.path,
# Specify an empty passphrase.
b"-N", b"",
# Generate as little output as possible.
b"-q"])
key = Key.fromFile(self.key_path.path)
sshd_path = base_path.child(b"sshd")
sshd_path.makedirs()
self.host_key_path = sshd_path.child(b"ssh_host_key")
check_call(
[b"ssh-keygen",
# See above for option explanations.
b"-f", self.host_key_path.path,
b"-N", b"",
b"-q"])
factory = OpenSSHFactory()
realm = UnixSSHRealm(self.home)
checker = _InMemoryPublicKeyChecker(public_key=key.public())
factory.portal = Portal(realm, [checker])
factory.dataRoot = sshd_path.path
factory.moduliRoot = b"/etc/ssh"
self._port = reactor.listenTCP(0, factory, interface=b"127.0.0.1")
self.ip = IPAddress(self._port.getHost().host)
self.port = self._port.getHost().port
def getNetworksFromRoutes():
from scapy.all import conf, ltoa, read_routes
from ipaddr import IPNetwork, IPAddress
## Hide the 'no routes' warnings
conf.verb = 0
networks = []
for nw, nm, gw, iface, addr in read_routes():
n = IPNetwork(ltoa(nw))
(n.netmask, n.gateway,
n.ipaddr) = [IPAddress(x) for x in [nm, gw, addr]]
n.iface = iface
if not n.compressed in networks:
networks.append(n)
return networks
def get_object(self, queryset=None):
""" Hook to ensure object is owned by request.user. """
obj = super(RecordDelete, self).get_object()
subdomain = "%s.%s" % (self.request.user.username.lower(), ROOT_DOMAIN)
subnets = Subnet.objects.filter(owner=self.request.user)
if obj.name != subdomain and \
not obj.name.endswith(".%s" % subdomain):
# name doesn't end with user.hamwan.net, check reverse
try:
ip = IPAddress(reverse(obj.name))
except ValueError:
raise Http404
if not any([(ip in subnet.network) for subnet in subnets]):
# reverse is not valid either
raise Http404
return obj
def test_get_node_cpu_times(self):
"""
Success results in output of dictionary containing process names.
"""
d = get_node_cpu_times(
Clock(),
_LocalRunner(),
Node(uuid=uuid4(), public_address=IPAddress('10.0.0.1')),
_pid_1_name,
[_pid_1_name],
)
def check(result):
self.assertEqual(result.keys(), [_pid_1_name, WALLCLOCK_LABEL])
d.addCallback(check)
return d
