def scan(graph, connectionInfo, logger, thread):
"""
Get Interface configuration from hosts over ssh. -> MTU, Rate, Ips
This method uses prepacked scripts on templates (read_InterfaceInformation).
We will not add the loopback interface or interfaces with the mac 00:00:00:00:00:00 to the graph.
Necessary values in the configuration file of this collector module:
- timeout Timeout this collector module shall use (Integer)
:param graph: Data interface object for this collector module
:type graph: insalata.model.Graph.Graph
:param connectionInfo: Configuration of this collector -> Login information
:type connectionInfo: dict
:param logger: The logger this collector shall use
:type logger: logging:Logger
:param thread: Thread executing this collector
:type thread: insalata.scanner.Worker.Worker
"""
logger.info(
"Collecting interface configuration information from network components"
)
timeout = int(connectionInfo['timeout'])
name = connectionInfo['name']
for host in graph.getAllNeighbors(Host):
if not ((host.getPowerState() is None) or
(host.getPowerState() == 'Running')):
continue
ssh = base.getSSHConnection(host)
logger.info("Starting interface configuration scan on host: {}".format(
host.getID()))
if ssh is None: #No ssh connecton is possible -> Skip this host
logger.info("Skipping host {0} as ssh connection failed.".format(
host.getID()))
continue
interfaceInformation = ssh.getInterfaceInfo()
ansible = base.getAnsibleInfo(host)
staticInterface = False
if not ansible:
continue
#get Information
for intf in interfaceInformation:
if intf['type'] == 'loopback' or intf['mac'] != "00:00:00:00:00:00":
continue
interface = graph.getOrCreateInterface(intf['mac'], name, timeout)
interface.verify(name, timeout)
host.addInterface(interface, name, timeout)
#interface = [interface for interface in hostInterfaces if interface.getMAC() == intf['mac']] #Get the right host interface
#if len(interface) == 0:
# continue
#interface = interface[0]
try:
interface.setMtu(intf['mtu'], name, timeout)
interface.setRate(int(intf['speed']) * 1000, name, timeout)
except:
pass #Normal case on virtual interfaces
if 'type' in intf:
if intf['type'] == 'manual' or intf['type'] == 'static':
staticInterface = True
else:
staticInterface = False
ansibleIntf = "ansible_" + intf['name'].replace("-", "_")
if 'ipv4' not in ansible['ansible_facts'][ansibleIntf].keys():
continue
stillExistingAddresses = set(
) # Set of still existing addresses on this host (insalata.model.Layer3Address)
if isinstance(ansible['ansible_facts'][ansibleIntf]['ipv4'], list):
addressesElements = list(
ansible['ansible_facts'][ansibleIntf]['ipv4'])
else: # Single dict
addressesElements = [
ansible['ansible_facts'][ansibleIntf]['ipv4']
]
for addressEl in addressesElements:
netmask = None
try:
interfaceAddress = addressEl['address']
netmask = addressEl['netmask']
except KeyError as e:
logger.error(
"Ansible was not able to detect the {0} on interface {1}"
.format(e.args[0], intf['name']))
continue
gateway = intf['gateway'] if "gateway" in list(
intf.keys()) else None
address = graph.getOrCreateLayer3Address(interfaceAddress,
name,
timeout,
netmask=netmask,
gateway=gateway)
address.setStatic(staticInterface)
address.verify(name, timeout)
interface.addAddress(address)
stillExistingAddresses.add(address)
# Remove old addresses
for old_adr_edge in [
e for e in interface.getEdges()
if isinstance(e.getOther(interface), Layer3Address)
and e.getOther(interface) not in stillExistingAddresses
]:
logger.critical(str(type(old_adr_edge)))
logger.critical(old_adr_edge.getOther(interface).getID())
old_adr_edge.removeVerification(name, timeout)
old_adr_edge.getOther(interface).removeVerification(
name, timeout)
# At the end: Create Layer3networks
for address in stillExistingAddresses:
netmask = address.getNetmask()
if not netmask: # we are not able to determine the network address if no netmask is set => Take /32
netmask = "255.255.255.255"
netAddress = ipAddressHelper.getNetAddress(
address.getID(), netmask)
l3network = graph.getOrCreateLayer3Network(
netAddress + "/" + str(ipAddressHelper.getPrefix(netmask)),
name, timeout, netAddress, netmask)
l3network.verify(name, timeout)
address.setNetwork(l3network)
base.releaseSSHConnection(ssh)
def scan(graph, connectionInfo, logger, thread):
"""
Get all routes from software routers in the network.
Method uses SSH over keys and a host Script on the template to get the routes.
Necessary values in the configuration file of this collector module:
- timeout Timeout this collector module shall use (Integer)
:param graph: Data interface object for this collector module
:type graph: insalata.model.Graph.Graph
:param connectionInfo: Information needed to connect to xen server
:type connectionInfo: dict
:param logger: The logger this scanner shall use
:type logger: logging:Logger
:param thread: Thread executing this collector
:type thread: insalata.scanner.Worker.Worker
"""
logger.info("Collecting routing information.")
timeout = int(connectionInfo['timeout'])
name = connectionInfo['name']
for host in graph.getAllNeighbors(Host):
if not ((host.getPowerState() is None) or
(host.getPowerState() == 'Running')):
continue
ssh = base.getSSHConnection(host)
if ssh is None: #No ssh connecton is possible -> Skip this host
logger.info(
"Skipping host {0} as ssh connection failed in Routing scan.".
format(host.getID()))
continue
hostInterfaces = host.getAllNeighbors(Interface)
currentHostRoutes = set(host.getAllNeighbors(Route))
routingInformation = ssh.getRoutingInfo()
if not routingInformation:
logger.debug("No routing informaiton available for host {}".format(
host.getID()))
continue
routingInfo = ssh.getRoutingInfo()
if not routingInfo:
logger.debug("No routing information (None) for host. '{}'".format(
host.getID()))
continue
for entry in routingInfo:
if "delimiter" in list(entry.keys()) or entry[
'gateway'] == "0.0.0.0": #No routes in directly connected networks should be depicted
continue
interface = [
i for i in hostInterfaces if i.getMAC() == entry['mac']
]
if len(interface) == 0:
logger.error(
"No interface found with mac {0} on host {1}. Maybe this is docker's bridging interface."
.format(entry['mac'], host.getID()))
continue
interface = interface[0]
route = graph.getOrCreateRoute(name,
timeout,
host,
entry['destination'],
entry['genmask'],
entry['gateway'],
interface=None)
currentHostRoutes.discard(route)
host.addRoute(route)
route.setInterface(interface)
for route in currentHostRoutes:
route.removeVerification(name)
base.releaseSSHConnection(ssh)
def scan(graph, connectionInfo, logger, thread):
"""
Get all services using a nmap scan.
Scanning is executed on the host given in the configuration.
Therefore, Nmap must be installed on the scanning device.
Necessary values in the configuration file of this collector module:
- timeout Timeout this collector module shall use (Integer)
- hosts List of network components we shall use to run Nmap. The collector
connects to the network components in the list using ssh and runs athe Nmap scan.
We will not use ssh for 'localhost'.
The used config parser generates a list if the elements are separated by a comma: localhost, myServer as an example
- control_networks (Optional) Json-Array of Layer three Networks we do NOT want to scan using Nmap
- options (Optional) Additional Options we want to use for the Nmap scan
:param graph: Data interface object for this collector module
:type graph: insalata.model.Graph.Graph
:param connectionInfo: Information needed to connect to xen server
:type connectionInfo: dict
:param logger: The logger this scanner shall use
:type logger: logging:Logger
:param thread: Thread executing this collector
:type thread: insalata.scanner.Worker.Worker
"""
timeout = int(connectionInfo['timeout'])
name = connectionInfo['name']
logger.info("Executing nmap service scan.")
if(type(connectionInfo['hosts']) != list):
connectionInfo['hosts'] = [connectionInfo['hosts']]
for hostName in connectionInfo['hosts']:
logger.debug("Executing Nmap on network component: '{}'".format(hostName))
if hostName != "localhost":
scanHost = [h for h in graph.getAllNeighbors(Host) if h.getID() == hostName] #Host executing nmap
if len(scanHost) == 0: # Error handling if nmap host not in graph
logger.error("No host object found for hostname {0}. Nmap scan failed!".format(hostName))
continue
host = scanHost[0]
if not ((host.getPowerState() is None) or (host.getPowerState() == 'Running')):
logger.error("Host {0} is not running. Skipping nmap scan")
continue
ssh = base.getSSHConnection(host)
logger.debug("Execute nmap service scan on host {0}.".format(hostName))
if ssh is None: #No ssh connecton is possible -> Skip this host
logger.info("Skipping host {0} as ssh connection failed.".format(host.getID()))
continue
for networkNode in graph.getAllNeighbors(Layer3Network):
net = networkNode.getAddress() + "/" + str(networkNode.getPrefix())
if "control_networks" in connectionInfo and net in json.loads(connectionInfo["control_networks"]): #Skip this one
logger.debug("Skipping nmap on host {0} for network: {1}.".format(hostName, net))
continue
logger.debug("Executing nmap with additional options '{0}' on host {1} for network: {2}.".format(connectionInfo["options"], hostName, net))
try: # Error handling e.g. if no nmap executable on host
options = connectionInfo['options'] if "options" in connectionInfo else ""
if hostName != "localhost":
scanXml = ssh.executeNmapServiceScan(options, net)
else:
proc = subprocess.Popen("nmap -iL hosts -oX - -sV {}".format(options).split(" "), stdout=subprocess.PIPE)
stdout = proc.stdout
res = stdout.channel.recv_exit_status() # Error handling e.g. if no nmap executable on localhost
if res != 0:
raise OSError(res, "Nmap service scan on localhost failed. Result: {}".format(res))
output = ""
for line in stdout:
output += line
subprocess.Popen("rm hosts")
output = output.replace('encoding="UTF-8"','') # Avoid encoding problems in ElementTree
scanXml = etree.fromstring(output)
except OSError as e:
logger.error("Exit status {1} during nmap scan on host {0}: {2}. Is Nmap installed on the scanning device?".format(host.getID(), e.errno, e.strerror))
continue
for hostXml in scanXml.findall("host"):
for addrXml in hostXml.findall("address"):
if addrXml.attrib["addrtype"] == "mac":
continue
address = addrXml.attrib["addr"]
logger.debug("Found entry for address {0} in nmap scan.".format(address))
addressNode = getAddressNode(networkNode, address)
if addressNode is None:
addressNode = graph.getOrCreateLayer3Address(address, name, timeout)
portsXml = hostXml.find("ports")
for portXml in portsXml.findall("port"):
protocol = portXml.attrib["protocol"]
port = int(portXml.attrib["portid"])
serviceXml = portXml.find("service")
if serviceXml is not None:
if "name" in serviceXml.attrib and serviceXml.attrib["name"] != "unknown":
serviceName = serviceXml.attrib["name"]
logger.debug("Add service '{0}' to address {1}".format(serviceName, address))
serviceNode = None
if serviceName == "domain": #DNS
serviceNode = graph.getOrCreateDnsService(name, timeout, addressNode)
if serviceName == "dhcps":
serviceNode = graph.getOrCreateDhcpService(name, timeout, addressNode)
else:
serviceNode = graph.getOrCreateService(port, protocol, name, timeout, serviceName ,addressNode)
if "product" in serviceXml:
serviceNode.setProduct(serviceXml.attrib["product"]), name, timeout
if "version" in serviceXml:
serviceNode.setVersion(serviceXml["version"], name, timeout)
serviceNode.verify(name, timeout)
addressNode.addService(serviceNode)
addressNode.verify(name, timeout)
for host in [h for h in graph.getAllNeighbors(Host) if h.getID() == hostName.split(".")[0]]:
host.verify(name, timeout)
base.releaseSSHConnection(ssh)
def scan(graph, connectionInfo, logger, thread):
"""
Detect network components using tcpdump.
We will only add addresses as Hosts to the the Graph, if there is no Layer3Address with the same ID!
Necessary values in the configuration file of this collector module:
- timeout Timeout this collector module shall use (Integer)
- monitoringServer The network component we want to start tcpdump on.
If the device is localhost, no ssh connection is used.
If the monitoring server != localhost, we will establish an ssh connection
:param graph: Data interface object for this collector module
:type graph: insalata.model.Graph.Graph
:param connectionInfo: Information needed to connect to xen server
:type connectionInfo: dict
:param logger: The logger this scanner shall use
:type logger: logging:Logger
:param thread: Thread executing this collector
:type thread: insalata.scanner.Worker.Worker
"""
name = connectionInfo["name"]
timeout = int(connectionInfo["timeout"])
monServer = connectionInfo["monitoringServer"]
logger.debug("Starting host scan using tcpdump on monitoring server {0}".format(monServer))
options = "-n -i any not host localhost and not host 127.0.0.1 and not arp and not rarp and not ip6" # By now we do not support IPv6
if monServer != "localhost":
ssh = base.getSSHConnection(monServer)
if ssh is None: #No ssh connecton is possible -> Skip this host
logger.error("No ssh connection to the monitoring server {} is possible in the tcpdump collector module!".format(monServer))
return
stdout = ssh.executeTcpdump(options)
else:
proc = subprocess.Popen("tcpdump {}".format(options).split(" "), stdout=subprocess.PIPE)
stdout = proc.stdout
while(thread.stopRequested):
packet = stdout.readline()
if isinstance(packet, bytes):
packet = packet.decode("ascii")
# Start ssh connection and read output line by line
res = re.search(r"[0-9\:\.]+([^,]+,)? IP ([^ ]+) >", packet)
if not res:
continue
res = res.group(2)
src = ".".join(res.split(".")[:4]) # Source address of the packet
location = graph.getOrCreateLocation("physical", name, timeout)
# We only want to add the address as a host to the graph if no l3address with the ID already exists
# This would lead to an incorrect graph state, as one host is represented multiple times
if len([a for a in graph.getAllNeighbors(Layer3Address) if a.getID() == src]) == 0:
host = graph.getOrCreateHost(src, name, timeout, location=location)
host.setLocation(location)
logger.debug("Tcpdump: Added host to graph: {}".format(host.getID()))
ssh.close()
def scan(graph, connectionInfo, logger, thread):
"""
Get DHCP information from each Host by using SSH and login with key.
Necessary values in the configuration file of this collector module:
- timeout Timeout this collector module shall use (Integer)
:param graph: Data interface object for this collector module
:type graph: insalata.model.Graph.Graph
:param connectionInfo: Information needed to connect to xen server
:type connectionInfo: dict
:param logger: The logger this scanner shall use
:type logger: logging:Logger
:param thread: Thread executing this collector
:type thread: insalata.scanner.Worker.Worker
"""
logger.info("Collection DHCP information")
timeout = int(connectionInfo['timeout'])
name = connectionInfo['name']
for host in graph.getAllNeighbors(Host):
if not ((host.getPowerState() is None) or
(host.getPowerState() == 'Running')):
continue
ssh = base.getSSHConnection(host)
logger.debug("Starting DHCP scan on host: {0}".format(host.getID()))
if ssh is None: #No ssh connecton is possible -> Skip this host
logger.info(
"Skipping host {0} as ssh connection failed in DHCP scan.".
format(host.getID()))
continue
dhcpInformation = ssh.getDHCPInfo()
if not dhcpInformation:
logger.debug("No DHCP information available for host {0}".format(
host.getID()))
continue
hostInterfaces = host.getAllNeighbors(Interface)
for mac in list(dhcpInformation['ranges'].keys()):
if mac == "delimiter":
continue
interface = [i for i in hostInterfaces if i.getMAC() == mac]
if len(interface) == 0:
logger.error(
"No interface found for DHCPInterface with mac {0} on host {1}."
.format(mac, host.getID()))
continue
interface = interface[0]
gateway = None
if mac in (list(dhcpInformation['options'].keys())) and (
"announced_gateway" in list(
dhcpInformation['options'][mac].keys())):
gateway = dhcpInformation['options'][mac]["announced_gateway"]
lease = dhcpInformation['ranges'][mac]['lease']
start = dhcpInformation['ranges'][mac]['from']
end = dhcpInformation['ranges'][mac]['to']
for address in interface.getAllNeighbors(Layer3Address):
service = graph.getOrCreateDhcpService(name, timeout, address)
service.setStartEnd(start, end, name, timeout)
service.setLease(lease, name, timeout)
if gateway:
service.setAnnouncedGateway(gateway, name, timeout)
service.verify(name, timeout)
address.addService(service, name, timeout)
base.releaseSSHConnection(ssh)