def insert_service(service, relation_properties, host_node, external_check):
service_name = service.get('name', 'unknown')
service_node_handle = nlu.get_unique_node_handle(service_name, 'Host Service', 'Logical')
service_node = service_node_handle.get_node()
helpers.update_noclook_auto_manage(service_node)
result = host_node.get_host_service(service_node.handle_id, **relation_properties)
if not result.get('Depends_on'):
result = host_node.set_host_service(service_node.handle_id, **relation_properties)
relationship_id = result.get('Depends_on')[0].get('relationship_id')
relationship = nlu.get_relationship_model(relationship_id)
created = result.get('Depends_on')[0].get('created')
# Set or update relationship properties
relation_properties.update(service)
if external_check:
relation_properties.update({
'public': True,
'noclook_last_external_check': datetime.now().isoformat()
})
user = nlu.get_user()
if created:
activitylog.create_relationship(user, relationship)
#TODO: log creation
if host_node.data.get('services_locked', False):
#TODO: log warning with new port found
relation_properties['rouge_port'] = True
else:
#TODO: log found service
None
helpers.update_noclook_auto_manage(relationship)
#TODO: is properties_keys needed?
helpers.dict_update_relationship(user, relationship.id, relation_properties)
def address_is_a(addresses, node_types):
"""
:param addresses: List of IP addresses
:param node_types: List of acceptable node types
:return: True if the addresses belongs to a host or does not belong to anything
"""
ip_addresses = [ipaddress.ip_address(item) for item in addresses]
for address in addresses:
q = '''
MATCH (n:Node)
USING SCAN n:Node
WHERE any(x IN n.ip_addresses WHERE x =~ {address}) OR n.ip_address =~ {address}
RETURN distinct n
'''
address = '{!s}{!s}'.format(
address, '.*') # Match addresses with / network notation
for hit in nc.query_to_list(nc.graphdb.manager, q, address=address):
node = nc.get_node_model(nc.graphdb.manager, node=hit['n'])
node_addresses = node.data.get('ip_addresses', [])
if not node_addresses and node.data.get('ip_address', None):
node_addresses = [node.data['ip_address']]
for addr in node_addresses:
try:
node_address = ipaddress.ip_address(addr.split('/')[0])
except ValueError:
continue
if node_address in ip_addresses:
if not [
l for l in node.labels
if l.replace(' ', '_') in node_types
]:
helpers.update_noclook_auto_manage(node)
return False
return True
def insert_nmap(json_list, external_check=False):
"""
Inserts the data loaded from the json files created by
the nerds producer nmap_services.
"""
user = nt.get_user()
node_type = "Host"
meta_type = 'Logical'
# Insert the host
for i in json_list:
name = i['host']['name']
logger.info('%s loaded' % name)
addresses = i['host']['nmap_services_py']['addresses']
# Check if the ipaddresses matches any non-host node as a router interface for example
if not is_host(addresses):
logger.info('%s does not appear to be a host.' % name)
continue
# Get or create the NodeHandle and the Node by name, bail if there are more than one match
node_handle = nt.get_unique_node_handle_by_name(
name, node_type, meta_type, ALLOWED_NODE_TYPE_SET)
if not node_handle or node_handle.node_type.type not in ALLOWED_NODE_TYPE_SET:
continue
# Set Node attributes
node = node_handle.get_node()
helpers.update_noclook_auto_manage(node)
properties = {
'hostnames': i['host']['nmap_services_py']['hostnames'],
'ip_addresses': addresses
}
if 'os' in i['host']['nmap_services_py']:
if 'class' in i['host']['nmap_services_py']['os']:
properties['os'] = i['host']['nmap_services_py']['os'][
'class']['osfamily']
properties['os_version'] = i['host']['nmap_services_py']['os'][
'class']['osgen']
elif 'match' in i['host']['nmap_services_py']['os']:
properties['os'] = i['host']['nmap_services_py']['os'][
'match']['name']
if 'uptime' in i['host']['nmap_services_py']:
properties['lastboot'] = i['host']['nmap_services_py']['uptime'][
'lastboot']
properties['uptime'] = i['host']['nmap_services_py']['uptime'][
'seconds']
insert_services(i['host']['nmap_services_py']['services'], node,
external_check)
# Check if the host has backup
properties['backup'] = helpers.get_host_backup(node)
# Set operational state if it is missing
if not node.data.get('operational_state', None):
properties['operational_state'] = 'In service'
# Update host node
helpers.dict_update_node(user, node.handle_id, properties,
properties.keys())
# Set host user depending on the domain.
set_host_user(node)
logger.info('%s done.' % name)
def insert(json_list):
for item in json_list:
base = item['host'].get('raritan')
pdu_handle = utils.get_unique_node_handle_by_name(
item['host']['name'], 'PDU', 'Physical', ['Host', 'PDU'])
pdu_node = pdu_handle.get_node()
helpers.update_noclook_auto_manage(pdu_node)
# If needed add node update with ip/hostnames
insert_ports(base.get('ports'), pdu_node)
def nmap_import(nerds_json, external_check=False):
"""
Inserts the data loaded from a json file created by
the nerds producer nmap_services_py.
"""
item = nerds_json['host']
data = item['nmap_services_py']
addresses = data['addresses']
if not nlu.address_is_a(addresses, ALLOWED_NODE_TYPE_SET):
#Address existed but was not a "Host"
return None
# find or create node
name = item['name']
node_handle = nlu.get_unique_node_handle(name, "Host", "Logical", ALLOWED_NODE_TYPE_SET)
if not node_handle or node_handle.node_type.type not in ALLOWED_NODE_TYPE_SET:
#TODO: log that it is not in ALLOWED_NODE_TYPE_SET
return None
node = node_handle.get_node()
helpers.update_noclook_auto_manage(node)
properties = {
'hostnames': data['hostnames'],
'ip_addresses': addresses
}
if 'os' in data:
if 'class' in data['os']:
os = data['os']['class']
properties['os'] = os['osfamily']
properties['os_version'] = os['osgen']
elif 'match' in data['os']:
os = data['os']['match']
properties['os'] = os['name']
if 'uptime' in data:
properties['lastboot'] = data['uptime']['lastboot']
properties['uptime'] = data['uptime']['seconds']
#handle services
insert_services(data['services'], node, external_check)
properties['backup'] = helpers.get_host_backup(node)
if not node.data.get('operational_state', None):
properties['operational_state'] = 'In service'
user = nlu.get_user()
helpers.dict_update_node(user, node.handle_id, properties)
add_host_user(node)
def insert_hosts(json_list):
user = utils.get_user()
node_type = 'Host'
meta_type = 'Logical'
sunet_user = utils.get_unique_node_handle('SUNET', 'Host User', 'Relation')
for item in json_list:
name = item['host']['name']
data = item['host'].get('nunoc_cosmos', {})
if not address_is_a(data.get('addresses', []), ALLOWED_NODE_TYPE_SET):
logger.info(
'%s had an address that belongs to something that is not a host',
name)
continue
node_handle = utils.get_unique_node_handle_by_name(
name, node_type, meta_type, ALLOWED_NODE_TYPE_SET)
if not node_handle or node_handle.node_type.type not in ALLOWED_NODE_TYPE_SET:
logger.warning("%s is not in %s", name, ALLOWED_NODE_TYPE_SET)
continue
node = node_handle.get_node()
helpers.update_noclook_auto_manage(node)
properties = {
'ip_addresses': data.get('addresses', []),
'sunet_iaas': data.get('sunet_iaas', False)
}
if data.get('managed_by'):
properties['managed_by'] = data.get('managed_by')
# Set operational state if it is missing
if not node.data.get('operational_state',
None) and properties['ip_addresses']:
properties['operational_state'] = 'In service'
# Update host node
helpers.dict_update_node(user, node.handle_id, properties)
if data.get('sunet_iaas', False):
if node.meta_type == 'Logical':
helpers.set_user(user, node, sunet_user.handle_id)
elif node.meta_type == 'Physical':
helpers.set_owner(user, node, sunet_user.handle_id)
logger.info("%s has been imported", name)
def insert(json_list):
# Setup persistent storage for collections done over multiple hosts
netapp_collection = getattr(django_settings, 'NETAPP_REPORT_SETTINGS', [])
# Parse collected Nagios data
for item in json_list:
base = item['host'].get('checkmk_livestatus')
if not base:
base = item['host']['nagiosxi_api']
base['host_name'] = base['host_name'].lower()
host = nc.get_unique_node_by_name(nc.graphdb.manager,
base['host_name'], 'Host')
if not host:
host = get_host(base['host_address'])
if host:
check_descriptions = []
for check in base['checks']:
check_descriptions.append(
check.get('description', 'Missing description'))
if check[
'check_command'] == 'CHECK_NRPE!check_uptime': # Host uptime
set_uptime(host, check)
if check[
'check_command'] == 'CHECK_NRPE!check_backup': # TSM backup process
set_backup(host, check)
if check['check_command'].startswith(
'check_netapp_vol'): # NetApp storage usage
netapp_collection = collect_netapp_storage_usage(
host, check, netapp_collection)
if check[
'check_command'] == 'CHECK_NRPE!check_openmanage': # Dell OpenManage info
set_dell_service_tag(host, check)
if check['check_command'] == 'check_esxi': # Dell esxi HW info
set_dell_service_tag(host, check)
set_nagios_checks(host, check_descriptions)
helpers.update_noclook_auto_manage(host)
logger.info('{name} done.'.format(name=host.data['name']))
# Set data collected from multiple hosts
set_netapp_storage_usage(netapp_collection)
def insert_services(service_dict, host_node, external_check=False):
"""
Takes a dictionary of services and a node id for a host. Gets or creates a
service and makes a Depends_on relationship between the service and host.
Example service_dict:
{"127.0.0.1": {
"tcp": {
"80": {
"conf": "10",
"extrainfo": "",
"name": "http",
"product": "VMware ESXi Server httpd",
"reason": "syn-ack",
"state": "open",
"version": ""
},
"8443": {
"conf": "10",
"extrainfo": "",
"name": "ssl",
"product": "TLS",
"reason": "syn-ack",
"state": "open",
"version": "1.0"
}
}
}
"""
user = nt.get_user()
node_type = "Host Service"
meta_type = 'Logical'
services_locked = host_node.data.get('services_locked', False)
# Expected service data from nmap
property_keys = [
'ip_address', 'protocol', 'port', 'conf', 'extrainfo', 'name',
'product', 'reason', 'state', 'version'
]
if external_check:
property_keys.extend(['public', 'noclook_last_external_check'])
external_dict = {
'public': True,
'noclook_last_external_check': datetime.now().isoformat()
}
set_not_public(host_node)
for address in service_dict.keys():
for protocol in service_dict[address].keys():
for port in service_dict[address][protocol].keys():
service = service_dict[address][protocol][port]
if service['state'] != 'closed':
service_name = service['name']
if not service_name: # Blank
service_name = 'unknown'
service_node_handle = nt.get_unique_node_handle(
service_name, node_type, meta_type)
service_node = service_node_handle.get_node()
helpers.update_noclook_auto_manage(service_node)
relationship_properties = {
'ip_address': address,
'protocol': protocol,
'port': port
}
result = host_node.get_host_service(
service_node.handle_id, **relationship_properties)
if not result.get('Depends_on'):
result = host_node.set_host_service(
service_node.handle_id, **relationship_properties)
relationship_id = result.get('Depends_on')[0].get(
'relationship_id')
relationship = nc.get_relationship_model(
nc.graphdb.manager, relationship_id)
created = result.get('Depends_on')[0].get('created')
# Set or update relationship properties
relationship_properties.update(service)
if external_check:
relationship_properties.update(external_dict)
if created:
activitylog.create_relationship(user, relationship)
if services_locked:
logger.warn(
'New open port found for host {name}.'.format(
name=host_node.data['name']))
property_keys.append('rogue_port')
relationship_properties['rogue_port'] = True
logger.info(
'Host Service {host_service_name} using port {port}/{protocol} created.'
.format(
host_service_name=service_node.data['name'],
port=relationship.data['port'],
protocol=relationship.data['protocol']))
if not created:
logger.info(
'Host Service {host_service_name} using port {port}/{protocol} found.'
.format(
host_service_name=service_node.data['name'],
port=relationship.data['port'],
protocol=relationship.data['protocol']))
helpers.update_noclook_auto_manage(relationship)
helpers.dict_update_relationship(user, relationship.id,
relationship_properties,
property_keys)
logger.info(
'{name} {ip_address} {port}/{protocol} processed...'.
format(name=host_node.data['name'],
ip_address=address,
protocol=protocol,
port=port))
def insert_snap(json_list):
"""
Inserts snap metadata as Hosts.
"""
user = utils.get_user()
node_type = "Host"
for data in json_list:
# Handle nerds data
try:
d = data['host']['snap_metadata']
except KeyError:
d = data
name = d['hostname'].lower()
logger.info('{} loaded'.format(name))
meta_type = 'Logical' if d.get('virtual') else 'Physical'
# find host first hostname, then ip? else create
node_handle = utils.get_unique_node_handle_by_name(
name, node_type, meta_type, ALLOWED_NODE_TYPE_SET)
# Check it is a host
if not node_handle or node_handle.node_type.type not in ALLOWED_NODE_TYPE_SET:
logger.info('{} is not a {} skipping.'.format(
name, ALLOWED_NODE_TYPE_SET))
continue
# Update host
node = node_handle.get_node()
# change from logical to physical if needed?
if node_handle.node_meta_type == 'Logical' and not d.get('virtual'):
logger.warning(
'Converting {} from logical to physical'.format(name))
helpers.logical_to_physical(user, node.handle_id)
helpers.update_noclook_auto_manage(node)
managed_by = 'Puppet' if d.get('managed') else 'Manual'
responsible = d.get('responsible', 'SEI')
properties = {
'os': d['os'],
'managed_by': managed_by,
'responsible_group': responsible,
'description': d.get('description')
}
if d.get('network'):
ipv4 = [
n['ip'].split('/')[0] for n in d.get('network', [])
if 'ip' in n
]
ipv4_service = [
n['service_ip'].split('/')[0] for n in d.get('network', [])
if 'service_ip' in n
]
ipv6 = [
n['ipv6'].split('/')[0] for n in d.get('network', [])
if 'ipv6' in n
]
ipv6_service = [
n['service_ipv6'].split('/')[0] for n in d.get('network', [])
if 'service_ipv6' in n
]
properties[
'ip_addresses'] = ipv4 + ipv4_service + ipv6 + ipv6_service
if d.get('managed'):
# Dont default to False
properties['syslog'] = True
helpers.dict_update_node(user, node.handle_id, properties,
properties.keys())
logger.info('{} has been imported'.format(name))
