def insert_service(service, relation_properties, host_node, external_check): service_name = service.get('name', 'unknown') service_node_handle = nlu.get_unique_node_handle(service_name, 'Host Service', 'Logical') service_node = service_node_handle.get_node() helpers.update_noclook_auto_manage(service_node) result = host_node.get_host_service(service_node.handle_id, **relation_properties) if not result.get('Depends_on'): result = host_node.set_host_service(service_node.handle_id, **relation_properties) relationship_id = result.get('Depends_on')[0].get('relationship_id') relationship = nlu.get_relationship_model(relationship_id) created = result.get('Depends_on')[0].get('created') # Set or update relationship properties relation_properties.update(service) if external_check: relation_properties.update({ 'public': True, 'noclook_last_external_check': datetime.now().isoformat() }) user = nlu.get_user() if created: activitylog.create_relationship(user, relationship) #TODO: log creation if host_node.data.get('services_locked', False): #TODO: log warning with new port found relation_properties['rouge_port'] = True else: #TODO: log found service None helpers.update_noclook_auto_manage(relationship) #TODO: is properties_keys needed? helpers.dict_update_relationship(user, relationship.id, relation_properties)
def address_is_a(addresses, node_types): """ :param addresses: List of IP addresses :param node_types: List of acceptable node types :return: True if the addresses belongs to a host or does not belong to anything """ ip_addresses = [ipaddress.ip_address(item) for item in addresses] for address in addresses: q = ''' MATCH (n:Node) USING SCAN n:Node WHERE any(x IN n.ip_addresses WHERE x =~ {address}) OR n.ip_address =~ {address} RETURN distinct n ''' address = '{!s}{!s}'.format( address, '.*') # Match addresses with / network notation for hit in nc.query_to_list(nc.graphdb.manager, q, address=address): node = nc.get_node_model(nc.graphdb.manager, node=hit['n']) node_addresses = node.data.get('ip_addresses', []) if not node_addresses and node.data.get('ip_address', None): node_addresses = [node.data['ip_address']] for addr in node_addresses: try: node_address = ipaddress.ip_address(addr.split('/')[0]) except ValueError: continue if node_address in ip_addresses: if not [ l for l in node.labels if l.replace(' ', '_') in node_types ]: helpers.update_noclook_auto_manage(node) return False return True
def insert_nmap(json_list, external_check=False): """ Inserts the data loaded from the json files created by the nerds producer nmap_services. """ user = nt.get_user() node_type = "Host" meta_type = 'Logical' # Insert the host for i in json_list: name = i['host']['name'] logger.info('%s loaded' % name) addresses = i['host']['nmap_services_py']['addresses'] # Check if the ipaddresses matches any non-host node as a router interface for example if not is_host(addresses): logger.info('%s does not appear to be a host.' % name) continue # Get or create the NodeHandle and the Node by name, bail if there are more than one match node_handle = nt.get_unique_node_handle_by_name( name, node_type, meta_type, ALLOWED_NODE_TYPE_SET) if not node_handle or node_handle.node_type.type not in ALLOWED_NODE_TYPE_SET: continue # Set Node attributes node = node_handle.get_node() helpers.update_noclook_auto_manage(node) properties = { 'hostnames': i['host']['nmap_services_py']['hostnames'], 'ip_addresses': addresses } if 'os' in i['host']['nmap_services_py']: if 'class' in i['host']['nmap_services_py']['os']: properties['os'] = i['host']['nmap_services_py']['os'][ 'class']['osfamily'] properties['os_version'] = i['host']['nmap_services_py']['os'][ 'class']['osgen'] elif 'match' in i['host']['nmap_services_py']['os']: properties['os'] = i['host']['nmap_services_py']['os'][ 'match']['name'] if 'uptime' in i['host']['nmap_services_py']: properties['lastboot'] = i['host']['nmap_services_py']['uptime'][ 'lastboot'] properties['uptime'] = i['host']['nmap_services_py']['uptime'][ 'seconds'] insert_services(i['host']['nmap_services_py']['services'], node, external_check) # Check if the host has backup properties['backup'] = helpers.get_host_backup(node) # Set operational state if it is missing if not node.data.get('operational_state', None): properties['operational_state'] = 'In service' # Update host node helpers.dict_update_node(user, node.handle_id, properties, properties.keys()) # Set host user depending on the domain. set_host_user(node) logger.info('%s done.' % name)
def insert(json_list): for item in json_list: base = item['host'].get('raritan') pdu_handle = utils.get_unique_node_handle_by_name( item['host']['name'], 'PDU', 'Physical', ['Host', 'PDU']) pdu_node = pdu_handle.get_node() helpers.update_noclook_auto_manage(pdu_node) # If needed add node update with ip/hostnames insert_ports(base.get('ports'), pdu_node)
def nmap_import(nerds_json, external_check=False): """ Inserts the data loaded from a json file created by the nerds producer nmap_services_py. """ item = nerds_json['host'] data = item['nmap_services_py'] addresses = data['addresses'] if not nlu.address_is_a(addresses, ALLOWED_NODE_TYPE_SET): #Address existed but was not a "Host" return None # find or create node name = item['name'] node_handle = nlu.get_unique_node_handle(name, "Host", "Logical", ALLOWED_NODE_TYPE_SET) if not node_handle or node_handle.node_type.type not in ALLOWED_NODE_TYPE_SET: #TODO: log that it is not in ALLOWED_NODE_TYPE_SET return None node = node_handle.get_node() helpers.update_noclook_auto_manage(node) properties = { 'hostnames': data['hostnames'], 'ip_addresses': addresses } if 'os' in data: if 'class' in data['os']: os = data['os']['class'] properties['os'] = os['osfamily'] properties['os_version'] = os['osgen'] elif 'match' in data['os']: os = data['os']['match'] properties['os'] = os['name'] if 'uptime' in data: properties['lastboot'] = data['uptime']['lastboot'] properties['uptime'] = data['uptime']['seconds'] #handle services insert_services(data['services'], node, external_check) properties['backup'] = helpers.get_host_backup(node) if not node.data.get('operational_state', None): properties['operational_state'] = 'In service' user = nlu.get_user() helpers.dict_update_node(user, node.handle_id, properties) add_host_user(node)
def insert_hosts(json_list): user = utils.get_user() node_type = 'Host' meta_type = 'Logical' sunet_user = utils.get_unique_node_handle('SUNET', 'Host User', 'Relation') for item in json_list: name = item['host']['name'] data = item['host'].get('nunoc_cosmos', {}) if not address_is_a(data.get('addresses', []), ALLOWED_NODE_TYPE_SET): logger.info( '%s had an address that belongs to something that is not a host', name) continue node_handle = utils.get_unique_node_handle_by_name( name, node_type, meta_type, ALLOWED_NODE_TYPE_SET) if not node_handle or node_handle.node_type.type not in ALLOWED_NODE_TYPE_SET: logger.warning("%s is not in %s", name, ALLOWED_NODE_TYPE_SET) continue node = node_handle.get_node() helpers.update_noclook_auto_manage(node) properties = { 'ip_addresses': data.get('addresses', []), 'sunet_iaas': data.get('sunet_iaas', False) } if data.get('managed_by'): properties['managed_by'] = data.get('managed_by') # Set operational state if it is missing if not node.data.get('operational_state', None) and properties['ip_addresses']: properties['operational_state'] = 'In service' # Update host node helpers.dict_update_node(user, node.handle_id, properties) if data.get('sunet_iaas', False): if node.meta_type == 'Logical': helpers.set_user(user, node, sunet_user.handle_id) elif node.meta_type == 'Physical': helpers.set_owner(user, node, sunet_user.handle_id) logger.info("%s has been imported", name)
def insert(json_list): # Setup persistent storage for collections done over multiple hosts netapp_collection = getattr(django_settings, 'NETAPP_REPORT_SETTINGS', []) # Parse collected Nagios data for item in json_list: base = item['host'].get('checkmk_livestatus') if not base: base = item['host']['nagiosxi_api'] base['host_name'] = base['host_name'].lower() host = nc.get_unique_node_by_name(nc.graphdb.manager, base['host_name'], 'Host') if not host: host = get_host(base['host_address']) if host: check_descriptions = [] for check in base['checks']: check_descriptions.append( check.get('description', 'Missing description')) if check[ 'check_command'] == 'CHECK_NRPE!check_uptime': # Host uptime set_uptime(host, check) if check[ 'check_command'] == 'CHECK_NRPE!check_backup': # TSM backup process set_backup(host, check) if check['check_command'].startswith( 'check_netapp_vol'): # NetApp storage usage netapp_collection = collect_netapp_storage_usage( host, check, netapp_collection) if check[ 'check_command'] == 'CHECK_NRPE!check_openmanage': # Dell OpenManage info set_dell_service_tag(host, check) if check['check_command'] == 'check_esxi': # Dell esxi HW info set_dell_service_tag(host, check) set_nagios_checks(host, check_descriptions) helpers.update_noclook_auto_manage(host) logger.info('{name} done.'.format(name=host.data['name'])) # Set data collected from multiple hosts set_netapp_storage_usage(netapp_collection)
def insert_services(service_dict, host_node, external_check=False): """ Takes a dictionary of services and a node id for a host. Gets or creates a service and makes a Depends_on relationship between the service and host. Example service_dict: {"127.0.0.1": { "tcp": { "80": { "conf": "10", "extrainfo": "", "name": "http", "product": "VMware ESXi Server httpd", "reason": "syn-ack", "state": "open", "version": "" }, "8443": { "conf": "10", "extrainfo": "", "name": "ssl", "product": "TLS", "reason": "syn-ack", "state": "open", "version": "1.0" } } } """ user = nt.get_user() node_type = "Host Service" meta_type = 'Logical' services_locked = host_node.data.get('services_locked', False) # Expected service data from nmap property_keys = [ 'ip_address', 'protocol', 'port', 'conf', 'extrainfo', 'name', 'product', 'reason', 'state', 'version' ] if external_check: property_keys.extend(['public', 'noclook_last_external_check']) external_dict = { 'public': True, 'noclook_last_external_check': datetime.now().isoformat() } set_not_public(host_node) for address in service_dict.keys(): for protocol in service_dict[address].keys(): for port in service_dict[address][protocol].keys(): service = service_dict[address][protocol][port] if service['state'] != 'closed': service_name = service['name'] if not service_name: # Blank service_name = 'unknown' service_node_handle = nt.get_unique_node_handle( service_name, node_type, meta_type) service_node = service_node_handle.get_node() helpers.update_noclook_auto_manage(service_node) relationship_properties = { 'ip_address': address, 'protocol': protocol, 'port': port } result = host_node.get_host_service( service_node.handle_id, **relationship_properties) if not result.get('Depends_on'): result = host_node.set_host_service( service_node.handle_id, **relationship_properties) relationship_id = result.get('Depends_on')[0].get( 'relationship_id') relationship = nc.get_relationship_model( nc.graphdb.manager, relationship_id) created = result.get('Depends_on')[0].get('created') # Set or update relationship properties relationship_properties.update(service) if external_check: relationship_properties.update(external_dict) if created: activitylog.create_relationship(user, relationship) if services_locked: logger.warn( 'New open port found for host {name}.'.format( name=host_node.data['name'])) property_keys.append('rogue_port') relationship_properties['rogue_port'] = True logger.info( 'Host Service {host_service_name} using port {port}/{protocol} created.' .format( host_service_name=service_node.data['name'], port=relationship.data['port'], protocol=relationship.data['protocol'])) if not created: logger.info( 'Host Service {host_service_name} using port {port}/{protocol} found.' .format( host_service_name=service_node.data['name'], port=relationship.data['port'], protocol=relationship.data['protocol'])) helpers.update_noclook_auto_manage(relationship) helpers.dict_update_relationship(user, relationship.id, relationship_properties, property_keys) logger.info( '{name} {ip_address} {port}/{protocol} processed...'. format(name=host_node.data['name'], ip_address=address, protocol=protocol, port=port))
def insert_snap(json_list): """ Inserts snap metadata as Hosts. """ user = utils.get_user() node_type = "Host" for data in json_list: # Handle nerds data try: d = data['host']['snap_metadata'] except KeyError: d = data name = d['hostname'].lower() logger.info('{} loaded'.format(name)) meta_type = 'Logical' if d.get('virtual') else 'Physical' # find host first hostname, then ip? else create node_handle = utils.get_unique_node_handle_by_name( name, node_type, meta_type, ALLOWED_NODE_TYPE_SET) # Check it is a host if not node_handle or node_handle.node_type.type not in ALLOWED_NODE_TYPE_SET: logger.info('{} is not a {} skipping.'.format( name, ALLOWED_NODE_TYPE_SET)) continue # Update host node = node_handle.get_node() # change from logical to physical if needed? if node_handle.node_meta_type == 'Logical' and not d.get('virtual'): logger.warning( 'Converting {} from logical to physical'.format(name)) helpers.logical_to_physical(user, node.handle_id) helpers.update_noclook_auto_manage(node) managed_by = 'Puppet' if d.get('managed') else 'Manual' responsible = d.get('responsible', 'SEI') properties = { 'os': d['os'], 'managed_by': managed_by, 'responsible_group': responsible, 'description': d.get('description') } if d.get('network'): ipv4 = [ n['ip'].split('/')[0] for n in d.get('network', []) if 'ip' in n ] ipv4_service = [ n['service_ip'].split('/')[0] for n in d.get('network', []) if 'service_ip' in n ] ipv6 = [ n['ipv6'].split('/')[0] for n in d.get('network', []) if 'ipv6' in n ] ipv6_service = [ n['service_ipv6'].split('/')[0] for n in d.get('network', []) if 'service_ipv6' in n ] properties[ 'ip_addresses'] = ipv4 + ipv4_service + ipv6 + ipv6_service if d.get('managed'): # Dont default to False properties['syslog'] = True helpers.dict_update_node(user, node.handle_id, properties, properties.keys()) logger.info('{} has been imported'.format(name))