def get_unique_node(name, node_type, meta_type): """ Gets or creates a NodeHandle with the provided name. Returns the NodeHandles node. """ name = normalize_whitespace(name) node_handle = utils.get_unique_node_handle(name, node_type, meta_type) node = node_handle.get_node() return node
def insert_site(site_dict): """ :param site_dict: data :type site_dict: dict :return: None :rtype: None Expected dict { u'name': u'', u'area': u'', u'country': u'', u'longitude': u'', u'node_type': u'Site', u'meta_type': u'Location', u'site_owner': u'', u'latitude': u'', u'site_type': u'' } """ user = utils.get_user() # Remove meta data and relationship data from the site dict name = site_dict.pop('name') node_type = site_dict.pop('node_type') meta_type = site_dict.pop('meta_type') site_owner = site_dict.pop('site_owner') # Get or create Site site_nh = utils.get_unique_node_handle(name, node_type, meta_type) # Set or update node properties helpers.dict_update_node(user, site_nh.handle_id, site_dict, site_dict.keys()) if site_owner: # Get or create Site owner site_owner_nh = utils.get_unique_node_handle(site_owner, 'Site Owner', 'Relation') # Set relationship to site owner helpers.set_responsible_for(user, site_nh.get_node(), site_owner_nh.handle_id) logger.info(u'Imported site {}.'.format(name))
def set_host_user(host): """ Tries to set a Uses or Owns relationship between the Host and a Host User if there are none. """ user = utils.get_user() domain = '.'.join(host.data['name'].split('.')[-2:]) relations = host.get_relations() host_user_name = HOST_USERS_MAP.get(domain, None) if host_user_name and not (relations.get('Uses', None) or relations.get('Owns', None)): relation_node_handle = utils.get_unique_node_handle(host_user_name, 'Host User', 'Relation') if host.meta_type == 'Logical': helpers.set_user(user, host, relation_node_handle.handle_id) elif host.meta_type == 'Physical': helpers.set_owner(user, host, relation_node_handle.handle_id) logger.info('Host User {user_name} set for host {host_name}.'.format(user_name=host_user_name, host_name=host.data['name']))
def insert_juniper_bgp_peerings(bgp_peerings): """ Inserts all BGP peerings for all routers collected by the juniper_conf producer. This is to be able to get all the peerings associated to the right interfaces. """ for peering in bgp_peerings: peering_group = peering.get('group', 'Unknown Peering Group') peering_group_handle = utils.get_unique_node_handle( peering_group, 'Peering Group', 'Logical', case_insensitive=False) peering_group_node = peering_group_handle.get_node() helpers.set_noclook_auto_manage(peering_group_node, True) peering_type = peering.get('type') if peering_type == 'internal': continue # Not implemented elif peering_type == 'external': insert_external_bgp_peering(peering, peering_group_node)
def insert_juniper_node(name, model, version, node_type='Router', hardware=None): """ Inserts a physical meta type node of the type Router. Returns the node created. """ logger.info('Processing {name}...'.format(name=name)) user = utils.get_user() node_handle = utils.get_unique_node_handle(name, node_type, 'Physical') node = node_handle.get_node() node_dict = {'name': name, 'model': model, 'version': version} if hardware: node_dict['serial_number'] = hardware.get('serial_number') helpers.dict_update_node(user, node.handle_id, node_dict, node_dict.keys()) helpers.set_noclook_auto_manage(node, True) return node
def insert_hosts(json_list): user = utils.get_user() node_type = 'Host' meta_type = 'Logical' sunet_user = utils.get_unique_node_handle('SUNET', 'Host User', 'Relation') for item in json_list: name = item['host']['name'] data = item['host'].get('nunoc_cosmos', {}) if not address_is_a(data.get('addresses', []), ALLOWED_NODE_TYPE_SET): logger.info( '%s had an address that belongs to something that is not a host', name) continue node_handle = utils.get_unique_node_handle_by_name( name, node_type, meta_type, ALLOWED_NODE_TYPE_SET) if not node_handle or node_handle.node_type.type not in ALLOWED_NODE_TYPE_SET: logger.warning("%s is not in %s", name, ALLOWED_NODE_TYPE_SET) continue node = node_handle.get_node() helpers.update_noclook_auto_manage(node) properties = { 'ip_addresses': data.get('addresses', []), 'sunet_iaas': data.get('sunet_iaas', False) } if data.get('managed_by'): properties['managed_by'] = data.get('managed_by') # Set operational state if it is missing if not node.data.get('operational_state', None) and properties['ip_addresses']: properties['operational_state'] = 'In service' # Update host node helpers.dict_update_node(user, node.handle_id, properties) if data.get('sunet_iaas', False): if node.meta_type == 'Logical': helpers.set_user(user, node, sunet_user.handle_id) elif node.meta_type == 'Physical': helpers.set_owner(user, node, sunet_user.handle_id) logger.info("%s has been imported", name)
def insert_services(service_dict, host_node, external_check=False): """ Takes a dictionary of services and a node id for a host. Gets or creates a service and makes a Depends_on relationship between the service and host. Example service_dict: {"127.0.0.1": { "tcp": { "80": { "conf": "10", "extrainfo": "", "name": "http", "product": "VMware ESXi Server httpd", "reason": "syn-ack", "state": "open", "version": "" }, "8443": { "conf": "10", "extrainfo": "", "name": "ssl", "product": "TLS", "reason": "syn-ack", "state": "open", "version": "1.0" } } } """ user = utils.get_user() node_type = "Host Service" meta_type = 'Logical' services_locked = host_node.data.get('services_locked', False) # Expected service data from nmap property_keys = [ 'ip_address', 'protocol', 'port', 'conf', 'name', 'product', 'state', ] if external_check: property_keys.extend(['public', 'noclook_last_external_check']) external_dict = { 'public': True, 'noclook_last_external_check': datetime.now().isoformat() } set_not_public(host_node) for address in service_dict.keys(): for protocol in service_dict[address].keys(): for port in service_dict[address][protocol].keys(): service = service_dict[address][protocol][port] # Ignore anything but open if service['state'] == 'open': service_name = service['name'] if not service_name: # Blank logger.warn( 'Skipping unknown service on port: %s (%s), address: %s, hostname: %s, data: %s', port, protocol, address, host_node.data.get('name'), service) continue service_node_handle = utils.get_unique_node_handle( service_name, node_type, meta_type) service_node = service_node_handle.get_node() helpers.update_noclook_auto_manage(service_node) relationship_properties = { 'ip_address': address, 'protocol': protocol, 'port': port } result = host_node.get_host_service( service_node.handle_id, **relationship_properties) if not result.get('Depends_on'): result = host_node.set_host_service( service_node.handle_id, **relationship_properties) relationship_id = result.get('Depends_on')[0].get( 'relationship_id') relationship = nc.get_relationship_model( nc.graphdb.manager, relationship_id) created = result.get('Depends_on')[0].get('created') # Set or update relationship properties relationship_properties.update(service) if external_check: relationship_properties.update(external_dict) if created: activitylog.create_relationship(user, relationship) if services_locked: logger.warn( 'New open port found for host {name}.'.format( name=host_node.data['name'])) property_keys.append('rogue_port') relationship_properties['rogue_port'] = True logger.info( 'Host Service {host_service_name} using port {port}/{protocol} created.' .format( host_service_name=service_node.data['name'], port=relationship.data['port'], protocol=relationship.data['protocol'])) if not created: logger.info( 'Host Service {host_service_name} using port {port}/{protocol} found.' .format( host_service_name=service_node.data['name'], port=relationship.data['port'], protocol=relationship.data['protocol'])) helpers.update_noclook_auto_manage(relationship) helpers.dict_update_relationship(user, relationship.id, relationship_properties, property_keys) logger.info( '{name} {ip_address} {port}/{protocol} processed...'. format(name=host_node.data['name'], ip_address=address, protocol=protocol, port=port))
def get_peering_partner(peering): """ Inserts a new node of the type Peering partner and ensures that this node is unique for AS number. Returns the created node. """ try: return PEER_AS_CACHE[peering['as_number']] except KeyError: logger.info('Peering Partner {name} not in cache.'.format( name=peering.get('description'))) pass user = utils.get_user() peer_node = None peer_properties = {'name': 'Missing description', 'as_number': '0'} # neither description or as_number error and return if not (peering.get('description') or peering.get('as_number')): logger.error('Neither AS number nor description in peering %s', peering) return None if peering.get('description'): peer_properties['name'] = peering.get('description') if peering.get('as_number'): peer_properties['as_number'] = peering.get('as_number') # as number is most important hits = nc.get_nodes_by_value(nc.graphdb.manager, prop='as_number', value=peer_properties['as_number']) found = 0 for node in hits: peer_node = nc.get_node_model(nc.graphdb.manager, node['handle_id']) helpers.set_noclook_auto_manage(peer_node, True) if peer_node.data[ 'name'] == 'Missing description' and peer_properties[ 'name'] != 'Missing description': helpers.dict_update_node(user, peer_node.handle_id, peer_properties) logger.info('Peering Partner {name} fetched.'.format( name=peer_properties['name'])) found += 1 if found > 1: logger.error( 'Found more then one Peering Partner with AS number {!s}'. format(peer_properties['as_number'])) if not peer_node: # since we have a AS number we will create a new Peering Partner, even if name is missing or exists node_handle = utils.create_node_handle(peer_properties['name'], 'Peering Partner', 'Relation') peer_node = node_handle.get_node() helpers.set_noclook_auto_manage(peer_node, True) helpers.dict_update_node(user, peer_node.handle_id, peer_properties, peer_properties.keys()) logger.info('Peering Partner %s AS(%s) created.', peer_properties['name'], peer_properties['as_number']) # Handle peer with name only if not peer_node and peering.get('description'): # Try and get peer_partners res = NodeHandle.objects.filter( node_name__iexact=peer_properties['name'], node_type__type='Peering Partner').order_by('-modified') for ph in res: peer_node = ph.get_node() break if not peer_node: # create peer_nh = utils.get_unique_node_handle(peer_properties['name'], 'Peering Partner', 'Relation') peer_node = peer_nh.get_node() if not peer_node.data.get('as_number'): # Peering partner did not exist logger.warning( 'Peering Partner %s without AS number created for peering: %s', peer_node.data.get('name'), peering) # AS number is going to be 0, but that is ok helpers.dict_update_node(user, peer_node.handle_id, peer_properties, peer_properties.keys()) elif peer_node.data.get('as_number') != '0': # warn about as number not being in peering logger.warning( 'Peering found for Peering Partner %s without the AS number %s mentioned. Peering: %s', peer_properties['name'], peer_node.data.get('as_number'), peering) helpers.set_noclook_auto_manage(peer_node, True) PEER_AS_CACHE[peering['as_number']] = peer_node return peer_node