def remove_router_conf(user, data_age): routerq = """ MATCH (router:Node:Router) OPTIONAL MATCH (router)-[:Has*1..]->(physical)<-[:Part_of]-(logical) WHERE (physical.noclook_auto_manage = true) OR (logical.noclook_auto_manage = true) RETURN collect(distinct physical.handle_id) as physical, collect(distinct logical.handle_id) as logical """ router_result = nc.query_to_dict(nc.graphdb.manager, routerq) for handle_id in router_result.get('logical', []): logical = nc.get_node_model(nc.graphdb.manager, handle_id) if logical: last_seen, expired = helpers.neo4j_data_age(logical.data, data_age) if expired: helpers.delete_node(user, logical.handle_id) logger.warning('Deleted logical router: %s (%s).', logical.data.get('name'), handle_id) for handle_id in router_result.get('physical', []): physical = nc.get_node_model(nc.graphdb.manager, handle_id) if physical: last_seen, expired = helpers.neo4j_data_age( physical.data, data_age) if expired: helpers.delete_node(user, physical.handle_id) logger.warning('Deleted physical router: %s (%s).', physical.data.get('name'), handle_id)
def remove_peer_conf(user, data_age): peerq = """ MATCH (peer_group:Node:Peering_Group) MATCH (peer_group)<-[r:Uses]-(peering_partner:Peering_Partner) WHERE (peer_group.noclook_auto_manage = true) OR (r.noclook_auto_manage = true) RETURN collect(distinct peer_group.handle_id) as peer_groups, collect(id(r)) as uses_relationships """ peer_result = nc.query_to_dict(nc.graphdb.manager, peerq) for relationship_id in peer_result.get('uses_relationships', []): relationship = nc.get_relationship_model(nc.graphdb.manager, relationship_id) if relationship: last_seen, expired = helpers.neo4j_data_age( relationship.data, data_age) if expired: rel_info = helpers.relationship_to_str(relationship) helpers.delete_relationship(user, relationship.id) logger.warning('Deleted relationship {rel_info}'.format( rel_info=rel_info)) for handle_id in peer_result.get('peer_groups', []): peer_group = nc.get_node_model(nc.graphdb.manager, handle_id) if peer_group: last_seen, expired = helpers.neo4j_data_age( peer_group.data, data_age) if expired: helpers.delete_node(user, peer_group.handle_id) logger.warning('Deleted node {name} ({handle_id}).'.format( name=peer_group.data.get('name'), handle_id=handle_id))
def address_is_a(addresses, node_types): """ :param addresses: List of IP addresses :param node_types: List of acceptable node types :return: True if the addresses belongs to a host or does not belong to anything """ ip_addresses = [ipaddress.ip_address(item) for item in addresses] for address in addresses: q = ''' MATCH (n:Node) USING SCAN n:Node WHERE any(x IN n.ip_addresses WHERE x =~ {address}) OR n.ip_address =~ {address} RETURN distinct n ''' address = '{!s}{!s}'.format( address, '.*') # Match addresses with / network notation for hit in nc.query_to_list(nc.graphdb.manager, q, address=address): node = nc.get_node_model(nc.graphdb.manager, node=hit['n']) node_addresses = node.data.get('ip_addresses', []) if not node_addresses and node.data.get('ip_address', None): node_addresses = [node.data['ip_address']] for addr in node_addresses: try: node_address = ipaddress.ip_address(addr.split('/')[0]) except ValueError: continue if node_address in ip_addresses: if not [ l for l in node.labels if l.replace(' ', '_') in node_types ]: helpers.update_noclook_auto_manage(node) return False return True
def get_node_type(handle_id): model = nc.get_node_model(nc.graphdb.manager, handle_id) for t in model.labels: try: return NodeType.objects.get(type=t.replace('_', ' ')).type except NodeType.DoesNotExist: pass
def get_nh_node(handle_id): """ Takes a node handle id and returns the node handle and the node model. """ node_handle = get_object_or_404(NodeHandle, pk=handle_id) node_model = nc.get_node_model(nc.graphdb.manager, node_handle.handle_id) return node_handle, node_model
def host_detail(request, handle_id): nh = get_object_or_404(NodeHandle, pk=handle_id) # Get node from neo4j-database host = nc.get_node_model(nc.graphdb.manager, nh.handle_id) last_seen, expired = helpers.neo4j_data_age(host.data) location_path = host.get_location_path() # Handle relationships host_services = host.get_host_services() relations = host.get_relations() dependent = host.get_dependent_as_types() # Get ports in Host connections = host.get_connections() if not any(dependent.values()): dependent = None dependencies = host.get_dependencies_as_types() urls = helpers.get_node_urls(relations, host_services, dependent, dependencies) scan_enabled = helpers.app_enabled("apps.scan") return render( request, 'noclook/detail/host_detail.html', { 'node_handle': nh, 'node': host, 'last_seen': last_seen, 'expired': expired, 'relations': relations, 'host_services': host_services, 'dependent': dependent, 'dependencies': dependencies, 'location_path': location_path, 'history': True, 'urls': urls, 'connections': connections, 'scan_enabled': scan_enabled, })
def noclook_get_ports(handle_id): """ Return port nodes that are either dependencies or connected to item. Also returns the ports top parent. :param handle_id: unique id :return: list """ return nc.get_node_model(nc.graphdb.manager, handle_id).get_ports()
def noclook_get_model(handle_id): """ :param handle_id: unique id :return: Node model """ try: return nc.get_node_model(nc.graphdb.manager, handle_id) except nc.exceptions.NodeNotFound: return ''
def remove_router_conf(router_name, data_age, dry_run=False): routerq = """ MATCH (router:Node:Router {{name: '{}'}}) OPTIONAL MATCH (router)-[:Has*1..]->(physical) OPTIONAL MATCH (physical)<-[:Part_of]-(logical) WHERE (physical.noclook_auto_manage = true) OR (logical.noclook_auto_manage = true) RETURN collect(distinct physical.handle_id) as physical, collect(distinct logical.handle_id) as logical """.format(router_name) router_result = nc.query_to_dict(nc.graphdb.manager, routerq) for handle_id in router_result.get('logical', []): logical = nc.get_node_model(nc.graphdb.manager, handle_id) if logical: last_seen, expired = helpers.neo4j_data_age(logical.data, data_age) if expired: delete_node(logical, dry_run) for handle_id in router_result.get('physical', []): physical = nc.get_node_model(nc.graphdb.manager, handle_id) if physical: last_seen, expired = helpers.neo4j_data_age(physical.data, data_age) if expired: delete_node(physical, dry_run)
def get_host(ip_address): """ :param ip_address: string :return: neo4j node or None """ q = ''' MATCH (n:Host) USING SCAN n:Host WHERE any(x IN n.ip_addresses WHERE x =~ {address}) RETURN distinct n ''' for hit in nc.query_to_list(nc.graphdb.manager, q, address=ip_address): return nc.get_node_model(nc.graphdb.manager, node=hit['n'])
def get_peering_partner(peering): """ Inserts a new node of the type Peering partner and ensures that this node is unique for AS number. Returns the created node. """ try: return PEER_AS_CACHE[peering['as_number']] except KeyError: logger.info('Peering Partner {name} not in cache.'.format( name=peering.get('description'))) pass user = utils.get_user() peer_node = None peer_properties = {'name': 'Missing description', 'as_number': '0'} if peering.get('description'): peer_properties['name'] = peering.get('description') if peering.get('as_number'): peer_properties['as_number'] = peering.get('as_number') hits = nc.get_nodes_by_value(nc.graphdb.manager, prop='as_number', value=peer_properties['as_number']) found = 0 for node in hits: peer_node = nc.get_node_model(nc.graphdb.manager, node['handle_id']) helpers.set_noclook_auto_manage(peer_node, True) if peer_node.data[ 'name'] == 'Missing description' and peer_properties[ 'name'] != 'Missing description': helpers.dict_update_node(user, peer_node.handle_id, peer_properties) logger.info('Peering Partner {name} fetched.'.format( name=peer_properties['name'])) found += 1 if found > 1: logger.error( 'Found more then one Peering Partner with AS number {!s}'. format(peer_properties['as_number'])) if not peer_node: node_handle = utils.create_node_handle(peer_properties['name'], 'Peering Partner', 'Relation') peer_node = node_handle.get_node() helpers.set_noclook_auto_manage(peer_node, True) helpers.dict_update_node(user, peer_node.handle_id, peer_properties, peer_properties.keys()) logger.info('Peering Partner {name} created.'.format( name=peer_properties['name'])) PEER_AS_CACHE[peering['as_number']] = peer_node return peer_node
def match_remote_ip_address(remote_address): """ Matches a remote address to a local interface. Returns a Unit node if match found or else None. """ # Check cache for local_network in REMOTE_IP_MATCH_CACHE.keys(): if remote_address in local_network: cache_hit = REMOTE_IP_MATCH_CACHE[local_network] return cache_hit['local_network_node'], cache_hit['address'] # No cache hit mask = None for prefix in [3, 2, 1]: if remote_address.version == 4: mask = '.'.join(str(remote_address).split('.')[0:prefix]) elif remote_address.version == 6: mask = ':'.join(str(remote_address).split(':')[0:prefix]) if mask: mask = '{!s}{!s}'.format(mask, '.*') q = ''' MATCH (n:Unit) USING SCAN n:Unit WHERE any(x IN n.ip_addresses WHERE x =~ {mask}) RETURN distinct n ''' for hit in nc.query_to_list(nc.graphdb.manager, q, mask=mask): for address in hit['n']['ip_addresses']: try: local_network = ipaddress.ip_network(address, strict=False) except ValueError: continue # ISO address if remote_address in local_network: # add local_network, address and node to cache local_network_node = nc.get_node_model( nc.graphdb.manager, hit['n']['handle_id']) REMOTE_IP_MATCH_CACHE[local_network] = { 'address': address, 'local_network_node': local_network_node } logger.info( 'Remote IP matched: {name} {ip_address} done.'.format( name=local_network_node.data['name'], ip_address=address)) return local_network_node, address logger.info('No local IP address matched for {remote_address}.'.format( remote_address=remote_address)) return None, None
def visualize_json(request, handle_id): """ Creates a JSON representation of the node and the adjacent nodes. This JSON data is then used by Arbor.js (http://arborjs.org/) to make a visual representation. """ # Get the node nh = NodeHandle.objects.get(pk=handle_id) root_node = nc.get_node_model(nc.graphdb.manager, nh.handle_id) if root_node: # Create the data JSON structure needed graph_dict = arborgraph.create_generic_graph(root_node) jsonstr = arborgraph.get_json(graph_dict) else: jsonstr = '{}' return HttpResponse(jsonstr, content_type='application/json')
def update_noclook_auto_manage(item): """ Updates the noclook_auto_manage and noclook_last_seen properties. If noclook_auto_manage is not set, it is set to True. :param item: norduclient model :return: None """ auto_manage_data = {} auto_manage = item.data.get('noclook_auto_manage', None) if auto_manage or auto_manage is None: auto_manage_data['noclook_auto_manage'] = True auto_manage_data['noclook_last_seen'] = datetime.now().isoformat() if isinstance(item, nc.models.BaseNodeModel): node = nc.get_node_model(nc.graphdb.manager, item.handle_id) node.data.update(auto_manage_data) nc.set_node_properties(nc.graphdb.manager, node.handle_id, node.data) elif isinstance(item, nc.models.BaseRelationshipModel): relationship = nc.get_relationship_model(nc.graphdb.manager, item.id) relationship.data.update(auto_manage_data) nc.set_relationship_properties(nc.graphdb.manager, relationship.id, relationship.data)
def set_noclook_auto_manage(item, auto_manage): """ Sets the node or relationship noclook_auto_manage flag to True or False. Also sets the noclook_last_seen flag to now. :param item: norduclient model :param auto_manage: boolean :return: None """ auto_manage_data = { 'noclook_auto_manage': auto_manage, 'noclook_last_seen': datetime.now().isoformat() } if isinstance(item, nc.models.BaseNodeModel): node = nc.get_node_model(nc.graphdb.manager, item.handle_id) node.data.update(auto_manage_data) nc.set_node_properties(nc.graphdb.manager, node.handle_id, node.data) elif isinstance(item, nc.models.BaseRelationshipModel): relationship = nc.get_relationship_model(nc.graphdb.manager, item.id) relationship.data.update(auto_manage_data) nc.set_relationship_properties(nc.graphdb.manager, relationship.id, relationship.data)
def forwards_func(apps, schema_editor): NodeType = apps.get_model('noclook', 'NodeType') NodeHandle = apps.get_model('noclook', 'NodeHandle') organization_type = NodeType.objects.get_or_create( type='Organization', slug='organization')[0] # organization all_organizations = NodeHandle.objects.filter(node_type=organization_type) check_fields = [ 'affiliation_customer', 'affiliation_end_customer', 'affiliation_provider', 'affiliation_partner', 'affiliation_host_user', 'affiliation_site_owner', ] for organization in all_organizations: orgnode = nc.get_node_model(nc.graphdb.manager, organization.handle_id) #orgnode = organization.get_node() for field in check_fields: org_val = orgnode.data.get(field, None) if not org_val: orgnode.add_property(field, False)
def noclook_get_location(handle_id): return nc.get_node_model(nc.graphdb.manager, handle_id).get_location()
def forwards_func(apps, schema_editor): NodeType = apps.get_model('noclook', 'NodeType') NodeHandle = apps.get_model('noclook', 'NodeHandle') Context = apps.get_model('noclook', 'Context') NodeHandleContext = apps.get_model('noclook', 'NodeHandleContext') Dropdown = apps.get_model('noclook', 'Dropdown') Choice = apps.get_model('noclook', 'Choice') User = apps.get_model('auth', 'User') # wait for neo4j to be available neo4j_inited = False failure_count = 3 while not neo4j_inited and failure_count != 0: if nc.graphdb.manager: neo4j_inited = True else: failure_count = failure_count - 1 time.sleep(2) if neo4j_inited: username = '******' passwd = User.objects.make_random_password(length=30) user = None try: user = User.objects.get(username=username) except: user = User(username=username, password=passwd).save() # get the values from the old group dropdown groups_dropname = 'responsible_groups' groupdropdown, created = \ Dropdown.objects.get_or_create(name=groups_dropname) choices = Choice.objects.filter(dropdown=groupdropdown) group_type, created = NodeType.objects.get_or_create(type='Group', slug='group') groups_dict = {} community_context = sriutils.get_community_context(Context) host_type_objs = [] for host_type_str in host_types: host_type, created = NodeType.objects.get_or_create( type=host_type_str, slug=slugify(host_type_str)) host_type_objs.append(host_type) if NodeHandle.objects.filter(node_type__in=host_type_objs).exists(): for choice in choices: node_name = choice.name group_nh, created = NodeHandle.objects.get_or_create( node_name=node_name, node_type=group_type, node_meta_type=nc.META_TYPES[1], # Logical creator=user, modifier=user, ) if created: try: nc.create_node(nc.graphdb.manager, node_name, group_nh.node_meta_type, group_type.type, group_nh.handle_id) except CypherError: pass NodeHandleContext(nodehandle=group_nh, context=community_context).save() groups_dict[node_name] = group_nh # if there's nodes on the db, create groups with these values prop_methods = { 'responsible_group': 'set_takes_responsibility', 'support_group': 'set_supports', } # loop over entity types for host_type in host_type_objs: nhs = NodeHandle.objects.filter(node_type=host_type) # loop over entities of this type for nh in nhs: host_node = nc.get_node_model(nc.graphdb.manager, nh.handle_id) for prop, method_name in prop_methods.items(): # get old data prop_value = host_node.data.get(prop, None) # link matched group if prop_value and prop_value in groups_dict: group_nh = groups_dict[prop_value] group_node = nc.get_node_model(nc.graphdb.manager,\ group_nh.handle_id) method = getattr(group_node, method_name, None) if method: method(nh.handle_id) # remove old property host_node.remove_property(prop)
def handle(self, *args, **options): for contract_number in options['contract_number']: utcnow = datetime.utcnow() last_month = utcnow - relativedelta(months=1) subject = 'NOCLook host report for %s' % contract_number to = getattr(django_settings, 'REPORTS_TO', []) cc = getattr(django_settings, 'REPORTS_CC', None) bcc = getattr(django_settings, 'REPORTS_BCC', None) extra_report = getattr(django_settings, 'EXTRA_REPORT_TO', {}) extended_to = to + extra_report.get(contract_number, []) # Avoid changing REPORTS_TO :) body = ''' This is an auto generated report from NOCLook for contract number %s. This report was generated on %s UTC. ''' % (contract_number, utcnow.strftime('%Y-%m-%d %H:%M')) price_per_host = 95 header = [ 'Host user', 'Host', 'Host type', 'IP address(es)', 'Contract number', 'Description', 'Responsible', 'Backup', 'Syslog', 'Nagios', 'Operational State', 'Security Class', 'Location', 'Last seen', 'Uptime (days)' ] result = [] q = ''' MATCH (host_user:Host_User)-[r:Uses|Owns]->(host:Host) WHERE host.contract_number = {contract_number} RETURN host_user.name as host_user_name, host.name, host.handle_id as host_handle_id ORDER BY host_user.name, host.name ''' for item in nc.query_to_list(nc.graphdb.manager, q, contract_number=contract_number): host = nc.get_node_model(nc.graphdb.manager, item['host_handle_id']) age = helpers.neo4j_report_age(host.data, 15, 30) operational_state = host.data.get('operational_state', 'Not set') host_type = host.meta_type host_user = item['host_user_name'] uptime = host.data.get('uptime', '') if uptime: uptime = timestamp_to_td(uptime).days if host_type == 'Logical' and age != 'very_old' and operational_state != 'Decommissioned': values = [ u'{}'.format(host_user), u'{}'.format(host.data['name']), u'{}'.format(host_type).capitalize(), u', '.join([address for address in host.data['ip_addresses']]), u'{}'.format(host.data['contract_number']), u'{}'.format(host.data.get('description', '')), u'{}'.format(host.data.get('responsible_group', '')), u'{}'.format(host.data.get('backup', 'Not set')), u'{}'.format(yesno(host.data.get('syslog', None), 'Yes,No,Not set')), u'{}'.format(yesno(host.data.get('nagios_checks', False), 'Yes,No,Not set')), u'{}'.format(operational_state), u'{}'.format(host.data.get('security_class', '')), u'{}'.format(''), u'{}'.format(date(helpers.isots_to_dt(host.data), "Y-m-d")), u'{}'.format(uptime), ] result.append(dict(zip(header, values))) num_hosts = len(result) filename = '%s hosts %s.xls' % (contract_number, last_month.strftime('%B %Y')) wb = helpers.dicts_to_xls(result, header, contract_number) # Calculate and write pricing info ws = wb.get_sheet(0) ws.write(num_hosts + 2, 1, 'Number of Virtual Servers') ws.write(num_hosts + 2, 4, '%d' % num_hosts) ws.write(num_hosts + 3, 1, 'Price') ws.write(num_hosts + 3, 4, '%d' % price_per_host) ws.write(num_hosts + 4, 1, 'Total Invoice amount ex. VAT') ws.write(num_hosts + 4, 4, '%d' % (num_hosts * price_per_host)) with tempfile.TemporaryFile() as temp: wb.save(temp) temp.seek(0) msg = helpers.create_email(subject, body, extended_to, cc, bcc, temp.read(), filename, 'application/excel') msg.send() self.stdout.write('Sent report for contract number {}'.format(contract_number))
def get_peering_partner(peering): """ Inserts a new node of the type Peering partner and ensures that this node is unique for AS number. Returns the created node. """ try: return PEER_AS_CACHE[peering['as_number']] except KeyError: logger.info('Peering Partner {name} not in cache.'.format( name=peering.get('description'))) pass user = utils.get_user() peer_node = None peer_properties = {'name': 'Missing description', 'as_number': '0'} # neither description or as_number error and return if not (peering.get('description') or peering.get('as_number')): logger.error('Neither AS number nor description in peering %s', peering) return None if peering.get('description'): peer_properties['name'] = peering.get('description') if peering.get('as_number'): peer_properties['as_number'] = peering.get('as_number') # as number is most important hits = nc.get_nodes_by_value(nc.graphdb.manager, prop='as_number', value=peer_properties['as_number']) found = 0 for node in hits: peer_node = nc.get_node_model(nc.graphdb.manager, node['handle_id']) helpers.set_noclook_auto_manage(peer_node, True) if peer_node.data[ 'name'] == 'Missing description' and peer_properties[ 'name'] != 'Missing description': helpers.dict_update_node(user, peer_node.handle_id, peer_properties) logger.info('Peering Partner {name} fetched.'.format( name=peer_properties['name'])) found += 1 if found > 1: logger.error( 'Found more then one Peering Partner with AS number {!s}'. format(peer_properties['as_number'])) if not peer_node: # since we have a AS number we will create a new Peering Partner, even if name is missing or exists node_handle = utils.create_node_handle(peer_properties['name'], 'Peering Partner', 'Relation') peer_node = node_handle.get_node() helpers.set_noclook_auto_manage(peer_node, True) helpers.dict_update_node(user, peer_node.handle_id, peer_properties, peer_properties.keys()) logger.info('Peering Partner %s AS(%s) created.', peer_properties['name'], peer_properties['as_number']) # Handle peer with name only if not peer_node and peering.get('description'): # Try and get peer_partners res = NodeHandle.objects.filter( node_name__iexact=peer_properties['name'], node_type__type='Peering Partner').order_by('-modified') for ph in res: peer_node = ph.get_node() break if not peer_node: # create peer_nh = utils.get_unique_node_handle(peer_properties['name'], 'Peering Partner', 'Relation') peer_node = peer_nh.get_node() if not peer_node.data.get('as_number'): # Peering partner did not exist logger.warning( 'Peering Partner %s without AS number created for peering: %s', peer_node.data.get('name'), peering) # AS number is going to be 0, but that is ok helpers.dict_update_node(user, peer_node.handle_id, peer_properties, peer_properties.keys()) elif peer_node.data.get('as_number') != '0': # warn about as number not being in peering logger.warning( 'Peering found for Peering Partner %s without the AS number %s mentioned. Peering: %s', peer_properties['name'], peer_node.data.get('as_number'), peering) helpers.set_noclook_auto_manage(peer_node, True) PEER_AS_CACHE[peering['as_number']] = peer_node return peer_node
def get_node(self): """ Returns the NodeHandles node. """ return nc.get_node_model(nc.graphdb.manager, self.handle_id)
def backwards_func(apps, schema_editor): NodeType = apps.get_model('noclook', 'NodeType') NodeHandle = apps.get_model('noclook', 'NodeHandle') Dropdown = apps.get_model('noclook', 'Dropdown') Choice = apps.get_model('noclook', 'Choice') User = apps.get_model('auth', 'User') user = get_user(usermodel=User) # get the values from the old group dropdown groups_dropname = 'responsible_groups' groupdropdown, created = \ Dropdown.objects.get_or_create(name=groups_dropname) choices = Choice.objects.filter(dropdown=groupdropdown) # get group options groups_opts_dict = {} host_type_objs = [] for host_type_str in host_types: host_type, created = NodeType.objects.get_or_create( type=host_type_str, slug=slugify(host_type_str)) host_type_objs.append(host_type) if NodeHandle.objects.filter(node_type__in=host_type_objs).exists(): # fill choice dict for choice in choices: choice_name = choice.name groups_opts_dict[choice_name] = choice # loop over entity types for host_type_str in host_types: host_type, created = NodeType.objects.get_or_create( type=host_type_str, slug=slugify(host_type_str)) nhs = NodeHandle.objects.filter(node_type=host_type) # loop over entities of this type for nh in nhs: host_node = nc.get_node_model(nc.graphdb.manager, nh.handle_id) attr_val_dict = { 'responsible_group': host_node.incoming.get('Takes_responsibility'), 'support_group': host_node.incoming.get('Supports'), } for attr_name, rels in attr_val_dict.items(): if rels: # unlink group and get its name relationship = rels[0] node = relationship['node'] ngroup_name = node.data.get('name', None) if ngroup_name in groups_opts_dict: group_choice = groups_opts_dict[ngroup_name] # add old property value host_node.add_property(attr_name, group_choice.value) # delete relationship anyways nc.delete_relationship(nc.graphdb.manager, relationship['relationship_id']) # delete created groups (both postgresql and neo4j) group_type, created = NodeType.objects.get_or_create(type='Group', slug='group') groups_nhs = NodeHandle.objects.filter(node_type=group_type) for group_nh in groups_nhs: q = """ MATCH (n:Group {handle_id:{handle_id}}) DETACH DELETE n """ nc.query_to_dict(nc.graphdb.manager, q, handle_id=group_nh.handle_id) group_nh.delete()