def remove_peer_conf(user, data_age): peerq = """ MATCH (peer_group:Node:Peering_Group) MATCH (peer_group)<-[r:Uses]-(peering_partner:Peering_Partner) WHERE (peer_group.noclook_auto_manage = true) OR (r.noclook_auto_manage = true) RETURN collect(distinct peer_group.handle_id) as peer_groups, collect(id(r)) as uses_relationships """ peer_result = nc.query_to_dict(nc.graphdb.manager, peerq) for relationship_id in peer_result.get('uses_relationships', []): relationship = nc.get_relationship_model(nc.graphdb.manager, relationship_id) if relationship: last_seen, expired = helpers.neo4j_data_age( relationship.data, data_age) if expired: rel_info = helpers.relationship_to_str(relationship) helpers.delete_relationship(user, relationship.id) logger.warning('Deleted relationship {rel_info}'.format( rel_info=rel_info)) for handle_id in peer_result.get('peer_groups', []): peer_group = nc.get_node_model(nc.graphdb.manager, handle_id) if peer_group: last_seen, expired = helpers.neo4j_data_age( peer_group.data, data_age) if expired: helpers.delete_node(user, peer_group.handle_id) logger.warning('Deleted node {name} ({handle_id}).'.format( name=peer_group.data.get('name'), handle_id=handle_id))
def remove_router_conf(user, data_age): routerq = """ MATCH (router:Node:Router) OPTIONAL MATCH (router)-[:Has*1..]->(physical)<-[:Part_of]-(logical) WHERE (physical.noclook_auto_manage = true) OR (logical.noclook_auto_manage = true) RETURN collect(distinct physical.handle_id) as physical, collect(distinct logical.handle_id) as logical """ router_result = nc.query_to_dict(nc.graphdb.manager, routerq) for handle_id in router_result.get('logical', []): logical = nc.get_node_model(nc.graphdb.manager, handle_id) if logical: last_seen, expired = helpers.neo4j_data_age(logical.data, data_age) if expired: helpers.delete_node(user, logical.handle_id) logger.warning('Deleted logical router: %s (%s).', logical.data.get('name'), handle_id) for handle_id in router_result.get('physical', []): physical = nc.get_node_model(nc.graphdb.manager, handle_id) if physical: last_seen, expired = helpers.neo4j_data_age( physical.data, data_age) if expired: helpers.delete_node(user, physical.handle_id) logger.warning('Deleted physical router: %s (%s).', physical.data.get('name'), handle_id)
def checkOrgInUse(self, organization_id): q = """ MATCH (o:Organization) WHERE o.organization_id = '{organization_id}' RETURN COUNT(DISTINCT o.organization_id) AS orgs; """.format(organization_id=organization_id) res = nc.query_to_dict(nc.graphdb.manager, q) return res['orgs']
def count_different_orgids(self): q = """ MATCH (o:Organization) RETURN COUNT(DISTINCT o.organization_id) AS orgs """ res = nc.query_to_dict(nc.graphdb.manager, q) return res['orgs']
def get_object_list(self, request, **kwargs): q = """ MATCH (node:Service) WHERE node.service_type = "L2VPN" OR node.service_type = "Interface Switch" RETURN collect(node.handle_id) as handle_ids """ hits = nc.query_to_dict(nc.graphdb.manager, q) return NodeHandle.objects.filter(pk__in=hits['handle_ids'])
def handle(self, *args, **options): unauthorized_q = """ MATCH (host:Host) MATCH host<-[r:Depends_on]-() WHERE not(host.operational_state = "Decommissioned") and exists(r.rogue_port) RETURN count(DISTINCT host) as unauthorized_host_count, count(r) as unauthorized_port_count """ public_q = """ MATCH (host:Host) MATCH host<-[r:Depends_on]-() WHERE not(host.operational_state = "Decommissioned") and r.public and (not(r.public_service) or not(exists(r.public_service))) RETURN count(DISTINCT host) as public_host_count, count(r) as public_port_count """ results = nc.query_to_dict(nc.graphdb.manager, unauthorized_q) results.update(nc.query_to_dict(nc.graphdb.manager, public_q)) results['now'] = datetime.utcnow().strftime('%Y-%m-%d %H:%M') results['domain'] = Site.objects.get_current().domain subject = 'NOCLook host security report' to = getattr(django_settings, 'SECURITY_REPORTS_TO', []) cc = getattr(django_settings, 'SECURITY_REPORTS_CC', None) bcc = getattr(django_settings, 'SECURITY_REPORTS_BCC', None) body = ''' This is an auto generated host security report from NOCLook. Unauthorized ports: {unauthorized_host_count} hosts have unauthorized ports. There are a total of {unauthorized_port_count} unauthorized ports. See https://{domain}/reports/hosts/host-services/unauthorized-ports/ for more information. --- Public ports: {public_host_count} hosts have unverified public ports. There are a total of {public_port_count} unverified public ports. See https://{domain}/reports/hosts/host-services/public/ for more information. This report was generated on {now} UTC. '''.format(**results) msg = helpers.create_email(subject, body, to, cc, bcc) msg.send()
def get_expected_length(self, search): q = """ MATCH (n:Node) WHERE any(prop in keys(n) WHERE n[prop] =~ "(?i).*{search}.*") RETURN count(n) as total """.format(search=search) res = nc.query_to_dict(nc.graphdb.manager, q, search=search) return res['total']
def noclook_has_rogue_ports(handle_id): """ :param handle_id: unique id :return: Boolean """ q = """ MATCH (host:Node {handle_id: {handle_id}})<-[r:Depends_on]-() RETURN count(r.rogue_port) as count """ d = nc.query_to_dict(nc.graphdb.manager, q, handle_id=handle_id) if d['count']: return True return False
def add_parent(self, user, physical_nh, physical_parent_nh): handle_id = physical_nh.handle_id parent_handle_id = physical_parent_nh.handle_id q = """ MATCH (n:Node:Physical {handle_id: {handle_id}}), (p:Node:Physical {handle_id: {parent_handle_id}}) MERGE (n)<-[r:Has]-(p) RETURN n, r, p """ result = nc.query_to_dict(nc.graphdb.manager, q, handle_id=handle_id, parent_handle_id=parent_handle_id)
def count_dependencies(handle_id, last_seen): q_deps = """ MATCH (n:Host_Service {handle_id: $handle_id})-[r:Depends_on]->(n2:Node) WHERE r.noclook_last_seen < $last_seen RETURN count(r) as count """ result = nc.query_to_dict( nc.graphdb.manager, q_deps, handle_id=handle_id, last_seen=last_seen, ) return result['count']
def cleanup_host_service(nh, max_last_seen, dry_run): logger.info('Cleaning old dependencies and activity log for %s', nh.node_name) dep_count = count_dependencies(nh.handle_id, max_last_seen.isoformat()) if dry_run: logger.warning( "[Dry-run] would delete %d old dependencies for %s (%s)", dep_count, nh.node_name, nh.handle_id) else: q_delete_deps = """ MATCH (n:Host_Service {handle_id: $handle_id})-[r:Depends_on]->(n2:Node) WHERE r.noclook_last_seen < $last_seen DELETE r """ # remove old relations nc.query_to_dict( nc.graphdb.manager, q_delete_deps, handle_id=nh.handle_id, last_seen=max_last_seen.isoformat(), ) logger.warning("Deleted %d old dependencies for %s (%s)", dep_count, nh.node_name, nh.handle_id) # clean up activity log # Just delete old stuffA as it is useless # Not year 10000 proof :P actions = Action.objects.filter(action_object_object_id=nh.handle_id, timestamp__lt=max_last_seen) if dry_run: logger.warning( "[Dry-run] would delete %s activity log entries for %s (%s)", actions.count(), nh.node_name, nh.handle_id) else: action_count, _ = actions.delete() logger.warning("Deleted %s activity log entries for %s (%s)", action_count, nh.node_name, nh.handle_id)
def link_parent(cls, user, physical_nh, physical_parent_nh): # TODO: Make helper method handle_id = physical_nh.handle_id parent_handle_id = physical_parent_nh.handle_id q = """ MATCH (n:Node:Physical {handle_id: {handle_id}}), (p:Node:Physical {handle_id: {parent_handle_id}}) MERGE (n)<-[r:Has]-(p) RETURN n, r, p """ result = nc.query_to_dict(nc.graphdb.manager, q, handle_id=handle_id, parent_handle_id=parent_handle_id)
def auto_depend_services(handle_id, description, service_id_regex, _type="Port"): """ Using interface description to depend one or more services. """ if not service_id_regex: return if not description: description = "" desc_services = service_id_regex.findall(description) for service_id in desc_services: service = _find_service(service_id) if service: if service.data.get('operational_state') == 'Decommissioned': logger.warning( '{} {} description mentions decommissioned service {}'. format(_type, handle_id, service_id)) else: # Add it # logger.warning('Service {} should depend on port {}'.format(service_id, handle_id)) helpers.set_depends_on(utils.get_user(), service, handle_id) else: logger.info('{} {} description mentions unknown service {}'.format( _type, handle_id, service_id)) # check if "other services are dependent" q = """ MATCH (n:Node {handle_id: {handle_id}})<-[:Depends_on]-(s:Service) WHERE s.operational_state <> 'Decommissioned' and NOT(s.name in [{desc_services}]) RETURN collect(s) as unregistered """ result = nc.query_to_dict(nc.graphdb.manager, q, handle_id=handle_id, desc_services=','.join(desc_services)).get( 'unregistered', []) unregistered_services = [ u"{}({})".format(s['name'], s['handle_id']) for s in result ] if unregistered_services: logger.info( u"{} {} has services depending on it that is not in description: {}" .format(_type, handle_id, ','.join(unregistered_services)))
def remove_rogue_service_marker(user, handle_id): """ :param user: Django user :param handle_id: unique id :return: True Removed the property rogue_port from all Depends_on relationships. """ q = """ MATCH (host:Node {handle_id:{handle_id}})<-[r:Depends_on]-(host_service:Host_Service) WHERE exists(r.rogue_port) RETURN collect(id(r)) as ids """ result = nc.query_to_dict(nc.graphdb.manager, q, handle_id=handle_id) properties = {'rogue_port': ''} for relationship_id in result['ids']: dict_update_relationship(user, relationship_id, properties, properties.keys()) return True
def remove_router_conf(router_name, data_age, dry_run=False): routerq = """ MATCH (router:Node:Router {{name: '{}'}}) OPTIONAL MATCH (router)-[:Has*1..]->(physical) OPTIONAL MATCH (physical)<-[:Part_of]-(logical) WHERE (physical.noclook_auto_manage = true) OR (logical.noclook_auto_manage = true) RETURN collect(distinct physical.handle_id) as physical, collect(distinct logical.handle_id) as logical """.format(router_name) router_result = nc.query_to_dict(nc.graphdb.manager, routerq) for handle_id in router_result.get('logical', []): logical = nc.get_node_model(nc.graphdb.manager, handle_id) if logical: last_seen, expired = helpers.neo4j_data_age(logical.data, data_age) if expired: delete_node(logical, dry_run) for handle_id in router_result.get('physical', []): physical = nc.get_node_model(nc.graphdb.manager, handle_id) if physical: last_seen, expired = helpers.neo4j_data_age(physical.data, data_age) if expired: delete_node(physical, dry_run)
def backwards_func(apps, schema_editor): NodeType = apps.get_model('noclook', 'NodeType') NodeHandle = apps.get_model('noclook', 'NodeHandle') Dropdown = apps.get_model('noclook', 'Dropdown') Choice = apps.get_model('noclook', 'Choice') User = apps.get_model('auth', 'User') user = get_user(usermodel=User) # get the values from the old group dropdown groups_dropname = 'responsible_groups' groupdropdown, created = \ Dropdown.objects.get_or_create(name=groups_dropname) choices = Choice.objects.filter(dropdown=groupdropdown) # get group options groups_opts_dict = {} host_type_objs = [] for host_type_str in host_types: host_type, created = NodeType.objects.get_or_create( type=host_type_str, slug=slugify(host_type_str)) host_type_objs.append(host_type) if NodeHandle.objects.filter(node_type__in=host_type_objs).exists(): # fill choice dict for choice in choices: choice_name = choice.name groups_opts_dict[choice_name] = choice # loop over entity types for host_type_str in host_types: host_type, created = NodeType.objects.get_or_create( type=host_type_str, slug=slugify(host_type_str)) nhs = NodeHandle.objects.filter(node_type=host_type) # loop over entities of this type for nh in nhs: host_node = nc.get_node_model(nc.graphdb.manager, nh.handle_id) attr_val_dict = { 'responsible_group': host_node.incoming.get('Takes_responsibility'), 'support_group': host_node.incoming.get('Supports'), } for attr_name, rels in attr_val_dict.items(): if rels: # unlink group and get its name relationship = rels[0] node = relationship['node'] ngroup_name = node.data.get('name', None) if ngroup_name in groups_opts_dict: group_choice = groups_opts_dict[ngroup_name] # add old property value host_node.add_property(attr_name, group_choice.value) # delete relationship anyways nc.delete_relationship(nc.graphdb.manager, relationship['relationship_id']) # delete created groups (both postgresql and neo4j) group_type, created = NodeType.objects.get_or_create(type='Group', slug='group') groups_nhs = NodeHandle.objects.filter(node_type=group_type) for group_nh in groups_nhs: q = """ MATCH (n:Group {handle_id:{handle_id}}) DETACH DELETE n """ nc.query_to_dict(nc.graphdb.manager, q, handle_id=group_nh.handle_id) group_nh.delete()