def remove_peer_conf(user, data_age):
peerq = """
MATCH (peer_group:Node:Peering_Group)
MATCH (peer_group)<-[r:Uses]-(peering_partner:Peering_Partner)
WHERE (peer_group.noclook_auto_manage = true) OR (r.noclook_auto_manage = true)
RETURN collect(distinct peer_group.handle_id) as peer_groups, collect(id(r)) as uses_relationships
"""
peer_result = nc.query_to_dict(nc.graphdb.manager, peerq)
for relationship_id in peer_result.get('uses_relationships', []):
relationship = nc.get_relationship_model(nc.graphdb.manager,
relationship_id)
if relationship:
last_seen, expired = helpers.neo4j_data_age(
relationship.data, data_age)
if expired:
rel_info = helpers.relationship_to_str(relationship)
helpers.delete_relationship(user, relationship.id)
logger.warning('Deleted relationship {rel_info}'.format(
rel_info=rel_info))
for handle_id in peer_result.get('peer_groups', []):
peer_group = nc.get_node_model(nc.graphdb.manager, handle_id)
if peer_group:
last_seen, expired = helpers.neo4j_data_age(
peer_group.data, data_age)
if expired:
helpers.delete_node(user, peer_group.handle_id)
logger.warning('Deleted node {name} ({handle_id}).'.format(
name=peer_group.data.get('name'), handle_id=handle_id))
def remove_router_conf(user, data_age):
routerq = """
MATCH (router:Node:Router)
OPTIONAL MATCH (router)-[:Has*1..]->(physical)<-[:Part_of]-(logical)
WHERE (physical.noclook_auto_manage = true) OR (logical.noclook_auto_manage = true)
RETURN collect(distinct physical.handle_id) as physical, collect(distinct logical.handle_id) as logical
"""
router_result = nc.query_to_dict(nc.graphdb.manager, routerq)
for handle_id in router_result.get('logical', []):
logical = nc.get_node_model(nc.graphdb.manager, handle_id)
if logical:
last_seen, expired = helpers.neo4j_data_age(logical.data, data_age)
if expired:
helpers.delete_node(user, logical.handle_id)
logger.warning('Deleted logical router: %s (%s).',
logical.data.get('name'), handle_id)
for handle_id in router_result.get('physical', []):
physical = nc.get_node_model(nc.graphdb.manager, handle_id)
if physical:
last_seen, expired = helpers.neo4j_data_age(
physical.data, data_age)
if expired:
helpers.delete_node(user, physical.handle_id)
logger.warning('Deleted physical router: %s (%s).',
physical.data.get('name'), handle_id)
def checkOrgInUse(self, organization_id):
q = """
MATCH (o:Organization) WHERE o.organization_id = '{organization_id}'
RETURN COUNT(DISTINCT o.organization_id) AS orgs;
""".format(organization_id=organization_id)
res = nc.query_to_dict(nc.graphdb.manager, q)
return res['orgs']
def count_different_orgids(self):
q = """
MATCH (o:Organization)
RETURN COUNT(DISTINCT o.organization_id) AS orgs
"""
res = nc.query_to_dict(nc.graphdb.manager, q)
return res['orgs']
def get_object_list(self, request, **kwargs):
q = """
MATCH (node:Service)
WHERE node.service_type = "L2VPN" OR node.service_type = "Interface Switch"
RETURN collect(node.handle_id) as handle_ids
"""
hits = nc.query_to_dict(nc.graphdb.manager, q)
return NodeHandle.objects.filter(pk__in=hits['handle_ids'])
def handle(self, *args, **options):
unauthorized_q = """
MATCH (host:Host)
MATCH host<-[r:Depends_on]-()
WHERE not(host.operational_state = "Decommissioned") and exists(r.rogue_port)
RETURN count(DISTINCT host) as unauthorized_host_count, count(r) as unauthorized_port_count
"""
public_q = """
MATCH (host:Host)
MATCH host<-[r:Depends_on]-()
WHERE not(host.operational_state = "Decommissioned") and r.public and (not(r.public_service) or not(exists(r.public_service)))
RETURN count(DISTINCT host) as public_host_count, count(r) as public_port_count
"""
results = nc.query_to_dict(nc.graphdb.manager, unauthorized_q)
results.update(nc.query_to_dict(nc.graphdb.manager, public_q))
results['now'] = datetime.utcnow().strftime('%Y-%m-%d %H:%M')
results['domain'] = Site.objects.get_current().domain
subject = 'NOCLook host security report'
to = getattr(django_settings, 'SECURITY_REPORTS_TO', [])
cc = getattr(django_settings, 'SECURITY_REPORTS_CC', None)
bcc = getattr(django_settings, 'SECURITY_REPORTS_BCC', None)
body = '''
This is an auto generated host security report from NOCLook.
Unauthorized ports:
{unauthorized_host_count} hosts have unauthorized ports.
There are a total of {unauthorized_port_count} unauthorized ports.
See https://{domain}/reports/hosts/host-services/unauthorized-ports/ for more information.
---
Public ports:
{public_host_count} hosts have unverified public ports.
There are a total of {public_port_count} unverified public ports.
See https://{domain}/reports/hosts/host-services/public/ for more information.
This report was generated on {now} UTC.
'''.format(**results)
msg = helpers.create_email(subject, body, to, cc, bcc)
msg.send()
def get_expected_length(self, search):
q = """
MATCH (n:Node)
WHERE any(prop in keys(n) WHERE n[prop] =~ "(?i).*{search}.*")
RETURN count(n) as total
""".format(search=search)
res = nc.query_to_dict(nc.graphdb.manager, q, search=search)
return res['total']
def noclook_has_rogue_ports(handle_id):
"""
:param handle_id: unique id
:return: Boolean
"""
q = """
MATCH (host:Node {handle_id: {handle_id}})<-[r:Depends_on]-()
RETURN count(r.rogue_port) as count
"""
d = nc.query_to_dict(nc.graphdb.manager, q, handle_id=handle_id)
if d['count']:
return True
return False
def add_parent(self, user, physical_nh, physical_parent_nh):
handle_id = physical_nh.handle_id
parent_handle_id = physical_parent_nh.handle_id
q = """
MATCH (n:Node:Physical {handle_id: {handle_id}}),
(p:Node:Physical {handle_id: {parent_handle_id}})
MERGE (n)<-[r:Has]-(p)
RETURN n, r, p
"""
result = nc.query_to_dict(nc.graphdb.manager, q,
handle_id=handle_id, parent_handle_id=parent_handle_id)
def count_dependencies(handle_id, last_seen):
q_deps = """
MATCH (n:Host_Service {handle_id: $handle_id})-[r:Depends_on]->(n2:Node)
WHERE r.noclook_last_seen < $last_seen
RETURN count(r) as count
"""
result = nc.query_to_dict(
nc.graphdb.manager,
q_deps,
handle_id=handle_id,
last_seen=last_seen,
)
return result['count']
def cleanup_host_service(nh, max_last_seen, dry_run):
logger.info('Cleaning old dependencies and activity log for %s',
nh.node_name)
dep_count = count_dependencies(nh.handle_id, max_last_seen.isoformat())
if dry_run:
logger.warning(
"[Dry-run] would delete %d old dependencies for %s (%s)",
dep_count, nh.node_name, nh.handle_id)
else:
q_delete_deps = """
MATCH (n:Host_Service {handle_id: $handle_id})-[r:Depends_on]->(n2:Node)
WHERE r.noclook_last_seen < $last_seen
DELETE r
"""
# remove old relations
nc.query_to_dict(
nc.graphdb.manager,
q_delete_deps,
handle_id=nh.handle_id,
last_seen=max_last_seen.isoformat(),
)
logger.warning("Deleted %d old dependencies for %s (%s)", dep_count,
nh.node_name, nh.handle_id)
# clean up activity log
# Just delete old stuffA as it is useless
# Not year 10000 proof :P
actions = Action.objects.filter(action_object_object_id=nh.handle_id,
timestamp__lt=max_last_seen)
if dry_run:
logger.warning(
"[Dry-run] would delete %s activity log entries for %s (%s)",
actions.count(), nh.node_name, nh.handle_id)
else:
action_count, _ = actions.delete()
logger.warning("Deleted %s activity log entries for %s (%s)",
action_count, nh.node_name, nh.handle_id)
def link_parent(cls, user, physical_nh, physical_parent_nh):
# TODO: Make helper method
handle_id = physical_nh.handle_id
parent_handle_id = physical_parent_nh.handle_id
q = """
MATCH (n:Node:Physical {handle_id: {handle_id}}),
(p:Node:Physical {handle_id: {parent_handle_id}})
MERGE (n)<-[r:Has]-(p)
RETURN n, r, p
"""
result = nc.query_to_dict(nc.graphdb.manager,
q,
handle_id=handle_id,
parent_handle_id=parent_handle_id)
def auto_depend_services(handle_id,
description,
service_id_regex,
_type="Port"):
"""
Using interface description to depend one or more services.
"""
if not service_id_regex:
return
if not description:
description = ""
desc_services = service_id_regex.findall(description)
for service_id in desc_services:
service = _find_service(service_id)
if service:
if service.data.get('operational_state') == 'Decommissioned':
logger.warning(
'{} {} description mentions decommissioned service {}'.
format(_type, handle_id, service_id))
else:
# Add it
# logger.warning('Service {} should depend on port {}'.format(service_id, handle_id))
helpers.set_depends_on(utils.get_user(), service, handle_id)
else:
logger.info('{} {} description mentions unknown service {}'.format(
_type, handle_id, service_id))
# check if "other services are dependent"
q = """
MATCH (n:Node {handle_id: {handle_id}})<-[:Depends_on]-(s:Service)
WHERE s.operational_state <> 'Decommissioned' and NOT(s.name in [{desc_services}])
RETURN collect(s) as unregistered
"""
result = nc.query_to_dict(nc.graphdb.manager,
q,
handle_id=handle_id,
desc_services=','.join(desc_services)).get(
'unregistered', [])
unregistered_services = [
u"{}({})".format(s['name'], s['handle_id']) for s in result
]
if unregistered_services:
logger.info(
u"{} {} has services depending on it that is not in description: {}"
.format(_type, handle_id, ','.join(unregistered_services)))
def remove_rogue_service_marker(user, handle_id):
"""
:param user: Django user
:param handle_id: unique id
:return: True
Removed the property rogue_port from all Depends_on relationships.
"""
q = """
MATCH (host:Node {handle_id:{handle_id}})<-[r:Depends_on]-(host_service:Host_Service)
WHERE exists(r.rogue_port)
RETURN collect(id(r)) as ids
"""
result = nc.query_to_dict(nc.graphdb.manager, q, handle_id=handle_id)
properties = {'rogue_port': ''}
for relationship_id in result['ids']:
dict_update_relationship(user, relationship_id, properties, properties.keys())
return True
def remove_router_conf(router_name, data_age, dry_run=False):
routerq = """
MATCH (router:Node:Router {{name: '{}'}})
OPTIONAL MATCH (router)-[:Has*1..]->(physical)
OPTIONAL MATCH (physical)<-[:Part_of]-(logical)
WHERE (physical.noclook_auto_manage = true) OR (logical.noclook_auto_manage = true)
RETURN collect(distinct physical.handle_id) as physical, collect(distinct logical.handle_id) as logical
""".format(router_name)
router_result = nc.query_to_dict(nc.graphdb.manager, routerq)
for handle_id in router_result.get('logical', []):
logical = nc.get_node_model(nc.graphdb.manager, handle_id)
if logical:
last_seen, expired = helpers.neo4j_data_age(logical.data, data_age)
if expired:
delete_node(logical, dry_run)
for handle_id in router_result.get('physical', []):
physical = nc.get_node_model(nc.graphdb.manager, handle_id)
if physical:
last_seen, expired = helpers.neo4j_data_age(physical.data, data_age)
if expired:
delete_node(physical, dry_run)
def backwards_func(apps, schema_editor):
NodeType = apps.get_model('noclook', 'NodeType')
NodeHandle = apps.get_model('noclook', 'NodeHandle')
Dropdown = apps.get_model('noclook', 'Dropdown')
Choice = apps.get_model('noclook', 'Choice')
User = apps.get_model('auth', 'User')
user = get_user(usermodel=User)
# get the values from the old group dropdown
groups_dropname = 'responsible_groups'
groupdropdown, created = \
Dropdown.objects.get_or_create(name=groups_dropname)
choices = Choice.objects.filter(dropdown=groupdropdown)
# get group options
groups_opts_dict = {}
host_type_objs = []
for host_type_str in host_types:
host_type, created = NodeType.objects.get_or_create(
type=host_type_str, slug=slugify(host_type_str))
host_type_objs.append(host_type)
if NodeHandle.objects.filter(node_type__in=host_type_objs).exists():
# fill choice dict
for choice in choices:
choice_name = choice.name
groups_opts_dict[choice_name] = choice
# loop over entity types
for host_type_str in host_types:
host_type, created = NodeType.objects.get_or_create(
type=host_type_str, slug=slugify(host_type_str))
nhs = NodeHandle.objects.filter(node_type=host_type)
# loop over entities of this type
for nh in nhs:
host_node = nc.get_node_model(nc.graphdb.manager, nh.handle_id)
attr_val_dict = {
'responsible_group':
host_node.incoming.get('Takes_responsibility'),
'support_group':
host_node.incoming.get('Supports'),
}
for attr_name, rels in attr_val_dict.items():
if rels:
# unlink group and get its name
relationship = rels[0]
node = relationship['node']
ngroup_name = node.data.get('name', None)
if ngroup_name in groups_opts_dict:
group_choice = groups_opts_dict[ngroup_name]
# add old property value
host_node.add_property(attr_name,
group_choice.value)
# delete relationship anyways
nc.delete_relationship(nc.graphdb.manager,
relationship['relationship_id'])
# delete created groups (both postgresql and neo4j)
group_type, created = NodeType.objects.get_or_create(type='Group',
slug='group')
groups_nhs = NodeHandle.objects.filter(node_type=group_type)
for group_nh in groups_nhs:
q = """
MATCH (n:Group {handle_id:{handle_id}}) DETACH DELETE n
"""
nc.query_to_dict(nc.graphdb.manager, q, handle_id=group_nh.handle_id)
group_nh.delete()
