def get_unique_node(name, node_type, meta_type):
"""
Gets or creates a NodeHandle with the provided name.
Returns the NodeHandles node.
"""
name = normalize_whitespace(name)
node_handle = utils.get_unique_node_handle(name, node_type, meta_type)
node = node_handle.get_node()
return node
def insert_site(site_dict):
"""
:param site_dict: data
:type site_dict: dict
:return: None
:rtype: None
Expected dict
{
u'name': u'',
u'area': u'',
u'country': u'',
u'longitude': u'',
u'node_type': u'Site',
u'meta_type': u'Location',
u'site_owner': u'',
u'latitude': u'',
u'site_type': u''
}
"""
user = utils.get_user()
# Remove meta data and relationship data from the site dict
name = site_dict.pop('name')
node_type = site_dict.pop('node_type')
meta_type = site_dict.pop('meta_type')
site_owner = site_dict.pop('site_owner')
# Get or create Site
site_nh = utils.get_unique_node_handle(name, node_type, meta_type)
# Set or update node properties
helpers.dict_update_node(user, site_nh.handle_id, site_dict,
site_dict.keys())
if site_owner:
# Get or create Site owner
site_owner_nh = utils.get_unique_node_handle(site_owner, 'Site Owner',
'Relation')
# Set relationship to site owner
helpers.set_responsible_for(user, site_nh.get_node(),
site_owner_nh.handle_id)
logger.info(u'Imported site {}.'.format(name))
def set_host_user(host):
"""
Tries to set a Uses or Owns relationship between the Host and a Host User if there are none.
"""
user = utils.get_user()
domain = '.'.join(host.data['name'].split('.')[-2:])
relations = host.get_relations()
host_user_name = HOST_USERS_MAP.get(domain, None)
if host_user_name and not (relations.get('Uses', None) or relations.get('Owns', None)):
relation_node_handle = utils.get_unique_node_handle(host_user_name, 'Host User', 'Relation')
if host.meta_type == 'Logical':
helpers.set_user(user, host, relation_node_handle.handle_id)
elif host.meta_type == 'Physical':
helpers.set_owner(user, host, relation_node_handle.handle_id)
logger.info('Host User {user_name} set for host {host_name}.'.format(user_name=host_user_name,
host_name=host.data['name']))
def insert_juniper_bgp_peerings(bgp_peerings):
"""
Inserts all BGP peerings for all routers collected by the juniper_conf producer.
This is to be able to get all the peerings associated to the right interfaces.
"""
for peering in bgp_peerings:
peering_group = peering.get('group', 'Unknown Peering Group')
peering_group_handle = utils.get_unique_node_handle(
peering_group, 'Peering Group', 'Logical', case_insensitive=False)
peering_group_node = peering_group_handle.get_node()
helpers.set_noclook_auto_manage(peering_group_node, True)
peering_type = peering.get('type')
if peering_type == 'internal':
continue # Not implemented
elif peering_type == 'external':
insert_external_bgp_peering(peering, peering_group_node)
def insert_juniper_node(name,
model,
version,
node_type='Router',
hardware=None):
"""
Inserts a physical meta type node of the type Router.
Returns the node created.
"""
logger.info('Processing {name}...'.format(name=name))
user = utils.get_user()
node_handle = utils.get_unique_node_handle(name, node_type, 'Physical')
node = node_handle.get_node()
node_dict = {'name': name, 'model': model, 'version': version}
if hardware:
node_dict['serial_number'] = hardware.get('serial_number')
helpers.dict_update_node(user, node.handle_id, node_dict, node_dict.keys())
helpers.set_noclook_auto_manage(node, True)
return node
def insert_hosts(json_list):
user = utils.get_user()
node_type = 'Host'
meta_type = 'Logical'
sunet_user = utils.get_unique_node_handle('SUNET', 'Host User', 'Relation')
for item in json_list:
name = item['host']['name']
data = item['host'].get('nunoc_cosmos', {})
if not address_is_a(data.get('addresses', []), ALLOWED_NODE_TYPE_SET):
logger.info(
'%s had an address that belongs to something that is not a host',
name)
continue
node_handle = utils.get_unique_node_handle_by_name(
name, node_type, meta_type, ALLOWED_NODE_TYPE_SET)
if not node_handle or node_handle.node_type.type not in ALLOWED_NODE_TYPE_SET:
logger.warning("%s is not in %s", name, ALLOWED_NODE_TYPE_SET)
continue
node = node_handle.get_node()
helpers.update_noclook_auto_manage(node)
properties = {
'ip_addresses': data.get('addresses', []),
'sunet_iaas': data.get('sunet_iaas', False)
}
if data.get('managed_by'):
properties['managed_by'] = data.get('managed_by')
# Set operational state if it is missing
if not node.data.get('operational_state',
None) and properties['ip_addresses']:
properties['operational_state'] = 'In service'
# Update host node
helpers.dict_update_node(user, node.handle_id, properties)
if data.get('sunet_iaas', False):
if node.meta_type == 'Logical':
helpers.set_user(user, node, sunet_user.handle_id)
elif node.meta_type == 'Physical':
helpers.set_owner(user, node, sunet_user.handle_id)
logger.info("%s has been imported", name)
def insert_services(service_dict, host_node, external_check=False):
"""
Takes a dictionary of services and a node id for a host. Gets or creates a
service and makes a Depends_on relationship between the service and host.
Example service_dict:
{"127.0.0.1": {
"tcp": {
"80": {
"conf": "10",
"extrainfo": "",
"name": "http",
"product": "VMware ESXi Server httpd",
"reason": "syn-ack",
"state": "open",
"version": ""
},
"8443": {
"conf": "10",
"extrainfo": "",
"name": "ssl",
"product": "TLS",
"reason": "syn-ack",
"state": "open",
"version": "1.0"
}
}
}
"""
user = utils.get_user()
node_type = "Host Service"
meta_type = 'Logical'
services_locked = host_node.data.get('services_locked', False)
# Expected service data from nmap
property_keys = [
'ip_address',
'protocol',
'port',
'conf',
'name',
'product',
'state',
]
if external_check:
property_keys.extend(['public', 'noclook_last_external_check'])
external_dict = {
'public': True,
'noclook_last_external_check': datetime.now().isoformat()
}
set_not_public(host_node)
for address in service_dict.keys():
for protocol in service_dict[address].keys():
for port in service_dict[address][protocol].keys():
service = service_dict[address][protocol][port]
# Ignore anything but open
if service['state'] == 'open':
service_name = service['name']
if not service_name: # Blank
logger.warn(
'Skipping unknown service on port: %s (%s), address: %s, hostname: %s, data: %s',
port, protocol, address,
host_node.data.get('name'), service)
continue
service_node_handle = utils.get_unique_node_handle(
service_name, node_type, meta_type)
service_node = service_node_handle.get_node()
helpers.update_noclook_auto_manage(service_node)
relationship_properties = {
'ip_address': address,
'protocol': protocol,
'port': port
}
result = host_node.get_host_service(
service_node.handle_id, **relationship_properties)
if not result.get('Depends_on'):
result = host_node.set_host_service(
service_node.handle_id, **relationship_properties)
relationship_id = result.get('Depends_on')[0].get(
'relationship_id')
relationship = nc.get_relationship_model(
nc.graphdb.manager, relationship_id)
created = result.get('Depends_on')[0].get('created')
# Set or update relationship properties
relationship_properties.update(service)
if external_check:
relationship_properties.update(external_dict)
if created:
activitylog.create_relationship(user, relationship)
if services_locked:
logger.warn(
'New open port found for host {name}.'.format(
name=host_node.data['name']))
property_keys.append('rogue_port')
relationship_properties['rogue_port'] = True
logger.info(
'Host Service {host_service_name} using port {port}/{protocol} created.'
.format(
host_service_name=service_node.data['name'],
port=relationship.data['port'],
protocol=relationship.data['protocol']))
if not created:
logger.info(
'Host Service {host_service_name} using port {port}/{protocol} found.'
.format(
host_service_name=service_node.data['name'],
port=relationship.data['port'],
protocol=relationship.data['protocol']))
helpers.update_noclook_auto_manage(relationship)
helpers.dict_update_relationship(user, relationship.id,
relationship_properties,
property_keys)
logger.info(
'{name} {ip_address} {port}/{protocol} processed...'.
format(name=host_node.data['name'],
ip_address=address,
protocol=protocol,
port=port))
def get_peering_partner(peering):
"""
Inserts a new node of the type Peering partner and ensures that this node
is unique for AS number.
Returns the created node.
"""
try:
return PEER_AS_CACHE[peering['as_number']]
except KeyError:
logger.info('Peering Partner {name} not in cache.'.format(
name=peering.get('description')))
pass
user = utils.get_user()
peer_node = None
peer_properties = {'name': 'Missing description', 'as_number': '0'}
# neither description or as_number error and return
if not (peering.get('description') or peering.get('as_number')):
logger.error('Neither AS number nor description in peering %s',
peering)
return None
if peering.get('description'):
peer_properties['name'] = peering.get('description')
if peering.get('as_number'):
peer_properties['as_number'] = peering.get('as_number')
# as number is most important
hits = nc.get_nodes_by_value(nc.graphdb.manager,
prop='as_number',
value=peer_properties['as_number'])
found = 0
for node in hits:
peer_node = nc.get_node_model(nc.graphdb.manager,
node['handle_id'])
helpers.set_noclook_auto_manage(peer_node, True)
if peer_node.data[
'name'] == 'Missing description' and peer_properties[
'name'] != 'Missing description':
helpers.dict_update_node(user, peer_node.handle_id,
peer_properties)
logger.info('Peering Partner {name} fetched.'.format(
name=peer_properties['name']))
found += 1
if found > 1:
logger.error(
'Found more then one Peering Partner with AS number {!s}'.
format(peer_properties['as_number']))
if not peer_node:
# since we have a AS number we will create a new Peering Partner, even if name is missing or exists
node_handle = utils.create_node_handle(peer_properties['name'],
'Peering Partner',
'Relation')
peer_node = node_handle.get_node()
helpers.set_noclook_auto_manage(peer_node, True)
helpers.dict_update_node(user, peer_node.handle_id,
peer_properties, peer_properties.keys())
logger.info('Peering Partner %s AS(%s) created.',
peer_properties['name'], peer_properties['as_number'])
# Handle peer with name only
if not peer_node and peering.get('description'):
# Try and get peer_partners
res = NodeHandle.objects.filter(
node_name__iexact=peer_properties['name'],
node_type__type='Peering Partner').order_by('-modified')
for ph in res:
peer_node = ph.get_node()
break
if not peer_node:
# create
peer_nh = utils.get_unique_node_handle(peer_properties['name'],
'Peering Partner',
'Relation')
peer_node = peer_nh.get_node()
if not peer_node.data.get('as_number'):
# Peering partner did not exist
logger.warning(
'Peering Partner %s without AS number created for peering: %s',
peer_node.data.get('name'), peering)
# AS number is going to be 0, but that is ok
helpers.dict_update_node(user, peer_node.handle_id,
peer_properties, peer_properties.keys())
elif peer_node.data.get('as_number') != '0':
# warn about as number not being in peering
logger.warning(
'Peering found for Peering Partner %s without the AS number %s mentioned. Peering: %s',
peer_properties['name'], peer_node.data.get('as_number'),
peering)
helpers.set_noclook_auto_manage(peer_node, True)
PEER_AS_CACHE[peering['as_number']] = peer_node
return peer_node