def insert_hosts(json_list):
user = utils.get_user()
node_type = 'Host'
meta_type = 'Logical'
sunet_user = utils.get_unique_node_handle('SUNET', 'Host User', 'Relation')
for item in json_list:
name = item['host']['name']
data = item['host'].get('nunoc_cosmos', {})
if not address_is_a(data.get('addresses', []), ALLOWED_NODE_TYPE_SET):
logger.info(
'%s had an address that belongs to something that is not a host',
name)
continue
node_handle = utils.get_unique_node_handle_by_name(
name, node_type, meta_type, ALLOWED_NODE_TYPE_SET)
if not node_handle or node_handle.node_type.type not in ALLOWED_NODE_TYPE_SET:
logger.warning("%s is not in %s", name, ALLOWED_NODE_TYPE_SET)
continue
node = node_handle.get_node()
helpers.update_noclook_auto_manage(node)
properties = {
'ip_addresses': data.get('addresses', []),
'sunet_iaas': data.get('sunet_iaas', False)
}
if data.get('managed_by'):
properties['managed_by'] = data.get('managed_by')
# Set operational state if it is missing
if not node.data.get('operational_state',
None) and properties['ip_addresses']:
properties['operational_state'] = 'In service'
# Update host node
helpers.dict_update_node(user, node.handle_id, properties)
if data.get('sunet_iaas', False):
if node.meta_type == 'Logical':
helpers.set_user(user, node, sunet_user.handle_id)
elif node.meta_type == 'Physical':
helpers.set_owner(user, node, sunet_user.handle_id)
logger.info("%s has been imported", name)
def handle(self, *args, **options):
user = User.objects.get(username='******')
utcnow = datetime.utcnow()
last_month = utcnow - relativedelta(months=1)
services = getattr(django_settings, 'NETAPP_REPORT_SETTINGS', [])
for service in services:
service_node = nc.get_unique_node_by_name(nc.graphdb.manager,
service['service_id'],
'Service')
monthly_dict = json.loads(
service_node.data.get('netapp_storage_monthly', '{}'))
monthly_dict.setdefault(str(last_month.year), {})[str(last_month.month)] = \
service_node.data.get('netapp_storage_sum', 0.0)
property_dict = {
'netapp_storage_monthly': json.dumps(monthly_dict)
}
helpers.dict_update_node(user, service_node.handle_id,
property_dict, property_dict.keys())
self.stdout.write(
'Monthly netapp usage stored for service {}.'.format(
service['service_id']))
def insert_site(site_dict):
"""
:param site_dict: data
:type site_dict: dict
:return: None
:rtype: None
Expected dict
{
u'name': u'',
u'area': u'',
u'country': u'',
u'longitude': u'',
u'node_type': u'Site',
u'meta_type': u'Location',
u'site_owner': u'',
u'latitude': u'',
u'site_type': u''
}
"""
user = utils.get_user()
# Remove meta data and relationship data from the site dict
name = site_dict.pop('name')
node_type = site_dict.pop('node_type')
meta_type = site_dict.pop('meta_type')
site_owner = site_dict.pop('site_owner')
# Get or create Site
site_nh = utils.get_unique_node_handle(name, node_type, meta_type)
# Set or update node properties
helpers.dict_update_node(user, site_nh.handle_id, site_dict,
site_dict.keys())
if site_owner:
# Get or create Site owner
site_owner_nh = utils.get_unique_node_handle(site_owner, 'Site Owner',
'Relation')
# Set relationship to site owner
helpers.set_responsible_for(user, site_nh.get_node(),
site_owner_nh.handle_id)
logger.info(u'Imported site {}.'.format(name))
def insert(json_list):
"""
Inserts the data loaded from the json files created by
the nerds producer cfengine_reports.
"cfengine_report": [
{
"last_verified_(gmt_+00:00)": "06-10-2013 16:55",
"promisehandle": "system_administration_methods_syslog_conf",
"promisestatus": "kept"
},
{
"last_verified_(gmt_+00:00)": "06-10-2013 16:55",
"promisehandle": "system_administration_methods_scsi_timeout",
"promisestatus": "notkept"
},
]
"""
user = utils.get_user()
for i in json_list:
node = None
name = i['host']['name'].lower()
i['host']['name'] = name
promises = i['host']['cfengine_report']
try:
node = nc.get_unique_node_by_name(nc.graphdb.manager, name, 'Host')
except MultipleNodesReturned as e:
logger.error(e)
if node:
host_properties = {}
for promise in promises:
if promise['promisehandle'] in CFENGINE_MAP.keys():
promise_name = promise['promisehandle']
promise_status = promise['promisestatus']
host_properties.update(
CFENGINE_MAP[promise_name][promise_status])
helpers.dict_update_node(user, node.handle_id, host_properties,
host_properties.keys())
logger.info('{name} done.'.format(name=name))
def insert_snap(json_list):
"""
Inserts snap metadata as Hosts.
"""
user = utils.get_user()
node_type = "Host"
for data in json_list:
# Handle nerds data
try:
d = data['host']['snap_metadata']
except KeyError:
d = data
name = d['hostname'].lower()
logger.info('{} loaded'.format(name))
meta_type = 'Logical' if d.get('virtual') else 'Physical'
# find host first hostname, then ip? else create
node_handle = utils.get_unique_node_handle_by_name(
name, node_type, meta_type, ALLOWED_NODE_TYPE_SET)
# Check it is a host
if not node_handle or node_handle.node_type.type not in ALLOWED_NODE_TYPE_SET:
logger.info('{} is not a {} skipping.'.format(
name, ALLOWED_NODE_TYPE_SET))
continue
# Update host
node = node_handle.get_node()
# change from logical to physical if needed?
if node_handle.node_meta_type == 'Logical' and not d.get('virtual'):
logger.warning(
'Converting {} from logical to physical'.format(name))
helpers.logical_to_physical(user, node.handle_id)
helpers.update_noclook_auto_manage(node)
managed_by = 'Puppet' if d.get('managed') else 'Manual'
responsible = d.get('responsible', 'SEI')
properties = {
'os': d['os'],
'managed_by': managed_by,
'responsible_group': responsible,
'description': d.get('description')
}
if d.get('network'):
ipv4 = [
n['ip'].split('/')[0] for n in d.get('network', [])
if 'ip' in n
]
ipv4_service = [
n['service_ip'].split('/')[0] for n in d.get('network', [])
if 'service_ip' in n
]
ipv6 = [
n['ipv6'].split('/')[0] for n in d.get('network', [])
if 'ipv6' in n
]
ipv6_service = [
n['service_ipv6'].split('/')[0] for n in d.get('network', [])
if 'service_ipv6' in n
]
properties[
'ip_addresses'] = ipv4 + ipv4_service + ipv6 + ipv6_service
if d.get('managed'):
# Dont default to False
properties['syslog'] = True
helpers.dict_update_node(user, node.handle_id, properties,
properties.keys())
logger.info('{} has been imported'.format(name))
def get_peering_partner(peering):
"""
Inserts a new node of the type Peering partner and ensures that this node
is unique for AS number.
Returns the created node.
"""
try:
return PEER_AS_CACHE[peering['as_number']]
except KeyError:
logger.info('Peering Partner {name} not in cache.'.format(
name=peering.get('description')))
pass
user = utils.get_user()
peer_node = None
peer_properties = {'name': 'Missing description', 'as_number': '0'}
# neither description or as_number error and return
if not (peering.get('description') or peering.get('as_number')):
logger.error('Neither AS number nor description in peering %s',
peering)
return None
if peering.get('description'):
peer_properties['name'] = peering.get('description')
if peering.get('as_number'):
peer_properties['as_number'] = peering.get('as_number')
# as number is most important
hits = nc.get_nodes_by_value(nc.graphdb.manager,
prop='as_number',
value=peer_properties['as_number'])
found = 0
for node in hits:
peer_node = nc.get_node_model(nc.graphdb.manager,
node['handle_id'])
helpers.set_noclook_auto_manage(peer_node, True)
if peer_node.data[
'name'] == 'Missing description' and peer_properties[
'name'] != 'Missing description':
helpers.dict_update_node(user, peer_node.handle_id,
peer_properties)
logger.info('Peering Partner {name} fetched.'.format(
name=peer_properties['name']))
found += 1
if found > 1:
logger.error(
'Found more then one Peering Partner with AS number {!s}'.
format(peer_properties['as_number']))
if not peer_node:
# since we have a AS number we will create a new Peering Partner, even if name is missing or exists
node_handle = utils.create_node_handle(peer_properties['name'],
'Peering Partner',
'Relation')
peer_node = node_handle.get_node()
helpers.set_noclook_auto_manage(peer_node, True)
helpers.dict_update_node(user, peer_node.handle_id,
peer_properties, peer_properties.keys())
logger.info('Peering Partner %s AS(%s) created.',
peer_properties['name'], peer_properties['as_number'])
# Handle peer with name only
if not peer_node and peering.get('description'):
# Try and get peer_partners
res = NodeHandle.objects.filter(
node_name__iexact=peer_properties['name'],
node_type__type='Peering Partner').order_by('-modified')
for ph in res:
peer_node = ph.get_node()
break
if not peer_node:
# create
peer_nh = utils.get_unique_node_handle(peer_properties['name'],
'Peering Partner',
'Relation')
peer_node = peer_nh.get_node()
if not peer_node.data.get('as_number'):
# Peering partner did not exist
logger.warning(
'Peering Partner %s without AS number created for peering: %s',
peer_node.data.get('name'), peering)
# AS number is going to be 0, but that is ok
helpers.dict_update_node(user, peer_node.handle_id,
peer_properties, peer_properties.keys())
elif peer_node.data.get('as_number') != '0':
# warn about as number not being in peering
logger.warning(
'Peering found for Peering Partner %s without the AS number %s mentioned. Peering: %s',
peer_properties['name'], peer_node.data.get('as_number'),
peering)
helpers.set_noclook_auto_manage(peer_node, True)
PEER_AS_CACHE[peering['as_number']] = peer_node
return peer_node
def test_populated_site(self):
# Setup test data
site = helpers.create_unique_node_handle(self.user, "Test site",
"site", "Location")
site_node = site.get_node()
rack = helpers.get_generic_node_handle(self.user, "A.01", "rack",
"Location")
helpers.set_has(self.user, site_node, rack.handle_id)
odf = helpers.create_unique_node_handle(self.user, "NI-TEST-ODF-01",
"odf", "Physical")
odf_node = odf.get_node()
helpers.dict_update_node(self.user, odf.handle_id, {"max_ports": 24})
helpers.set_location(self.user, odf_node, rack.handle_id)
helpers.create_port(odf_node, "1", self.user)
helpers.create_port(odf_node, "2", self.user)
helpers.create_port(odf_node, "3", self.user)
router = helpers.create_unique_node_handle(
self.user, "ni-test.routers.nordu.net", "router", "Physical")
router_node = router.get_node()
helpers.dict_update_node(self.user, router.handle_id, {
"operational_state": "Testing",
"rack_units": 2
})
helpers.set_location(self.user, router_node, rack.handle_id)
odf2 = helpers.get_generic_node_handle(self.user, "NI-TEST-ODF-02",
"odf", "Physical")
odf2_node = odf2.get_node()
helpers.set_location(self.user, odf2_node, site.handle_id)
# Done setting up testdata
resp = self.client.get(self.get_full_url(site) + "export")
self.assertEquals('application/json', resp['Content-Type'])
result = resp.json()
# verify data
self.assertEquals(2, len(result))
self.assertDictContainsSubset({
'name': 'A.01',
'node_type': 'Rack'
}, result[0])
self.assertDictContainsSubset(
{
'node_type': 'ODF',
'name': 'NI-TEST-ODF-02'
}, result[1])
# Check racked equipment
rack_equp = result[0]['children']
self.assertEquals(2, len(rack_equp))
odf1_result = rack_equp[0]
self.assertDictContainsSubset(
{
'node_type': 'ODF',
'name': 'NI-TEST-ODF-01',
'max_ports': 24
}, odf1_result)
# Check ODF ports
odf1_ports = odf1_result['children']
self.assertEquals(3, len(odf1_ports))
self.assertDictContainsSubset(
{
'node_type': 'Port',
'name': '1',
'port_type': ''
}, odf1_ports[0])
self.assertDictContainsSubset(
{
'node_type': 'Port',
'name': '2',
'description': ''
}, odf1_ports[1])
self.assertDictContainsSubset({
'node_type': 'Port',
'name': '3'
}, odf1_ports[2])
# Check router
router = rack_equp[1]
self.assertDictContainsSubset(
{
'node_type': 'Router',
'name': 'ni-test.routers.nordu.net'
}, router)