def d42_update(dev42, nodes, options, static_opt, from_version='3', puppethost=None):
old_node = str(from_version or '3')[0] <= '3'
# get customer info
customer_name = static_opt.get('customer')
customer_id = str(static_opt.get('customer_id') or '') or None
if (not customer_id and customer_name) or (customer_id and not customer_name):
allcustomers = dev42._get('customers')['Customers']
for cst in allcustomers:
if customer_id and str(cst['id']) == customer_id:
customer_name = cst['name']
break
if customer_name and cst['name'] == customer_name:
customer_id = str(cst['id'])
break
logger.debug("Customer %s: '%s'" % (customer_id, customer_name))
# processing all nodes
for node in nodes:
if 'hostname' not in node:
logger.debug("Skip node: no name found")
continue
node_name = node['hostname']
if options.get('as_node_name').upper() == 'FQDN':
node_name = node.get('fqdn', node_name)
# filtering by attributes
if options.get('node_filter'):
if not node_filter(node, options['node_filter']):
logger.info("Skip node %s: filter not passed" % node_name)
continue # filter not passed
try:
# device = dev42.get_device_by_name(node_name)
# detect memory
totalmem = int(float(node['memorysize_mb']))
# detect HDD
hddcount = 0
hddsize = 0 # first in bytes, then should be converted to Gb
if 'disks' in node:
for p in node['disks'].values():
hddcount += 1
hddsize += int(p.get('size_bytes') or 0)
if hddsize > 0:
hddsize = 1.0 * hddsize / 1000 ** 3 # convert to Gb ( hddsize/ 1024**3 )
nodetype = None
is_virtual = str(node['is_virtual']).lower() == 'true'
virtual_subtype = None
if is_virtual:
is_virtual = 'yes'
nodetype = 'virtual'
virtual_subtype = 'other'
if 'ec2_metadata' in node:
virtual_subtype = 'ec2'
else:
is_virtual = 'no'
cpupower = 0
cpucount = node['physicalprocessorcount']
cpucores = node['processorcount']
cpupowers = cpuf_re.findall(node['processors']['models'][0])
if cpupowers:
cpupower = int(float(cpupowers[0]) * 1000)
data = {
'name': node_name,
'type': nodetype,
'is_it_virtual_host': is_virtual,
'virtual_subtype': virtual_subtype,
'os': node['operatingsystem'],
'osver': node['operatingsystemrelease'],
'memory': totalmem,
'cpucount': cpucount,
'cpucore': cpucores,
'cpupower': cpupower,
'hddcount': hddcount,
'hddsize': hddsize,
'macaddress': node['macaddress'],
'customer': customer_name,
'service_level': static_opt.get('service_level'),
}
logger.debug("Updating node %s" % node_name)
updateinfo = dev42.update_device(**data)
deviceid = updateinfo['msg'][1]
logger.info("Device %s updated/created (id %s)" % (node_name, deviceid))
if puppethost:
cfdata = {
'name': node_name,
'key': 'Puppet Node ID',
'value': node_name,
'notes': 'Puppet Server %s' % puppethost
}
updateinfo = dev42._put('device/custom_field', cfdata)
# Dealing with IPs
device_ips = dev42._get("ips", data={'device': node_name})['ips']
updated_ips = []
if old_node:
# Puppet with version <= 3 has plain structure and does not know about IPv6
for ifsname in node['interfaces'].split(','):
if ifsname == 'lo':
continue # filter out local interface
ipaddr = node['ipaddress_%s' % ifsname.lower()]
if ipaddr.startswith('127.0'):
continue # local loopbacks
macaddr = node['macaddress_%s' % ifsname.lower()]
# update IPv4
ipdata = {
'ipaddress': ipaddr,
'tag': ifsname.replace('_', ' '),
'device': node_name,
'macaddress': macaddr,
}
# logger.debug("IP data: %s" % ipdata)
updateinfo = dev42._post('ips', ipdata)
updated_ips.append(updateinfo['msg'][1])
logger.info("IP %s for device %s updated/created (id %s)" % (ipaddr, node_name, deviceid))
elif node.get('networking'):
# Puppet v4 is more detailed
for ifsname, ifs in node['networking']['interfaces'].items():
if ifsname == 'lo':
continue # filter out local interface
if ifs['ip'].startswith('127.0'):
continue # local loopbacks
# update IPv4
ipdata = {
'ipaddress': ifs['ip'],
'tag': ifsname,
'device': node_name,
'macaddress': ifs['mac'],
}
# logger.debug("IP data: %s" % ipdata)
updateinfo = dev42._post('ips', ipdata)
updated_ips.append(updateinfo['msg'][1])
logger.info("IP %s for device %s updated/created (id %s)" % (ifs['ip'], node_name, deviceid))
# update IPv6
if 'ip6' in ifs and len(ifs['ip6']) > 6:
ipdata = {
'ipaddress': ifs['ip6'],
'tag': ifsname,
'device': node_name,
'macaddress': ifs['mac'],
}
# logger.debug("IP data: %s" % ipdata)
updateinfo = dev42._post('ips', ipdata)
updated_ips.append(updateinfo['msg'][1])
logger.info("IP %s for device %s updated/created (id %s)" % (ifs['ip6'], node_name, deviceid))
# Delete other IPs from the device
if updated_ips:
for d_ip in device_ips:
if d_ip['id'] not in updated_ips:
dev42._delete('ips/%s' % d_ip['id'])
logger.debug("Deleted IP %s (id %s) for device %s (id %s)" %
(d_ip['ip'], d_ip['id'], node_name, deviceid))
except Exception as eee:
logger.exception("Error(%s) updating device %s" % (type(eee), node_name))
def d42_update(dev42,
nodes,
options,
static_opt,
mapping,
from_version='3',
puppethost=None):
old_node = str(from_version or '3')[0] <= '3'
# get customer info
customer_name = static_opt.get('customer')
customer_id = str(static_opt.get('customer_id') or '') or None
if (not customer_id and customer_name) or (customer_id
and not customer_name):
allcustomers = dev42._get('customers')['Customers']
for cst in allcustomers:
if customer_id and str(cst['id']) == customer_id:
customer_name = cst['name']
break
if customer_name and cst['name'] == customer_name:
customer_id = str(cst['id'])
break
logger.debug("Static: Customer ID: %s, Customer Name: '%s'" %
(customer_id, customer_name))
# processing all nodes
for node in nodes:
if 'hostname' not in node:
logger.debug("Skip node: no name found")
continue
if options.get('show_node'):
print(node)
node_name = node['hostname']
if options.get('as_node_name').upper() == 'FQDN':
node_name = node.get('fqdn', node_name)
if node_name == '' or node_name is None:
continue
# filtering by attributes
if options.get('node_filter'):
if not node_filter(node, options['node_filter']):
logger.info("Skip node %s: filter not passed" % node_name)
continue # filter not passed
try:
# device = dev42.get_device_by_name(node_name)
# try to detect memory
try:
totalmem = int(float(node['memorysize_mb']))
except (ValueError, KeyError):
totalmem = ''
# detect HDD
hddcount = 0
hddsize = 0 # first in bytes, then should be converted to Gb
if 'disks' in node:
for p in node['disks'].values():
hddcount += 1
hddsize += int(p.get('size_bytes') or 0)
if hddsize > 0:
hddsize = 1.0 * hddsize / 1000**3 # convert to Gb ( hddsize/ 1024**3 )
nodetype = None
try:
is_virtual = str(node['is_virtual']).lower() == 'true'
except KeyError:
is_virtual = False
virtual_subtype = None
if is_virtual:
nodetype = 'virtual'
virtual_subtype = 'other'
if 'ec2_metadata' in node:
virtual_subtype = 'ec2'
cpupower = 0
try:
cpucount = node['physicalprocessorcount']
cpucores = node['processorcount']
except KeyError:
cpucount = None
cpucores = None
try:
cpupowers = cpuf_re.findall(node['processors']['models'][0])
except TypeError:
cpupowers = None
if cpupowers:
cpupower = int(float(cpupowers[0]) * 1000)
data = {
'name':
node_name,
'type':
nodetype,
'virtual_subtype':
virtual_subtype,
'os':
node['operatingsystem'] if 'operatingsystem' in node else '',
'osver':
node['operatingsystemrelease']
if 'operatingsystemrelease' in node else '',
'osverno':
None, # d42 API mentioned all 3 of these must be included if an update is made
'memory':
totalmem,
'cpucount':
cpucount,
'cpucore':
cpucores,
'cpupower':
cpupower,
'hddcount':
hddcount,
'hddsize':
hddsize,
'macaddress':
node['macaddress'] if 'macaddress' in node else '',
'customer':
customer_name,
'service_level':
static_opt.get('service_level'),
}
if 'serial_no' in node:
data.update({'serial_no': node['serial_no']})
if options.get('hostname_precedence'):
data.update({'new_name': node_name})
if options.get('tags'):
data.update({'tags': options.get('tags')})
logger.debug("Updating node %s" % node_name)
updateinfo = dev42.update_device(**data)
deviceid = updateinfo['msg'][1]
logger.info("Device %s updated/created (id %s)" %
(node_name, deviceid))
if puppethost:
cfdata = {
'name': node_name,
'key': 'Puppet Node ID',
'value': node_name,
'notes': 'Puppet Server %s' % puppethost
}
updateinfo = dev42._put('device/custom_field', cfdata)
global depth
depth = []
res = []
def get_depth(obj):
global depth
for item in obj:
depth.append(item)
if type(obj[item]) == str:
res.append({obj[item]: depth})
depth = []
else:
get_depth(obj[item])
return res
if mapping:
full_depth = get_depth(mapping)
for element in full_depth:
for key in element:
value = None
step = node
try:
for x in element[key]:
step = step[x]
except KeyError:
continue
if type(step) in [str, int]:
value = step
elif type(step) in [list, tuple, dict]:
value = len(step)
cfdata = {
'name': node_name,
'key': key,
'value': value
}
updateinfo = dev42._put('device/custom_field', cfdata)
# Dealing with IPs
device_ips = dev42._get("ips", data={'device': node_name})['ips']
updated_ips = []
if old_node:
# Puppet with version <= 3 has plain structure and does not know about IPv6
for ifsname in node['interfaces'].split(','):
if ifsname == 'lo':
continue # filter out local interface
ipaddr = node['ipaddress_%s' % ifsname.lower()]
if ipaddr.startswith('127.0'):
continue # local loopbacks
if ipaddr.lower().startswith('fe80'):
continue
macaddr = node['macaddress_%s' % ifsname.lower()]
# update IPv4
ipdata = {
'ipaddress': ipaddr,
'tag': ifsname.replace('_', ' '),
'device': node_name,
'macaddress': macaddr,
}
# logger.debug("IP data: %s" % ipdata)
updateinfo = dev42._post('ips', ipdata)
updated_ips.append(updateinfo['msg'][1])
logger.info("IP %s for device %s updated/created (id %s)" %
(ipaddr, node_name, deviceid))
elif node.get('networking'):
# Puppet v4 is more detailed
for ifsname, ifs in node['networking']['interfaces'].items():
if ifsname == 'lo':
continue # filter out local interface
try:
if ifs['ip'].startswith('127.0'):
continue # local loopbacks
except KeyError:
continue
# update IPv4
ipdata = {
'ipaddress': ifs['ip'],
'tag': ifsname,
'device': node_name,
'macaddress': ifs['mac'],
}
# logger.debug("IP data: %s" % ipdata)
updateinfo = dev42._post('ips', ipdata)
updated_ips.append(updateinfo['msg'][1])
logger.info("IP %s for device %s updated/created (id %s)" %
(ifs['ip'], node_name, deviceid))
if 'network' in ifs and 'netmask' in ifs:
# update IPv4 subnet
mask = IPAddress(ifs['netmask']).netmask_bits()
subdata = {
'network': ifs['network'],
'mask_bits': mask
}
# logger.debug("IP data: %s" % ipdata)
updateinfo = dev42._post('subnets', subdata)
updated_ips.append(updateinfo['msg'][1])
logger.info(
"Subnet %s/%s for device %s updated/created (id %s)"
% (ifs['network'], mask, node_name, deviceid))
if 'ip6' in ifs and ifs['ip6'].startswith('fe80::'):
continue # local loopbacks
# update IPv6
if 'ip6' in ifs and len(ifs['ip6']) > 6:
ipdata = {
'ipaddress': ifs['ip6'],
'tag': ifsname,
'device': node_name,
'macaddress': ifs['mac'],
}
# logger.debug("IP data: %s" % ipdata)
try:
updateinfo = dev42._post('ips', ipdata)
updated_ips.append(updateinfo['msg'][1])
logger.info(
"IP %s for device %s updated/created (id %s)" %
(ifs['ip6'], node_name, deviceid))
except device42.Device42HTTPError as e:
print(e)
if 'network6' in ifs and 'netmask6' in ifs:
# update IPv6 subnet
mask = IPAddress(ifs['netmask6']).netmask_bits()
subdata = {
'network': ifs['network6'],
'mask_bits': mask
}
try:
# logger.debug("IP data: %s" % ipdata)
updateinfo = dev42._post('subnets', subdata)
updated_ips.append(updateinfo['msg'][1])
logger.info(
"Subnet %s/%s for device %s updated/created (id %s)"
% (ifs['network6'], mask, node_name,
deviceid))
except device42.Device42HTTPError as e:
print(e)
# Delete other IPs from the device
if updated_ips:
for d_ip in device_ips:
if d_ip['id'] not in updated_ips:
dev42._delete('ips/%s' % d_ip['id'])
logger.debug(
"Deleted IP %s (id %s) for device %s (id %s)" %
(d_ip['ip'], d_ip['id'], node_name, deviceid))
except Exception as eee:
logger.exception("Error(%s) updating device %s" %
(type(eee), node_name))
def d42_update(dev42, nodes, options, static_opt, chefhost=None):
# get customer info
customer_name = static_opt.get('customer')
customer_id = str(static_opt.get('customer_id') or '') or None
if (not customer_id and customer_name) or (customer_id and not customer_name):
allcustomers = dev42._get('customers')['Customers']
for cst in allcustomers:
if customer_id and str(cst['id']) == customer_id:
customer_name = cst['name']
break
if customer_name and cst['name'] == customer_name:
customer_id = str(cst['id'])
break
logger.debug("Customer %s: '%s'" % (customer_id, customer_name))
# processing all nodes
for node in nodes:
if 'hostname' not in node:
logger.debug("Skip node: no name found")
continue
node_name = node['hostname']
if options.get('as_node_name').upper() == 'FQDN':
node_name = node.get('fqdn', node_name)
# filtering by attributes
if options.get('node_filter'):
if not node_filter(node, options['node_filter']):
logger.info("Skip node %s: filter not passed" % node_name)
continue # filter not passed
try:
# device = dev42.get_device_by_name(node_name)
# detect memory
totalmem = '0'
if 'memory' in node:
# linux
totalmem = node['memory']['total']
if totalmem.endswith('kB'):
totalmem = int(totalmem[:-2]) / 1024
elif totalmem.endswith('mB'):
totalmem = int(totalmem[:-2])
elif totalmem.endswith('gB'):
totalmem = int(totalmem[:-2]) * 1024
else:
totalmem = int(totalmem)
else:
# win
totalmem = node.get('kernel', {}).get('cs_info', {}).get('total_physical_memory') or '0'
totalmem = int(totalmem) / (1024 * 1024)
# detect HDD
hddcount = 0
hddsize = 0 # first in bytes, then should be converted to Gb
for devname, dev in node['filesystem'].items():
fstype = dev.get('fs_type') or dev.get('fstype')
if fstype not in ALLOWED_FSTYPES:
continue
hddcount += 1
size = int(dev.get('kb_size', 0)) * 1024
hddsize += size
hddsize = hddsize >> 30 # convert to Gb ( hddsize/ 1024**3 )
nodetype = None
is_virtual = 'no'
virtual_subtype = None
if node.get('virtualization'):
# node['virtualization']['system']
is_virtual = 'yes'
nodetype = 'virtual'
if node.get('kernel', {}).get('os_info', {}).get('registered_user') == 'EC2':
is_virtual = 'yes'
virtual_subtype = 'ec2'
nodetype = 'virtual'
data = {
'name': node_name,
'type': nodetype,
'is_it_virtual_host': is_virtual,
'virtual_subtype': virtual_subtype,
'os': node['platform'],
'osver': node['platform_version'],
'memory': totalmem,
'cpucount': node['cpu']['total'],
'cpucore': node['cpu']['0'].get('cores', 0),
'cpupower': int(float(node['cpu']['0']['mhz'])),
'hddcount': hddcount,
'hddsize': hddsize,
'macaddress': node['macaddress'],
'customer': customer_name,
'service_level': static_opt.get('service_level'),
}
logger.debug("Updating node %s" % node_name)
updateinfo = dev42.update_device(**data)
deviceid = updateinfo['msg'][1]
logger.info("Device %s updated/created (id %s)" % (node_name, deviceid))
if chefhost:
cfdata = {
'name': node_name,
'key': 'Chef Node ID',
'value': node_name,
'notes': 'Chef Server %s' % chefhost
}
updateinfo = dev42._put('device/custom_field', cfdata)
# Dealing with IPs
device_ips = dev42._get("ips", data={'device': node_name})['ips']
updated_ips = []
for ifsname, ifs in node['network']['interfaces'].items():
if ifsname == 'lo':
continue # filter out local interface
if [aip for aip, a in ifs['addresses'].items() if aip.startswith('127.0')]:
continue # local loopbacks
macs = [aip for aip, a in ifs['addresses'].items() if a['family'] == 'lladdr']
macaddr = None
if macs:
macaddr = macs[0]
for nodeip, addr in ifs['addresses'].items():
if addr['family'] == 'lladdr':
continue # filter out mac
# update IP
ipdata = {
'ipaddress': nodeip,
'tag': ifsname,
'device': node_name,
'macaddress': macaddr,
}
# logger.debug("IP data: %s" % ipdata)
updateinfo = dev42._post('ips', ipdata)
updated_ips.append(updateinfo['msg'][1])
logger.info("IP %s for device %s updated/created (id %s)" % (nodeip, node_name, deviceid))
# Delete other IPs from the device
for d_ip in device_ips:
if d_ip['id'] not in updated_ips:
dev42._delete('ips/%s' % d_ip['id'])
logger.debug("Deleted IP %s (id %s) for device %s (id %s)" %
(d_ip['ip'], d_ip['id'], node_name, deviceid))
except Exception as eee:
logger.exception("Error(%s) updating device %s" % (type(eee), node_name))
def d42_insert(dev42, nodes, options, static_opt):
# get customer info
customer_name = static_opt.get('customer')
customer_id = str(static_opt.get('customer_id') or '') or None
if (not customer_id and customer_name) or (customer_id
and not customer_name):
all_customers = dev42._get('customers')['Customers']
for cst in all_customers:
if customer_id and str(cst['id']) == customer_id:
customer_name = cst['name']
break
if customer_name and cst['name'] == customer_name:
customer_id = str(cst['id'])
break
logger.debug("Customer %s: '%s'" % (customer_id, customer_name))
# processing all nodes
for node in [nodes[x] for x in nodes]:
if not node:
logger.debug("Skip node: no proper node data")
continue
if 'nodename' not in node:
logger.debug("Skip node: no name found")
continue
node_name = node['nodename']
if options.get('as_node_name').upper() == 'FQDN':
node_name = node.get('fqdn', node_name)
# filtering by attributes
if options.get('node_filter'):
if not node_filter(node, options['node_filter']):
logger.info("Skip node %s: filter not passed" % node_name)
continue # filter not passed
try:
# device = dev42.get_device_by_name(node_name)
# detect memory
totalmem = int(float(node['mem_total']))
cpupower = 0
cpus = node['num_cpus']
cpupowers = cpuf_re.findall(node['cpu_model'])
if cpupowers:
cpupower = int(float(cpupowers[0]) * 1000)
data = {
'name': node_name,
'os': node['os'],
'osver': node['osrelease'],
'cpupower': cpupower,
'memory': totalmem,
'cpucore': cpus,
'manufacturer': node['manufacturer'],
'customer': customer_name,
'service_level': static_opt.get('service_level')
}
uuid = None
if 'machine_id' in node:
uuid = node['machine_id']
if not uuid and 'uuid' in node:
uuid = node['uuid']
if uuid:
data.update({'uuid': uuid})
serial_no = None
if 'serialnumber' in node:
serial_no = node['serialnumber']
if not serial_no and 'system_serialnumber' in node:
serial_no = node['system_serialnumber']
if serial_no:
data.update({'serial_no': serial_no})
nodetype = 'physical'
virtual_subtype = None
is_virtual = 'no'
if node['virtual'] != nodetype:
is_virtual = 'yes'
nodetype = 'virtual'
if 'virtual_subtype' in node:
virtual_subtype = node['virtual_subtype']
else:
virtual_subtype = node['virtual']
if virtual_subtype is not None:
data.update({'virtual_subtype': virtual_subtype})
data.update({'type': nodetype, 'is_it_virtual_host': is_virtual})
osarch = None
if 'osarch' in node and '64' in node['osarch']:
osarch = 64
if 'osarch' in node and '86' in node['osarch']:
osarch = 32
if osarch is not None:
data.update({'osarch': osarch})
# detect disks
if 'disks' in node:
hdd_count = 0
hdd_size = 0
disks = {}
if node['id'] in node['disks'] and type(
node['disks'][node['id']]) == dict:
# get unique
for disk in node['disks'][node['id']]:
disk = node['disks'][node['id']][disk]
if 'UUID' in disk and disk['UUID'] not in disks:
disks[disk['UUID']] = disk
for disk in disks:
if 'TYPE' in disks[disk] and disks[disk]['TYPE'].lower(
) in ALLOWED_FSTYPES:
hdd_count += 1
if 'usage' in node and node['id'] in node['usage'] and type(
node['usage'][node['id']] == dict):
for disk in node['usage'][node['id']]:
disk = node['usage'][node['id']][disk]
if 'filesystem' in disk and disk[
'filesystem'] in node['disks'][
node['id']] and '1K-blocks' in disk:
hdd_size += int(disk['1K-blocks'])
data.update({
'hddcount': hdd_count,
'hddsize': float(hdd_size) / (1024 * 1024)
})
if 'cpus' in node:
if type(node['cpus'][node['id']]) == dict:
data.update({
'cpucount':
int(node['cpus'][node['id']]['physical id']) + 1
})
if options.get('hostname_precedence'):
data.update({'new_name': node_name})
logger.debug("Updating node %s" % node_name)
updateinfo = dev42.update_device(**data)
deviceid = updateinfo['msg'][1]
logger.info("Device %s updated/created (id %s)" %
(node_name, deviceid))
cfdata = {
'name': node_name,
'key': 'Salt Node ID',
'value': node_name,
'notes': 'Salt Master Server %s' % node['master']
}
dev42._put('device/custom_field', cfdata)
# Dealing with IPs
device_ips = dev42._get("ips", data={'device': node_name})['ips']
updated_ips = []
if node.get('hwaddr_interfaces'):
for ifsname, ifs in node.get('hwaddr_interfaces').items():
if ifsname.startswith('lo'):
continue
dev42._put('device', {
'device_id': deviceid,
'macaddress': ifs
})
if node.get('ip_interfaces') and node.get('hwaddr_interfaces'):
for ifsname, ifs in node.get('ip_interfaces').items():
if ifsname.startswith('lo') or ifsname.startswith(
'tun') or ifsname.startswith('tap'):
continue # filter out local and tunnel
for ip in ifs:
if ip.startswith('127.0'):
continue # local loopbacks
if ip.lower().startswith('fe80'):
continue # local loopbacks
if ifsname not in node.get('hwaddr_interfaces'):
continue
ipdata = {
'ipaddress': ip,
'tag': ifsname,
'device_id': deviceid,
'macaddress':
node.get('hwaddr_interfaces')[ifsname]
}
# logger.debug("IP data: %s" % ipdata)
updateinfo = dev42._post('ips', ipdata)
updated_ips.append(updateinfo['msg'][1])
logger.info(
"IP %s for device %s updated/created (id %s)" %
(ip, node_name, deviceid))
# Delete other IPs from the device
if updated_ips:
for d_ip in device_ips:
if d_ip['id'] not in updated_ips:
dev42._delete('ips/%s/' % d_ip['id'])
logger.debug(
"Deleted IP %s (id %s) for device %s (id %s)" %
(d_ip['ip'], d_ip['id'], node_name, deviceid))
except Exception as e:
logger.exception("Error (%s) updating device %s" %
(type(e), node_name))