def d42_update(dev42, nodes, options, static_opt, from_version='3', puppethost=None): old_node = str(from_version or '3')[0] <= '3' # get customer info customer_name = static_opt.get('customer') customer_id = str(static_opt.get('customer_id') or '') or None if (not customer_id and customer_name) or (customer_id and not customer_name): allcustomers = dev42._get('customers')['Customers'] for cst in allcustomers: if customer_id and str(cst['id']) == customer_id: customer_name = cst['name'] break if customer_name and cst['name'] == customer_name: customer_id = str(cst['id']) break logger.debug("Customer %s: '%s'" % (customer_id, customer_name)) # processing all nodes for node in nodes: if 'hostname' not in node: logger.debug("Skip node: no name found") continue node_name = node['hostname'] if options.get('as_node_name').upper() == 'FQDN': node_name = node.get('fqdn', node_name) # filtering by attributes if options.get('node_filter'): if not node_filter(node, options['node_filter']): logger.info("Skip node %s: filter not passed" % node_name) continue # filter not passed try: # device = dev42.get_device_by_name(node_name) # detect memory totalmem = int(float(node['memorysize_mb'])) # detect HDD hddcount = 0 hddsize = 0 # first in bytes, then should be converted to Gb if 'disks' in node: for p in node['disks'].values(): hddcount += 1 hddsize += int(p.get('size_bytes') or 0) if hddsize > 0: hddsize = 1.0 * hddsize / 1000 ** 3 # convert to Gb ( hddsize/ 1024**3 ) nodetype = None is_virtual = str(node['is_virtual']).lower() == 'true' virtual_subtype = None if is_virtual: is_virtual = 'yes' nodetype = 'virtual' virtual_subtype = 'other' if 'ec2_metadata' in node: virtual_subtype = 'ec2' else: is_virtual = 'no' cpupower = 0 cpucount = node['physicalprocessorcount'] cpucores = node['processorcount'] cpupowers = cpuf_re.findall(node['processors']['models'][0]) if cpupowers: cpupower = int(float(cpupowers[0]) * 1000) data = { 'name': node_name, 'type': nodetype, 'is_it_virtual_host': is_virtual, 'virtual_subtype': virtual_subtype, 'os': node['operatingsystem'], 'osver': node['operatingsystemrelease'], 'memory': totalmem, 'cpucount': cpucount, 'cpucore': cpucores, 'cpupower': cpupower, 'hddcount': hddcount, 'hddsize': hddsize, 'macaddress': node['macaddress'], 'customer': customer_name, 'service_level': static_opt.get('service_level'), } logger.debug("Updating node %s" % node_name) updateinfo = dev42.update_device(**data) deviceid = updateinfo['msg'][1] logger.info("Device %s updated/created (id %s)" % (node_name, deviceid)) if puppethost: cfdata = { 'name': node_name, 'key': 'Puppet Node ID', 'value': node_name, 'notes': 'Puppet Server %s' % puppethost } updateinfo = dev42._put('device/custom_field', cfdata) # Dealing with IPs device_ips = dev42._get("ips", data={'device': node_name})['ips'] updated_ips = [] if old_node: # Puppet with version <= 3 has plain structure and does not know about IPv6 for ifsname in node['interfaces'].split(','): if ifsname == 'lo': continue # filter out local interface ipaddr = node['ipaddress_%s' % ifsname.lower()] if ipaddr.startswith('127.0'): continue # local loopbacks macaddr = node['macaddress_%s' % ifsname.lower()] # update IPv4 ipdata = { 'ipaddress': ipaddr, 'tag': ifsname.replace('_', ' '), 'device': node_name, 'macaddress': macaddr, } # logger.debug("IP data: %s" % ipdata) updateinfo = dev42._post('ips', ipdata) updated_ips.append(updateinfo['msg'][1]) logger.info("IP %s for device %s updated/created (id %s)" % (ipaddr, node_name, deviceid)) elif node.get('networking'): # Puppet v4 is more detailed for ifsname, ifs in node['networking']['interfaces'].items(): if ifsname == 'lo': continue # filter out local interface if ifs['ip'].startswith('127.0'): continue # local loopbacks # update IPv4 ipdata = { 'ipaddress': ifs['ip'], 'tag': ifsname, 'device': node_name, 'macaddress': ifs['mac'], } # logger.debug("IP data: %s" % ipdata) updateinfo = dev42._post('ips', ipdata) updated_ips.append(updateinfo['msg'][1]) logger.info("IP %s for device %s updated/created (id %s)" % (ifs['ip'], node_name, deviceid)) # update IPv6 if 'ip6' in ifs and len(ifs['ip6']) > 6: ipdata = { 'ipaddress': ifs['ip6'], 'tag': ifsname, 'device': node_name, 'macaddress': ifs['mac'], } # logger.debug("IP data: %s" % ipdata) updateinfo = dev42._post('ips', ipdata) updated_ips.append(updateinfo['msg'][1]) logger.info("IP %s for device %s updated/created (id %s)" % (ifs['ip6'], node_name, deviceid)) # Delete other IPs from the device if updated_ips: for d_ip in device_ips: if d_ip['id'] not in updated_ips: dev42._delete('ips/%s' % d_ip['id']) logger.debug("Deleted IP %s (id %s) for device %s (id %s)" % (d_ip['ip'], d_ip['id'], node_name, deviceid)) except Exception as eee: logger.exception("Error(%s) updating device %s" % (type(eee), node_name))
def d42_update(dev42, nodes, options, static_opt, mapping, from_version='3', puppethost=None): old_node = str(from_version or '3')[0] <= '3' # get customer info customer_name = static_opt.get('customer') customer_id = str(static_opt.get('customer_id') or '') or None if (not customer_id and customer_name) or (customer_id and not customer_name): allcustomers = dev42._get('customers')['Customers'] for cst in allcustomers: if customer_id and str(cst['id']) == customer_id: customer_name = cst['name'] break if customer_name and cst['name'] == customer_name: customer_id = str(cst['id']) break logger.debug("Static: Customer ID: %s, Customer Name: '%s'" % (customer_id, customer_name)) # processing all nodes for node in nodes: if 'hostname' not in node: logger.debug("Skip node: no name found") continue if options.get('show_node'): print(node) node_name = node['hostname'] if options.get('as_node_name').upper() == 'FQDN': node_name = node.get('fqdn', node_name) if node_name == '' or node_name is None: continue # filtering by attributes if options.get('node_filter'): if not node_filter(node, options['node_filter']): logger.info("Skip node %s: filter not passed" % node_name) continue # filter not passed try: # device = dev42.get_device_by_name(node_name) # try to detect memory try: totalmem = int(float(node['memorysize_mb'])) except (ValueError, KeyError): totalmem = '' # detect HDD hddcount = 0 hddsize = 0 # first in bytes, then should be converted to Gb if 'disks' in node: for p in node['disks'].values(): hddcount += 1 hddsize += int(p.get('size_bytes') or 0) if hddsize > 0: hddsize = 1.0 * hddsize / 1000**3 # convert to Gb ( hddsize/ 1024**3 ) nodetype = None try: is_virtual = str(node['is_virtual']).lower() == 'true' except KeyError: is_virtual = False virtual_subtype = None if is_virtual: nodetype = 'virtual' virtual_subtype = 'other' if 'ec2_metadata' in node: virtual_subtype = 'ec2' cpupower = 0 try: cpucount = node['physicalprocessorcount'] cpucores = node['processorcount'] except KeyError: cpucount = None cpucores = None try: cpupowers = cpuf_re.findall(node['processors']['models'][0]) except TypeError: cpupowers = None if cpupowers: cpupower = int(float(cpupowers[0]) * 1000) data = { 'name': node_name, 'type': nodetype, 'virtual_subtype': virtual_subtype, 'os': node['operatingsystem'] if 'operatingsystem' in node else '', 'osver': node['operatingsystemrelease'] if 'operatingsystemrelease' in node else '', 'osverno': None, # d42 API mentioned all 3 of these must be included if an update is made 'memory': totalmem, 'cpucount': cpucount, 'cpucore': cpucores, 'cpupower': cpupower, 'hddcount': hddcount, 'hddsize': hddsize, 'macaddress': node['macaddress'] if 'macaddress' in node else '', 'customer': customer_name, 'service_level': static_opt.get('service_level'), } if 'serial_no' in node: data.update({'serial_no': node['serial_no']}) if options.get('hostname_precedence'): data.update({'new_name': node_name}) if options.get('tags'): data.update({'tags': options.get('tags')}) logger.debug("Updating node %s" % node_name) updateinfo = dev42.update_device(**data) deviceid = updateinfo['msg'][1] logger.info("Device %s updated/created (id %s)" % (node_name, deviceid)) if puppethost: cfdata = { 'name': node_name, 'key': 'Puppet Node ID', 'value': node_name, 'notes': 'Puppet Server %s' % puppethost } updateinfo = dev42._put('device/custom_field', cfdata) global depth depth = [] res = [] def get_depth(obj): global depth for item in obj: depth.append(item) if type(obj[item]) == str: res.append({obj[item]: depth}) depth = [] else: get_depth(obj[item]) return res if mapping: full_depth = get_depth(mapping) for element in full_depth: for key in element: value = None step = node try: for x in element[key]: step = step[x] except KeyError: continue if type(step) in [str, int]: value = step elif type(step) in [list, tuple, dict]: value = len(step) cfdata = { 'name': node_name, 'key': key, 'value': value } updateinfo = dev42._put('device/custom_field', cfdata) # Dealing with IPs device_ips = dev42._get("ips", data={'device': node_name})['ips'] updated_ips = [] if old_node: # Puppet with version <= 3 has plain structure and does not know about IPv6 for ifsname in node['interfaces'].split(','): if ifsname == 'lo': continue # filter out local interface ipaddr = node['ipaddress_%s' % ifsname.lower()] if ipaddr.startswith('127.0'): continue # local loopbacks if ipaddr.lower().startswith('fe80'): continue macaddr = node['macaddress_%s' % ifsname.lower()] # update IPv4 ipdata = { 'ipaddress': ipaddr, 'tag': ifsname.replace('_', ' '), 'device': node_name, 'macaddress': macaddr, } # logger.debug("IP data: %s" % ipdata) updateinfo = dev42._post('ips', ipdata) updated_ips.append(updateinfo['msg'][1]) logger.info("IP %s for device %s updated/created (id %s)" % (ipaddr, node_name, deviceid)) elif node.get('networking'): # Puppet v4 is more detailed for ifsname, ifs in node['networking']['interfaces'].items(): if ifsname == 'lo': continue # filter out local interface try: if ifs['ip'].startswith('127.0'): continue # local loopbacks except KeyError: continue # update IPv4 ipdata = { 'ipaddress': ifs['ip'], 'tag': ifsname, 'device': node_name, 'macaddress': ifs['mac'], } # logger.debug("IP data: %s" % ipdata) updateinfo = dev42._post('ips', ipdata) updated_ips.append(updateinfo['msg'][1]) logger.info("IP %s for device %s updated/created (id %s)" % (ifs['ip'], node_name, deviceid)) if 'network' in ifs and 'netmask' in ifs: # update IPv4 subnet mask = IPAddress(ifs['netmask']).netmask_bits() subdata = { 'network': ifs['network'], 'mask_bits': mask } # logger.debug("IP data: %s" % ipdata) updateinfo = dev42._post('subnets', subdata) updated_ips.append(updateinfo['msg'][1]) logger.info( "Subnet %s/%s for device %s updated/created (id %s)" % (ifs['network'], mask, node_name, deviceid)) if 'ip6' in ifs and ifs['ip6'].startswith('fe80::'): continue # local loopbacks # update IPv6 if 'ip6' in ifs and len(ifs['ip6']) > 6: ipdata = { 'ipaddress': ifs['ip6'], 'tag': ifsname, 'device': node_name, 'macaddress': ifs['mac'], } # logger.debug("IP data: %s" % ipdata) try: updateinfo = dev42._post('ips', ipdata) updated_ips.append(updateinfo['msg'][1]) logger.info( "IP %s for device %s updated/created (id %s)" % (ifs['ip6'], node_name, deviceid)) except device42.Device42HTTPError as e: print(e) if 'network6' in ifs and 'netmask6' in ifs: # update IPv6 subnet mask = IPAddress(ifs['netmask6']).netmask_bits() subdata = { 'network': ifs['network6'], 'mask_bits': mask } try: # logger.debug("IP data: %s" % ipdata) updateinfo = dev42._post('subnets', subdata) updated_ips.append(updateinfo['msg'][1]) logger.info( "Subnet %s/%s for device %s updated/created (id %s)" % (ifs['network6'], mask, node_name, deviceid)) except device42.Device42HTTPError as e: print(e) # Delete other IPs from the device if updated_ips: for d_ip in device_ips: if d_ip['id'] not in updated_ips: dev42._delete('ips/%s' % d_ip['id']) logger.debug( "Deleted IP %s (id %s) for device %s (id %s)" % (d_ip['ip'], d_ip['id'], node_name, deviceid)) except Exception as eee: logger.exception("Error(%s) updating device %s" % (type(eee), node_name))
def d42_update(dev42, nodes, options, static_opt, chefhost=None): # get customer info customer_name = static_opt.get('customer') customer_id = str(static_opt.get('customer_id') or '') or None if (not customer_id and customer_name) or (customer_id and not customer_name): allcustomers = dev42._get('customers')['Customers'] for cst in allcustomers: if customer_id and str(cst['id']) == customer_id: customer_name = cst['name'] break if customer_name and cst['name'] == customer_name: customer_id = str(cst['id']) break logger.debug("Customer %s: '%s'" % (customer_id, customer_name)) # processing all nodes for node in nodes: if 'hostname' not in node: logger.debug("Skip node: no name found") continue node_name = node['hostname'] if options.get('as_node_name').upper() == 'FQDN': node_name = node.get('fqdn', node_name) # filtering by attributes if options.get('node_filter'): if not node_filter(node, options['node_filter']): logger.info("Skip node %s: filter not passed" % node_name) continue # filter not passed try: # device = dev42.get_device_by_name(node_name) # detect memory totalmem = '0' if 'memory' in node: # linux totalmem = node['memory']['total'] if totalmem.endswith('kB'): totalmem = int(totalmem[:-2]) / 1024 elif totalmem.endswith('mB'): totalmem = int(totalmem[:-2]) elif totalmem.endswith('gB'): totalmem = int(totalmem[:-2]) * 1024 else: totalmem = int(totalmem) else: # win totalmem = node.get('kernel', {}).get('cs_info', {}).get('total_physical_memory') or '0' totalmem = int(totalmem) / (1024 * 1024) # detect HDD hddcount = 0 hddsize = 0 # first in bytes, then should be converted to Gb for devname, dev in node['filesystem'].items(): fstype = dev.get('fs_type') or dev.get('fstype') if fstype not in ALLOWED_FSTYPES: continue hddcount += 1 size = int(dev.get('kb_size', 0)) * 1024 hddsize += size hddsize = hddsize >> 30 # convert to Gb ( hddsize/ 1024**3 ) nodetype = None is_virtual = 'no' virtual_subtype = None if node.get('virtualization'): # node['virtualization']['system'] is_virtual = 'yes' nodetype = 'virtual' if node.get('kernel', {}).get('os_info', {}).get('registered_user') == 'EC2': is_virtual = 'yes' virtual_subtype = 'ec2' nodetype = 'virtual' data = { 'name': node_name, 'type': nodetype, 'is_it_virtual_host': is_virtual, 'virtual_subtype': virtual_subtype, 'os': node['platform'], 'osver': node['platform_version'], 'memory': totalmem, 'cpucount': node['cpu']['total'], 'cpucore': node['cpu']['0'].get('cores', 0), 'cpupower': int(float(node['cpu']['0']['mhz'])), 'hddcount': hddcount, 'hddsize': hddsize, 'macaddress': node['macaddress'], 'customer': customer_name, 'service_level': static_opt.get('service_level'), } logger.debug("Updating node %s" % node_name) updateinfo = dev42.update_device(**data) deviceid = updateinfo['msg'][1] logger.info("Device %s updated/created (id %s)" % (node_name, deviceid)) if chefhost: cfdata = { 'name': node_name, 'key': 'Chef Node ID', 'value': node_name, 'notes': 'Chef Server %s' % chefhost } updateinfo = dev42._put('device/custom_field', cfdata) # Dealing with IPs device_ips = dev42._get("ips", data={'device': node_name})['ips'] updated_ips = [] for ifsname, ifs in node['network']['interfaces'].items(): if ifsname == 'lo': continue # filter out local interface if [aip for aip, a in ifs['addresses'].items() if aip.startswith('127.0')]: continue # local loopbacks macs = [aip for aip, a in ifs['addresses'].items() if a['family'] == 'lladdr'] macaddr = None if macs: macaddr = macs[0] for nodeip, addr in ifs['addresses'].items(): if addr['family'] == 'lladdr': continue # filter out mac # update IP ipdata = { 'ipaddress': nodeip, 'tag': ifsname, 'device': node_name, 'macaddress': macaddr, } # logger.debug("IP data: %s" % ipdata) updateinfo = dev42._post('ips', ipdata) updated_ips.append(updateinfo['msg'][1]) logger.info("IP %s for device %s updated/created (id %s)" % (nodeip, node_name, deviceid)) # Delete other IPs from the device for d_ip in device_ips: if d_ip['id'] not in updated_ips: dev42._delete('ips/%s' % d_ip['id']) logger.debug("Deleted IP %s (id %s) for device %s (id %s)" % (d_ip['ip'], d_ip['id'], node_name, deviceid)) except Exception as eee: logger.exception("Error(%s) updating device %s" % (type(eee), node_name))
def d42_insert(dev42, nodes, options, static_opt): # get customer info customer_name = static_opt.get('customer') customer_id = str(static_opt.get('customer_id') or '') or None if (not customer_id and customer_name) or (customer_id and not customer_name): all_customers = dev42._get('customers')['Customers'] for cst in all_customers: if customer_id and str(cst['id']) == customer_id: customer_name = cst['name'] break if customer_name and cst['name'] == customer_name: customer_id = str(cst['id']) break logger.debug("Customer %s: '%s'" % (customer_id, customer_name)) # processing all nodes for node in [nodes[x] for x in nodes]: if not node: logger.debug("Skip node: no proper node data") continue if 'nodename' not in node: logger.debug("Skip node: no name found") continue node_name = node['nodename'] if options.get('as_node_name').upper() == 'FQDN': node_name = node.get('fqdn', node_name) # filtering by attributes if options.get('node_filter'): if not node_filter(node, options['node_filter']): logger.info("Skip node %s: filter not passed" % node_name) continue # filter not passed try: # device = dev42.get_device_by_name(node_name) # detect memory totalmem = int(float(node['mem_total'])) cpupower = 0 cpus = node['num_cpus'] cpupowers = cpuf_re.findall(node['cpu_model']) if cpupowers: cpupower = int(float(cpupowers[0]) * 1000) data = { 'name': node_name, 'os': node['os'], 'osver': node['osrelease'], 'cpupower': cpupower, 'memory': totalmem, 'cpucore': cpus, 'manufacturer': node['manufacturer'], 'customer': customer_name, 'service_level': static_opt.get('service_level') } uuid = None if 'machine_id' in node: uuid = node['machine_id'] if not uuid and 'uuid' in node: uuid = node['uuid'] if uuid: data.update({'uuid': uuid}) serial_no = None if 'serialnumber' in node: serial_no = node['serialnumber'] if not serial_no and 'system_serialnumber' in node: serial_no = node['system_serialnumber'] if serial_no: data.update({'serial_no': serial_no}) nodetype = 'physical' virtual_subtype = None is_virtual = 'no' if node['virtual'] != nodetype: is_virtual = 'yes' nodetype = 'virtual' if 'virtual_subtype' in node: virtual_subtype = node['virtual_subtype'] else: virtual_subtype = node['virtual'] if virtual_subtype is not None: data.update({'virtual_subtype': virtual_subtype}) data.update({'type': nodetype, 'is_it_virtual_host': is_virtual}) osarch = None if 'osarch' in node and '64' in node['osarch']: osarch = 64 if 'osarch' in node and '86' in node['osarch']: osarch = 32 if osarch is not None: data.update({'osarch': osarch}) # detect disks if 'disks' in node: hdd_count = 0 hdd_size = 0 disks = {} if node['id'] in node['disks'] and type( node['disks'][node['id']]) == dict: # get unique for disk in node['disks'][node['id']]: disk = node['disks'][node['id']][disk] if 'UUID' in disk and disk['UUID'] not in disks: disks[disk['UUID']] = disk for disk in disks: if 'TYPE' in disks[disk] and disks[disk]['TYPE'].lower( ) in ALLOWED_FSTYPES: hdd_count += 1 if 'usage' in node and node['id'] in node['usage'] and type( node['usage'][node['id']] == dict): for disk in node['usage'][node['id']]: disk = node['usage'][node['id']][disk] if 'filesystem' in disk and disk[ 'filesystem'] in node['disks'][ node['id']] and '1K-blocks' in disk: hdd_size += int(disk['1K-blocks']) data.update({ 'hddcount': hdd_count, 'hddsize': float(hdd_size) / (1024 * 1024) }) if 'cpus' in node: if type(node['cpus'][node['id']]) == dict: data.update({ 'cpucount': int(node['cpus'][node['id']]['physical id']) + 1 }) if options.get('hostname_precedence'): data.update({'new_name': node_name}) logger.debug("Updating node %s" % node_name) updateinfo = dev42.update_device(**data) deviceid = updateinfo['msg'][1] logger.info("Device %s updated/created (id %s)" % (node_name, deviceid)) cfdata = { 'name': node_name, 'key': 'Salt Node ID', 'value': node_name, 'notes': 'Salt Master Server %s' % node['master'] } dev42._put('device/custom_field', cfdata) # Dealing with IPs device_ips = dev42._get("ips", data={'device': node_name})['ips'] updated_ips = [] if node.get('hwaddr_interfaces'): for ifsname, ifs in node.get('hwaddr_interfaces').items(): if ifsname.startswith('lo'): continue dev42._put('device', { 'device_id': deviceid, 'macaddress': ifs }) if node.get('ip_interfaces') and node.get('hwaddr_interfaces'): for ifsname, ifs in node.get('ip_interfaces').items(): if ifsname.startswith('lo') or ifsname.startswith( 'tun') or ifsname.startswith('tap'): continue # filter out local and tunnel for ip in ifs: if ip.startswith('127.0'): continue # local loopbacks if ip.lower().startswith('fe80'): continue # local loopbacks if ifsname not in node.get('hwaddr_interfaces'): continue ipdata = { 'ipaddress': ip, 'tag': ifsname, 'device_id': deviceid, 'macaddress': node.get('hwaddr_interfaces')[ifsname] } # logger.debug("IP data: %s" % ipdata) updateinfo = dev42._post('ips', ipdata) updated_ips.append(updateinfo['msg'][1]) logger.info( "IP %s for device %s updated/created (id %s)" % (ip, node_name, deviceid)) # Delete other IPs from the device if updated_ips: for d_ip in device_ips: if d_ip['id'] not in updated_ips: dev42._delete('ips/%s/' % d_ip['id']) logger.debug( "Deleted IP %s (id %s) for device %s (id %s)" % (d_ip['ip'], d_ip['id'], node_name, deviceid)) except Exception as e: logger.exception("Error (%s) updating device %s" % (type(e), node_name))