def hostgroups_summary(foreman, user, passwd): """ Show a summary of the current used and unused profiles and machines :param foreman: URL to the foreman server :param user: username to login into Foreman :param passwd: Password to use when logging in """ frm = Foreman(foreman, (user, passwd)) hgdict = {} for hg in frm.index_hostgroups(per_page=900): hgdict[hg['hostgroup']['id']] = hg['hostgroup']['name'] notused = dict(hgdict.iteritems()) groups = [] ## Available hosts = frm.index_hosts(per_page=1000) puts(green("Total Available hosts: ", True) + green(len(hosts))) puts(blue("Available hosts by hostgroup:", True)) for next_host in hosts: gid = next_host['host']['hostgroup_id'] if gid: groups.append((hgdict[gid], gid)) gid in notused and notused.pop(gid) else: groups.append(('No group', 'None')) if COUNTER: groups_count = dict(Counter(groups)).items() else: dict([(item, groups.count(item)) for item in sorted(set(groups))]).items() groups_count.sort() for group, count in groups_count: gname, gid = group if gname != 'No group': puts(blue("\t%s (id=%d) = %s" % (gname, gid, white(count)))) else: puts(blue("\tNo group = %s" % white(count))) ## Unused puts(blue("Unused hostgroups:", True)) for gid, name in notused.iteritems(): puts(blue("\t%s (id=%s)" % (name, gid)))
from getpass import getpass import psycopg2 from foreman.client import Foreman import csv, time # create log file logfile = "log" + time.strftime("_%m_%d_%Y-%H_%M_%S") + ".csv" with open(logfile, "w") as f: writer = csv.writer(f) writer.writerow(["hostname", "hostgroup"]) # create connection object to foreman and object for all nodes username = raw_input("Enter username: "******"dbname='puppetdb' user='******' host='devitpuppet.tsi.lan' password='******'") cur = conn.cursor() cur.execute("""select h.name,hg.name from hosts h left join hostgroups hg on h.hostgroup_id=hg.id""") rows = cur.fetchall() # iterate through foreman nodes and add to necessary hostgroup for node in nodes:
class Client(object): def __init__(self, config_path, debug=False, version='1', log=None): self.config = utils.get_config(config_path) self.logger = utils.get_logger(__name__, self.config, debug, log) config = self.get_config_section('foreman') self.logger.debug('=> config file: %s' % config_path) self.logger.debug('=> foreman url: %s' % config['url']) self.foreman = Foreman(config['url'], (config['user'], config['password']), api_version=2, version=version, verify=False) def get_config(self, section, option): try: value = self.config.get(section, option) return value except ConfigParser.NoOptionError: self.logger.debug('=> config file section [%s] missing option %s' % (section, option)) except ConfigParser.NoSectionError: self.logger.debug('=> config file missing section %s' % section) return None def get_config_section(self, section): try: openstack = self.config.items(section) except ConfigParser.NoSectionError: self.logger.exception('missing [%s]' % section) self.logger.critical('Could not find section [%s] in %s', section, self.config_path) sys.exit(1) return dict(openstack) def get_logger(self): return self.logger def get_client(self): return self.foreman def get_compute_resources(self): resources = self.foreman.index_computeresources() found_resources = dict({}) for r in resources['results']: found_resources[r['name']] = r['id'] return found_resources def get_host(self, host): host = self.__set_host(host) return self.foreman.show_hosts(id=host) def set_host_build(self, host, build=True): host = self.__set_host(host) if len(self.foreman.show_hosts(id=host)) > 0: self.foreman.update_hosts(id=host, host={'build': build}) def get_hosts(self, search=None): hosts = self.foreman.index_hosts() self.logger.debug("=> fetch %s page(s) with a total of %s hosts" % (hosts['page'], hosts['total'])) return hosts def create_host(self, host): if 'name' not in host: self.logger.critical('host dict missing name') return self.logger.debug('=> create new host %s' % host['name']) result = self.foreman.create_host(host) self.logger.debug('=> host created: %s' % result) def create_node(self, name, node_data, region, dry_run=False): if self.get_host(name): self.logger.debug('=> node %s found, dropping create' % name) return found_resources = self.get_compute_resources() host = dict() host['name'] = name host['build'] = self.__get_node_data('build', node_data, '1') host['hostgroup_id'] = self.__get_node_data('hostgroup', node_data, '1') host['compute_profile_id'] = self.__get_node_data('compute_profile', node_data, '1') host['interfaces_attributes'] = self.__get_node_data( 'interfaces_attributes', node_data, {}) host['compute_attributes'] = self.__get_node_data( 'compute_attributes', node_data, {}) host['host_parameters_attributes'] = self.__get_node_data( 'host_parameters_attributes', node_data, {}) if 'mac' in node_data: host['mac'] = node_data['mac'] if 'compute_resource' in node_data: compute_resource = '%s-%s' % (region, node_data['compute_resource']) if compute_resource in found_resources: host['compute_resource_id'] = found_resources[compute_resource] else: self.logger.critical('=> compute resource %s not found' % compute_resource) return elif 'mac' not in node_data: self.logger.critical('=> mac or compute resource are mandatory for %s' % name) return if not dry_run: result = self.foreman.create_hosts(host) if not result: self.log_error('Could not create host. Check production.log on foreman host!') return if 'mac' not in node_data: self.foreman.hosts.power(id=result['name'], power_action='start') self.logger.debug('=> create host %s' % result) else: self.logger.debug('=> dry run: host config %s' % host) def __set_host(self, host): if not host: self.host = None return domain = self.config.get('openstack', 'domain') if domain and not '.' in host: self.logger.debug("=> prepend %s to %s" % (domain, host)) host = host + '.' + domain return host @staticmethod def log_error(msg, code=0): sys.stderr.write("%s\n" % msg) if code > 0: sys.exit(code) @staticmethod def __get_node_data(var, node_data, default=None): if var in node_data: return node_data[var] else: return default
class ForemanClient(Client): per_page = 100 def __init__(self, config_path, debug=False, version='1', log=None): super(ForemanClient, self).__init__(config_path, debug, log) self.logger.debug('=> config file: %s' % config_path) foreman_url = self.get_config('foreman', 'url') self.logger.debug('=> foreman url: %s' % foreman_url) foreman_user = self.get_config('foreman', 'user') foreman_password = self.get_config('foreman', 'password') self.foreman = Foreman(foreman_url, (foreman_user, foreman_password), api_version=2, version=version, verify=False) def set_per_page(self, per_page): self.per_page = per_page def get_config(self, section, option): try: value = self.config.get(section, option) return value except ConfigParser.NoOptionError: self.logger.debug('=> config file section [%s] missing option %s' % (section, option)) except ConfigParser.NoSectionError: self.logger.debug('=> config file missing section %s' % section) return None def get_config_section(self, section): try: openstack = self.config.items(section) except ConfigParser.NoSectionError: self.logger.debug('missing [%s]' % section) self.logger.debug('Could not find section [%s] in %s', section, self.config_path) sys.exit(1) return dict(openstack) def get_logger(self): return self.logger def get_client(self): return self.foreman def get_compute_resources(self): resources = self.foreman.index_computeresources() found_resources = dict({}) for r in resources['results']: found_resources[r['name']] = r['id'] return found_resources def get_compute_profiles(self): profiles = self.foreman.index_computeprofiles() found_profiles = dict({}) for p in profiles['results']: found_profiles[p['name']] = p['id'] return found_profiles def get_profile_id(self, profile_name): profile = self.foreman.show_computeprofiles(profile_name) return profile['id'] def get_host(self, host): host = self.__set_host(host) return self.foreman.show_hosts(id=host) def get_fact(self, host, fact): host = self.__set_host(host) facts = self.get_facts(host) fact = facts['results'][host][fact] return fact def get_facts(self, host_id): host = self.__set_host(host_id) return self.foreman.hosts.fact_values_index(host_id=host, per_page=self.per_page) def set_host_build(self, host, build=True): host = self.__set_host(host) if len(self.foreman.show_hosts(id=host)) > 0: self.foreman.update_hosts(id=host, host={'build': build}) def get_hosts(self, search=None): hosts = self.foreman.index_hosts(per_page=self.per_page) self.logger.debug("=> fetch %s page(s) with a total of %s hosts" % (hosts['page'], hosts['total'])) return hosts def create_host(self, host): if 'name' not in host: self.logger.debug('host dict missing name') return self.logger.debug('=> create new host %s' % host['name']) result = self.foreman.create_host(host) self.logger.debug('=> host created: %s' % result) def create_node(self, name, node_data, region): if self.get_host(name): self.logger.debug('=> node %s found, dropping create' % name) return found_resources = self.get_compute_resources() host = dict() host['name'] = name host['build'] = self.__get_node_data('build', node_data, '1') host['hostgroup_id'] = self.__get_node_data('hostgroup', node_data, '1') host['compute_profile_id'] = self.get_profile_id( self.__get_node_data('compute_profile', node_data, 'small')) host['interfaces_attributes'] = self.__get_node_data( 'interfaces_attributes', node_data, {}) host['compute_attributes'] = self.__get_node_data( 'compute_attributes', node_data, {}) host['host_parameters_attributes'] = self.__get_node_data( 'host_parameters_attributes', node_data, {}) if 'mac' in node_data: host['mac'] = node_data['mac'] if 'compute_resource' in node_data: compute_resource = '%s-%s' % (region, node_data['compute_resource']) if compute_resource in found_resources: host['compute_resource_id'] = found_resources[compute_resource] else: self.logger.debug('=> compute resource %s not found' % compute_resource) return elif 'mac' not in node_data: self.logger.debug('=> mac or compute resource are mandatory for %s' % name) return if not self.dry_run: result = self.foreman.create_hosts(host) if not result: self.log_error('Could not create host. Check production.log on foreman host!') return if 'mac' not in node_data: self.foreman.hosts.power(id=result['name'], power_action='start') self.logger.debug('=> create host %s' % result) else: self.logger.debug('=> dry run: host config %s' % host) def delete_node(self, host): host = self.__set_host(host) if not self.dry_run: result = self.foreman.destroy_hosts(host) if not result: self.log_error('Could not delete host.') return self.logger.debug('=> deleted node %s' % host) else: self.logger.debug('=> dry run: deleted node %s' % host) def __set_host(self, host): if not host: self.host = None return domain = self.config.get('openstack', 'domain') if domain and not '.' in host: self.logger.debug("=> prepend %s to %s" % (domain, host)) host = host + '.' + domain return host @staticmethod def log_error(msg, code=0): sys.stderr.write("%s\n" % msg) if code > 0: sys.exit(code) @staticmethod def __get_node_data(var, node_data, default=None): if var in node_data: return node_data[var] else: return default
def clean_ds(): """ This is a 'one time use' method that will clear from the DS all instances that doesn't still exist """ # Retrieve config from ENV # Stats saved = 0 deleted = 0 # connect to Foreman and ForemanProxy f = Foreman(FOREMAN_URL, (FOREMAN_USER, FOREMAN_PASSWORD), api_version=2) # Connect to the DS try: ds = AwsDs(LDAP_HOST, COMPUTERS_BASE_DN, BIND_USER_DN, BIND_PASSWORD) except ldap.INVALID_CREDENTIALS: raise "Your username or password is incorrect." # Get all host from foreman page = 1 result = [] last_len = 1 while last_len > 0: tmp_result = f.index_hosts(per_page="1000", page=str(page))['results'] last_len = len(tmp_result) page += 1 result += tmp_result foreman_hosts = {host["certname"]: host["ip"] for host in result} # Get all ds computer ds_computers = [] for c_dn, attr in ds.computers: if 'dNSHostName' in attr and re.match('.*\.cloud\.coveo\.com$', attr['dNSHostName'][0]): ds_computers.append(attr['dNSHostName'][0].lower()) continue else: ds_computers.append(build_from_cn(attr['cn'][0])) to_delete = ds_computers """ to_delete = [] # (Optional) filter which instances should be cleaned for ds_computer in ds_computers: if re.match('^npra-al.*', ds_computer) or re.match('^npra-aw.*', ds_computer): to_delete.append(ds_computer) """ # Exlude host that exist in foreman to the list of instances retrieve from the DS for foreman_host in foreman_hosts.keys(): for i, ds_computer in enumerate(to_delete): if re.match('^{}.*'.format(foreman_host), ds_computer): del to_delete[i] for host in to_delete: try: ip_address = socket.gethostbyname(host) found = True except: found = False # Make the following 2 call only at the end in order to avoid useless consuming API call try: if found: is_terminated = (get_ec2_instance_state( '', ip=ip_address) == 'terminated') if not is_terminated: print( "{} is not terminated, ignoring this instance".format( host)) saved += 1 continue logging.info("I will destroy the server {}".format(host)) # remove host in the DS ds.delete_computer(host) deleted += 1 except Exception as e: logging.error("Something went wrong : {}".format(e)) logging.info( "{} instances deleted\n{} instances saved\n{} instances in foreman\n". format(deleted, saved, len(foreman_hosts)))
from getpass import getpass from foreman.client import Foreman f = Foreman('https://192.168.99.100:8443', ('admin', 'changeme')) print(f.index_hosts()) print(f.show_hosts(id=1)) res = f.create_hosts(host={'name': 'mynewhost', 'ip': '192.168.1.1', 'mac': '00:00:00:00:00:00'})
class ForemanClient(Client): per_page = 100 def __init__(self, config_path, debug=False, version='1', log=None): super(ForemanClient, self).__init__(config_path, debug, log) self.logger.debug('=> config file: %s' % config_path) foreman_url = self.get_config('foreman', 'url') self.logger.debug('=> foreman url: %s' % foreman_url) foreman_user = self.get_config('foreman', 'user') foreman_password = self.get_config('foreman', 'password') self.foreman = Foreman(foreman_url, (foreman_user, foreman_password), api_version=2, version=version, verify=False) def set_per_page(self, per_page): self.per_page = per_page def get_config(self, section, option): try: value = self.config.get(section, option) return value except ConfigParser.NoOptionError: self.logger.debug('=> config file section [%s] missing option %s' % (section, option)) except ConfigParser.NoSectionError: self.logger.debug('=> config file missing section %s' % section) return None def get_config_section(self, section): try: openstack = self.config.items(section) except ConfigParser.NoSectionError: self.logger.debug('missing [%s]' % section) self.logger.debug('Could not find section [%s] in %s', section, self.config_path) sys.exit(1) return dict(openstack) def get_location(self): locations = self.foreman.index_locations() location_id = False for l in locations['results']: if l['name'] == 'Default Location': location_id = l['id'] return location_id def get_organization(self): organizations = self.foreman.index_organizations() organization_id = False for o in organizations['results']: if o['name'] == 'Default Organization': organization_id = o['id'] return organization_id def get_logger(self): return self.logger def get_client(self): return self.foreman def get_compute_resources(self): resources = self.foreman.index_computeresources() found_resources = dict({}) for r in resources['results']: found_resources[r['name']] = r['id'] return found_resources def get_compute_profiles(self): profiles = self.foreman.index_computeprofiles() found_profiles = dict({}) for p in profiles['results']: found_profiles[p['name']] = p['id'] return found_profiles def get_profile_id(self, profile_name): profile = self.foreman.show_computeprofiles(profile_name) return profile['id'] def get_host(self, host): host = self.__set_host(host) return self.foreman.show_hosts(id=host) def get_fact(self, host, fact): host = self.__set_host(host) facts = self.get_facts(host) fact = facts['results'][host][fact] return fact def get_facts(self, host_id): host = self.__set_host(host_id) return self.foreman.hosts.fact_values_index(host_id=host, per_page=self.per_page) def set_host_build(self, host, build=True): host = self.__set_host(host) if len(self.foreman.show_hosts(id=host)) > 0: self.foreman.update_hosts(id=host, host={'build': build}) def get_hosts(self, search=None): hosts = self.foreman.index_hosts(per_page=self.per_page) self.logger.debug("=> fetch %s page(s) with a total of %s hosts" % (hosts['page'], hosts['total'])) return hosts def create_host(self, host): if 'name' not in host: self.logger.debug('host dict missing name') return self.logger.debug('=> create new host %s' % host['name']) result = self.foreman.create_host(host) self.logger.debug('=> host created: %s' % result) def create_node(self, name, node_data, region): if self.get_host(name): self.logger.debug('=> node %s found, dropping create' % name) return found_resources = self.get_compute_resources() host = dict() host['name'] = name host['build'] = self.__get_node_data('build', node_data, '1') host['hostgroup_id'] = self.__get_node_data('hostgroup', node_data, '1') host['compute_profile_id'] = self.get_profile_id( self.__get_node_data('compute_profile', node_data, 'small')) host['organization_id'] = self.get_organization() host['location_id'] = self.get_location() host['interfaces_attributes'] = self.__get_node_data( 'interfaces_attributes', node_data, {}) host['compute_attributes'] = self.__get_node_data( 'compute_attributes', node_data, {}) host['host_parameters_attributes'] = self.__get_node_data( 'host_parameters_attributes', node_data, {}) if 'mac' in node_data: host['mac'] = node_data['mac'] if 'compute_resource' in node_data: compute_resource = '%s-%s' % (region, node_data['compute_resource']) if compute_resource in found_resources: host['compute_resource_id'] = found_resources[compute_resource] else: self.logger.debug('=> compute resource %s not found' % compute_resource) return elif 'mac' not in node_data: self.logger.debug( '=> mac or compute resource are mandatory for %s' % name) return if not self.dry_run: result = self.foreman.create_hosts(host) if not result: self.log_error( 'Could not create host. Check production.log on foreman host!' ) return if 'mac' not in node_data: self.foreman.hosts.power(id=result['name'], power_action='start') self.logger.debug('=> create host %s' % result) else: self.logger.debug('=> dry run: host config %s' % host) def delete_node(self, host): host = self.__set_host(host) if not self.dry_run: result = self.foreman.destroy_hosts(host) if not result: self.log_error('Could not delete host.') return self.logger.debug('=> deleted node %s' % host) else: self.logger.debug('=> dry run: deleted node %s' % host) def __set_host(self, host): if not host: self.host = None return domain = self.config.get('openstack', 'domain') if domain and not '.' in host: self.logger.debug("=> prepend %s to %s" % (domain, host)) host = host + '.' + domain return host @staticmethod def log_error(msg, code=0): sys.stderr.write("%s\n" % msg) if code > 0: sys.exit(code) @staticmethod def __get_node_data(var, node_data, default=None): if var in node_data: return node_data[var] else: return default