def hostgroups_summary(foreman, user, passwd):
"""
Show a summary of the current used and unused profiles and machines
:param foreman: URL to the foreman server
:param user: username to login into Foreman
:param passwd: Password to use when logging in
"""
frm = Foreman(foreman, (user, passwd))
hgdict = {}
for hg in frm.index_hostgroups(per_page=900):
hgdict[hg['hostgroup']['id']] = hg['hostgroup']['name']
notused = dict(hgdict.iteritems())
groups = []
## Available
hosts = frm.index_hosts(per_page=1000)
puts(green("Total Available hosts: ", True) + green(len(hosts)))
puts(blue("Available hosts by hostgroup:", True))
for next_host in hosts:
gid = next_host['host']['hostgroup_id']
if gid:
groups.append((hgdict[gid], gid))
gid in notused and notused.pop(gid)
else:
groups.append(('No group', 'None'))
if COUNTER:
groups_count = dict(Counter(groups)).items()
else:
dict([(item, groups.count(item))
for item in sorted(set(groups))]).items()
groups_count.sort()
for group, count in groups_count:
gname, gid = group
if gname != 'No group':
puts(blue("\t%s (id=%d) = %s" % (gname, gid, white(count))))
else:
puts(blue("\tNo group = %s" % white(count)))
## Unused
puts(blue("Unused hostgroups:", True))
for gid, name in notused.iteritems():
puts(blue("\t%s (id=%s)" % (name, gid)))
from getpass import getpass
import psycopg2
from foreman.client import Foreman
import csv, time
# create log file
logfile = "log" + time.strftime("_%m_%d_%Y-%H_%M_%S") + ".csv"
with open(logfile, "w") as f:
writer = csv.writer(f)
writer.writerow(["hostname", "hostgroup"])
# create connection object to foreman and object for all nodes
username = raw_input("Enter username: "******"dbname='puppetdb' user='******' host='devitpuppet.tsi.lan' password='******'")
cur = conn.cursor()
cur.execute("""select h.name,hg.name from hosts h left join hostgroups hg on h.hostgroup_id=hg.id""")
rows = cur.fetchall()
# iterate through foreman nodes and add to necessary hostgroup
for node in nodes:
class Client(object):
def __init__(self, config_path, debug=False, version='1', log=None):
self.config = utils.get_config(config_path)
self.logger = utils.get_logger(__name__, self.config, debug, log)
config = self.get_config_section('foreman')
self.logger.debug('=> config file: %s' % config_path)
self.logger.debug('=> foreman url: %s' % config['url'])
self.foreman = Foreman(config['url'],
(config['user'], config['password']),
api_version=2,
version=version,
verify=False)
def get_config(self, section, option):
try:
value = self.config.get(section, option)
return value
except ConfigParser.NoOptionError:
self.logger.debug('=> config file section [%s] missing option %s'
% (section, option))
except ConfigParser.NoSectionError:
self.logger.debug('=> config file missing section %s' % section)
return None
def get_config_section(self, section):
try:
openstack = self.config.items(section)
except ConfigParser.NoSectionError:
self.logger.exception('missing [%s]' % section)
self.logger.critical('Could not find section [%s] in %s', section, self.config_path)
sys.exit(1)
return dict(openstack)
def get_logger(self):
return self.logger
def get_client(self):
return self.foreman
def get_compute_resources(self):
resources = self.foreman.index_computeresources()
found_resources = dict({})
for r in resources['results']:
found_resources[r['name']] = r['id']
return found_resources
def get_host(self, host):
host = self.__set_host(host)
return self.foreman.show_hosts(id=host)
def set_host_build(self, host, build=True):
host = self.__set_host(host)
if len(self.foreman.show_hosts(id=host)) > 0:
self.foreman.update_hosts(id=host, host={'build': build})
def get_hosts(self, search=None):
hosts = self.foreman.index_hosts()
self.logger.debug("=> fetch %s page(s) with a total of %s hosts" %
(hosts['page'], hosts['total']))
return hosts
def create_host(self, host):
if 'name' not in host:
self.logger.critical('host dict missing name')
return
self.logger.debug('=> create new host %s' % host['name'])
result = self.foreman.create_host(host)
self.logger.debug('=> host created: %s' % result)
def create_node(self, name, node_data, region, dry_run=False):
if self.get_host(name):
self.logger.debug('=> node %s found, dropping create' % name)
return
found_resources = self.get_compute_resources()
host = dict()
host['name'] = name
host['build'] = self.__get_node_data('build', node_data, '1')
host['hostgroup_id'] = self.__get_node_data('hostgroup', node_data, '1')
host['compute_profile_id'] = self.__get_node_data('compute_profile', node_data, '1')
host['interfaces_attributes'] = self.__get_node_data(
'interfaces_attributes', node_data, {})
host['compute_attributes'] = self.__get_node_data(
'compute_attributes', node_data, {})
host['host_parameters_attributes'] = self.__get_node_data(
'host_parameters_attributes', node_data, {})
if 'mac' in node_data:
host['mac'] = node_data['mac']
if 'compute_resource' in node_data:
compute_resource = '%s-%s' % (region, node_data['compute_resource'])
if compute_resource in found_resources:
host['compute_resource_id'] = found_resources[compute_resource]
else:
self.logger.critical('=> compute resource %s not found' % compute_resource)
return
elif 'mac' not in node_data:
self.logger.critical('=> mac or compute resource are mandatory for %s' % name)
return
if not dry_run:
result = self.foreman.create_hosts(host)
if not result:
self.log_error('Could not create host. Check production.log on foreman host!')
return
if 'mac' not in node_data:
self.foreman.hosts.power(id=result['name'], power_action='start')
self.logger.debug('=> create host %s' % result)
else:
self.logger.debug('=> dry run: host config %s' % host)
def __set_host(self, host):
if not host:
self.host = None
return
domain = self.config.get('openstack', 'domain')
if domain and not '.' in host:
self.logger.debug("=> prepend %s to %s" % (domain, host))
host = host + '.' + domain
return host
@staticmethod
def log_error(msg, code=0):
sys.stderr.write("%s\n" % msg)
if code > 0:
sys.exit(code)
@staticmethod
def __get_node_data(var, node_data, default=None):
if var in node_data:
return node_data[var]
else:
return default
class ForemanClient(Client):
per_page = 100
def __init__(self, config_path, debug=False, version='1', log=None):
super(ForemanClient, self).__init__(config_path, debug, log)
self.logger.debug('=> config file: %s' % config_path)
foreman_url = self.get_config('foreman', 'url')
self.logger.debug('=> foreman url: %s' % foreman_url)
foreman_user = self.get_config('foreman', 'user')
foreman_password = self.get_config('foreman', 'password')
self.foreman = Foreman(foreman_url,
(foreman_user, foreman_password),
api_version=2,
version=version,
verify=False)
def set_per_page(self, per_page):
self.per_page = per_page
def get_config(self, section, option):
try:
value = self.config.get(section, option)
return value
except ConfigParser.NoOptionError:
self.logger.debug('=> config file section [%s] missing option %s'
% (section, option))
except ConfigParser.NoSectionError:
self.logger.debug('=> config file missing section %s' % section)
return None
def get_config_section(self, section):
try:
openstack = self.config.items(section)
except ConfigParser.NoSectionError:
self.logger.debug('missing [%s]' % section)
self.logger.debug('Could not find section [%s] in %s', section, self.config_path)
sys.exit(1)
return dict(openstack)
def get_logger(self):
return self.logger
def get_client(self):
return self.foreman
def get_compute_resources(self):
resources = self.foreman.index_computeresources()
found_resources = dict({})
for r in resources['results']:
found_resources[r['name']] = r['id']
return found_resources
def get_compute_profiles(self):
profiles = self.foreman.index_computeprofiles()
found_profiles = dict({})
for p in profiles['results']:
found_profiles[p['name']] = p['id']
return found_profiles
def get_profile_id(self, profile_name):
profile = self.foreman.show_computeprofiles(profile_name)
return profile['id']
def get_host(self, host):
host = self.__set_host(host)
return self.foreman.show_hosts(id=host)
def get_fact(self, host, fact):
host = self.__set_host(host)
facts = self.get_facts(host)
fact = facts['results'][host][fact]
return fact
def get_facts(self, host_id):
host = self.__set_host(host_id)
return self.foreman.hosts.fact_values_index(host_id=host, per_page=self.per_page)
def set_host_build(self, host, build=True):
host = self.__set_host(host)
if len(self.foreman.show_hosts(id=host)) > 0:
self.foreman.update_hosts(id=host, host={'build': build})
def get_hosts(self, search=None):
hosts = self.foreman.index_hosts(per_page=self.per_page)
self.logger.debug("=> fetch %s page(s) with a total of %s hosts" %
(hosts['page'], hosts['total']))
return hosts
def create_host(self, host):
if 'name' not in host:
self.logger.debug('host dict missing name')
return
self.logger.debug('=> create new host %s' % host['name'])
result = self.foreman.create_host(host)
self.logger.debug('=> host created: %s' % result)
def create_node(self, name, node_data, region):
if self.get_host(name):
self.logger.debug('=> node %s found, dropping create' % name)
return
found_resources = self.get_compute_resources()
host = dict()
host['name'] = name
host['build'] = self.__get_node_data('build', node_data, '1')
host['hostgroup_id'] = self.__get_node_data('hostgroup', node_data, '1')
host['compute_profile_id'] = self.get_profile_id(
self.__get_node_data('compute_profile',
node_data,
'small'))
host['interfaces_attributes'] = self.__get_node_data(
'interfaces_attributes', node_data, {})
host['compute_attributes'] = self.__get_node_data(
'compute_attributes', node_data, {})
host['host_parameters_attributes'] = self.__get_node_data(
'host_parameters_attributes', node_data, {})
if 'mac' in node_data:
host['mac'] = node_data['mac']
if 'compute_resource' in node_data:
compute_resource = '%s-%s' % (region, node_data['compute_resource'])
if compute_resource in found_resources:
host['compute_resource_id'] = found_resources[compute_resource]
else:
self.logger.debug('=> compute resource %s not found' % compute_resource)
return
elif 'mac' not in node_data:
self.logger.debug('=> mac or compute resource are mandatory for %s' % name)
return
if not self.dry_run:
result = self.foreman.create_hosts(host)
if not result:
self.log_error('Could not create host. Check production.log on foreman host!')
return
if 'mac' not in node_data:
self.foreman.hosts.power(id=result['name'], power_action='start')
self.logger.debug('=> create host %s' % result)
else:
self.logger.debug('=> dry run: host config %s' % host)
def delete_node(self, host):
host = self.__set_host(host)
if not self.dry_run:
result = self.foreman.destroy_hosts(host)
if not result:
self.log_error('Could not delete host.')
return
self.logger.debug('=> deleted node %s' % host)
else:
self.logger.debug('=> dry run: deleted node %s' % host)
def __set_host(self, host):
if not host:
self.host = None
return
domain = self.config.get('openstack', 'domain')
if domain and not '.' in host:
self.logger.debug("=> prepend %s to %s" % (domain, host))
host = host + '.' + domain
return host
@staticmethod
def log_error(msg, code=0):
sys.stderr.write("%s\n" % msg)
if code > 0:
sys.exit(code)
@staticmethod
def __get_node_data(var, node_data, default=None):
if var in node_data:
return node_data[var]
else:
return default
def clean_ds():
""" This is a 'one time use' method that will clear from the DS all instances that doesn't still exist """
# Retrieve config from ENV
# Stats
saved = 0
deleted = 0
# connect to Foreman and ForemanProxy
f = Foreman(FOREMAN_URL, (FOREMAN_USER, FOREMAN_PASSWORD), api_version=2)
# Connect to the DS
try:
ds = AwsDs(LDAP_HOST, COMPUTERS_BASE_DN, BIND_USER_DN, BIND_PASSWORD)
except ldap.INVALID_CREDENTIALS:
raise "Your username or password is incorrect."
# Get all host from foreman
page = 1
result = []
last_len = 1
while last_len > 0:
tmp_result = f.index_hosts(per_page="1000", page=str(page))['results']
last_len = len(tmp_result)
page += 1
result += tmp_result
foreman_hosts = {host["certname"]: host["ip"] for host in result}
# Get all ds computer
ds_computers = []
for c_dn, attr in ds.computers:
if 'dNSHostName' in attr and re.match('.*\.cloud\.coveo\.com$',
attr['dNSHostName'][0]):
ds_computers.append(attr['dNSHostName'][0].lower())
continue
else:
ds_computers.append(build_from_cn(attr['cn'][0]))
to_delete = ds_computers
"""
to_delete = []
# (Optional) filter which instances should be cleaned
for ds_computer in ds_computers:
if re.match('^npra-al.*', ds_computer) or re.match('^npra-aw.*', ds_computer):
to_delete.append(ds_computer)
"""
# Exlude host that exist in foreman to the list of instances retrieve from the DS
for foreman_host in foreman_hosts.keys():
for i, ds_computer in enumerate(to_delete):
if re.match('^{}.*'.format(foreman_host), ds_computer):
del to_delete[i]
for host in to_delete:
try:
ip_address = socket.gethostbyname(host)
found = True
except:
found = False
# Make the following 2 call only at the end in order to avoid useless consuming API call
try:
if found:
is_terminated = (get_ec2_instance_state(
'', ip=ip_address) == 'terminated')
if not is_terminated:
print(
"{} is not terminated, ignoring this instance".format(
host))
saved += 1
continue
logging.info("I will destroy the server {}".format(host))
# remove host in the DS
ds.delete_computer(host)
deleted += 1
except Exception as e:
logging.error("Something went wrong : {}".format(e))
logging.info(
"{} instances deleted\n{} instances saved\n{} instances in foreman\n".
format(deleted, saved, len(foreman_hosts)))
from getpass import getpass
from foreman.client import Foreman
f = Foreman('https://192.168.99.100:8443', ('admin', 'changeme'))
print(f.index_hosts())
print(f.show_hosts(id=1))
res = f.create_hosts(host={'name': 'mynewhost', 'ip': '192.168.1.1', 'mac': '00:00:00:00:00:00'})
class ForemanClient(Client):
per_page = 100
def __init__(self, config_path, debug=False, version='1', log=None):
super(ForemanClient, self).__init__(config_path, debug, log)
self.logger.debug('=> config file: %s' % config_path)
foreman_url = self.get_config('foreman', 'url')
self.logger.debug('=> foreman url: %s' % foreman_url)
foreman_user = self.get_config('foreman', 'user')
foreman_password = self.get_config('foreman', 'password')
self.foreman = Foreman(foreman_url, (foreman_user, foreman_password),
api_version=2,
version=version,
verify=False)
def set_per_page(self, per_page):
self.per_page = per_page
def get_config(self, section, option):
try:
value = self.config.get(section, option)
return value
except ConfigParser.NoOptionError:
self.logger.debug('=> config file section [%s] missing option %s' %
(section, option))
except ConfigParser.NoSectionError:
self.logger.debug('=> config file missing section %s' % section)
return None
def get_config_section(self, section):
try:
openstack = self.config.items(section)
except ConfigParser.NoSectionError:
self.logger.debug('missing [%s]' % section)
self.logger.debug('Could not find section [%s] in %s', section,
self.config_path)
sys.exit(1)
return dict(openstack)
def get_location(self):
locations = self.foreman.index_locations()
location_id = False
for l in locations['results']:
if l['name'] == 'Default Location':
location_id = l['id']
return location_id
def get_organization(self):
organizations = self.foreman.index_organizations()
organization_id = False
for o in organizations['results']:
if o['name'] == 'Default Organization':
organization_id = o['id']
return organization_id
def get_logger(self):
return self.logger
def get_client(self):
return self.foreman
def get_compute_resources(self):
resources = self.foreman.index_computeresources()
found_resources = dict({})
for r in resources['results']:
found_resources[r['name']] = r['id']
return found_resources
def get_compute_profiles(self):
profiles = self.foreman.index_computeprofiles()
found_profiles = dict({})
for p in profiles['results']:
found_profiles[p['name']] = p['id']
return found_profiles
def get_profile_id(self, profile_name):
profile = self.foreman.show_computeprofiles(profile_name)
return profile['id']
def get_host(self, host):
host = self.__set_host(host)
return self.foreman.show_hosts(id=host)
def get_fact(self, host, fact):
host = self.__set_host(host)
facts = self.get_facts(host)
fact = facts['results'][host][fact]
return fact
def get_facts(self, host_id):
host = self.__set_host(host_id)
return self.foreman.hosts.fact_values_index(host_id=host,
per_page=self.per_page)
def set_host_build(self, host, build=True):
host = self.__set_host(host)
if len(self.foreman.show_hosts(id=host)) > 0:
self.foreman.update_hosts(id=host, host={'build': build})
def get_hosts(self, search=None):
hosts = self.foreman.index_hosts(per_page=self.per_page)
self.logger.debug("=> fetch %s page(s) with a total of %s hosts" %
(hosts['page'], hosts['total']))
return hosts
def create_host(self, host):
if 'name' not in host:
self.logger.debug('host dict missing name')
return
self.logger.debug('=> create new host %s' % host['name'])
result = self.foreman.create_host(host)
self.logger.debug('=> host created: %s' % result)
def create_node(self, name, node_data, region):
if self.get_host(name):
self.logger.debug('=> node %s found, dropping create' % name)
return
found_resources = self.get_compute_resources()
host = dict()
host['name'] = name
host['build'] = self.__get_node_data('build', node_data, '1')
host['hostgroup_id'] = self.__get_node_data('hostgroup', node_data,
'1')
host['compute_profile_id'] = self.get_profile_id(
self.__get_node_data('compute_profile', node_data, 'small'))
host['organization_id'] = self.get_organization()
host['location_id'] = self.get_location()
host['interfaces_attributes'] = self.__get_node_data(
'interfaces_attributes', node_data, {})
host['compute_attributes'] = self.__get_node_data(
'compute_attributes', node_data, {})
host['host_parameters_attributes'] = self.__get_node_data(
'host_parameters_attributes', node_data, {})
if 'mac' in node_data:
host['mac'] = node_data['mac']
if 'compute_resource' in node_data:
compute_resource = '%s-%s' % (region,
node_data['compute_resource'])
if compute_resource in found_resources:
host['compute_resource_id'] = found_resources[compute_resource]
else:
self.logger.debug('=> compute resource %s not found' %
compute_resource)
return
elif 'mac' not in node_data:
self.logger.debug(
'=> mac or compute resource are mandatory for %s' % name)
return
if not self.dry_run:
result = self.foreman.create_hosts(host)
if not result:
self.log_error(
'Could not create host. Check production.log on foreman host!'
)
return
if 'mac' not in node_data:
self.foreman.hosts.power(id=result['name'],
power_action='start')
self.logger.debug('=> create host %s' % result)
else:
self.logger.debug('=> dry run: host config %s' % host)
def delete_node(self, host):
host = self.__set_host(host)
if not self.dry_run:
result = self.foreman.destroy_hosts(host)
if not result:
self.log_error('Could not delete host.')
return
self.logger.debug('=> deleted node %s' % host)
else:
self.logger.debug('=> dry run: deleted node %s' % host)
def __set_host(self, host):
if not host:
self.host = None
return
domain = self.config.get('openstack', 'domain')
if domain and not '.' in host:
self.logger.debug("=> prepend %s to %s" % (domain, host))
host = host + '.' + domain
return host
@staticmethod
def log_error(msg, code=0):
sys.stderr.write("%s\n" % msg)
if code > 0:
sys.exit(code)
@staticmethod
def __get_node_data(var, node_data, default=None):
if var in node_data:
return node_data[var]
else:
return default
