def main():
args = get_args()
# from requests.auth import HTTPBasicAuth
requests.packages.urllib3.disable_warnings()
# create connection object to foreman and nodes object
f = Foreman(args['apiurl'], (args['username'], args['password']))
if args['which'] == 'list' and args['groups']:
# list hostgroups within Foreman
group_dict = {}
hostgroups = f.index_hostgroups()
print("\n{0:40}{1}".format("Hostgroup", "ID"))
print("{0:40}{1}".format("---------", "--\n"))
for hostgroup in hostgroups:
group_dict[hostgroup[u'hostgroup']
[u'name']] = hostgroup[u'hostgroup'][u'id']
print("{0:40}{1}".format(hostgroup[u'hostgroup'][u'name'],
hostgroup[u'hostgroup'][u'id']))
if args['which'] == 'add':
# add specified host to specified hostgroup
hostid = get_host_id(f, args['hostname'])
groupname = args['hostgroup']
if not is_number(groupname):
groupid = get_groupid(f, groupname)
try:
f.update_hosts(id=hostid, host={'hostgroup_id': groupid})
print("Host successfully added to %s hostgroup" % groupname)
except ForemanException, e:
sys.exit(str(e) + ', see error description above')
def _iget_properties(props, foreman, user, passwd):
"""
Gets the given properties from the hosts. From the host properties or
parameters (you can search for example mac_addres property or RESERVED
parameter). Returns an iterator, if you want a list, you can pass it to
the list function `list(_iget_properties(*args))`
:param props: comma separated list of glob patterns to match the
properties names
:param foreman: URL to the foreman server
:param user: username to login into Foreman
:param passwd: Password to use when logging in
"""
frm = Foreman(foreman, (user, passwd))
props = props.split(':')
host = frm.show_hosts(env['host'])
if not host:
raise Exception("Host %s not found in foreman" % env['host'])
host = host['host']
for match in ifilter_glob(host.keys(), props):
yield (match, host[match])
params_dict = dict((
(x['parameter']['name'], x['parameter']['value'])
for x in host['parameters']
))
for host_param in ifilter_glob(params_dict.keys(), props):
yield (host_param, params_dict[host_param])
def __init__(self, config_path, debug=False, version='1', log=None):
super(ForemanClient, self).__init__(config_path, debug, log)
self.logger.debug('=> config file: %s' % config_path)
foreman_url = self.get_config('foreman', 'url')
self.logger.debug('=> foreman url: %s' % foreman_url)
foreman_user = self.get_config('foreman', 'user')
foreman_password = self.get_config('foreman', 'password')
self.foreman = Foreman(foreman_url, (foreman_user, foreman_password),
api_version=2,
version=version,
verify=False)
def __init__(self, config_path, debug=False, version='1', log=None):
self.config = utils.get_config(config_path)
self.logger = utils.get_logger(__name__, self.config, debug, log)
config = self.get_config_section('foreman')
self.logger.debug('=> config file: %s' % config_path)
self.logger.debug('=> foreman url: %s' % config['url'])
self.foreman = Foreman(config['url'],
(config['user'], config['password']),
api_version=2,
version=version,
verify=False)
def main():
argument_spec = dict(
url=dict(required=True),
foreman_user=dict(required=True),
foreman_password=dict(required=True, no_log=True),
name=dict(required=True),
power_action=dict(choices=[
'start', 'stop', 'poweroff', 'reboot', 'reset', 'state', 'on',
'off', 'soft', 'cycle'
],
required=True),
)
module = AnsibleModule(argument_spec)
if not HAS_REQS:
module.fail_json(msg='python-foreman and retrying required for module')
try:
host_params = dict(module.params)
foreman_client = Foreman(
host_params['url'],
(host_params['foreman_user'], host_params['foreman_password']),
api_version=2)
power = _power_action(module, foreman_client)
except Exception as e:
module.fail_json(msg=e.message)
def main():
argument_spec = dict(
url=dict(required=True),
foreman_user=dict(required=True),
foreman_password=dict(required=True, no_log=True),
name=dict(required=True),
)
module = AnsibleModule(argument_spec)
if not HAS_FOREMAN_CLIENT:
module.fail_json(msg='python-foreman is required for this module')
try:
host_params = dict(module.params)
foreman_client = Foreman(
host_params['url'],
(host_params['foreman_user'], host_params['foreman_password']),
api_version=2)
host = single_element_from_name('hosts', module.params['name'], module,
foreman_client)
status = _get_host_status(module, foreman_client)
module.exit_json(changed=False,
ansible_facts=dict(foreman_host=host,
foreman_status=status))
except Exception as e:
module.fail_json(msg=e.message)
def generate_api(url, foreman_version, api_version):
requests.Session = SessionMock(url, foreman_version)
print("Generating api")
return Foreman(
url,
version=foreman_version,
api_version=api_version,
)
def generate_api(self):
requests.Session = SessionMock(self.url, self.frmn_ver)
self.logger.info("Generate api")
return Foreman(
self.url,
version=self.frmn_ver,
api_version=self.api_ver,
)
def _get_conn(self):
try:
self.client = Foreman(self.base_url, (self.username, self.password))
except ConnectionError, e:
print '''It looks like Foreman's API is unreachable.'''
print e
sys.exit(1)
def fetch_api(self):
now = datetime.datetime.now()
api_definition = "./api_definitions_{}".format(now.strftime('%Y_%m_%d'))
print("Using cache dir {}".format(api_definition))
self.client = Foreman('https://{}/'.format(self.cli_args.server),
auth=(self.cli_args.username, self.cli_args.password),
api_version=self.api_version,
use_cache=False,
cache_dir=api_definition)
def connect(self):
try:
logging.disable(logging.WARNING)
self.fm = Foreman(self.config['auth']['url'], (self.config['auth']['user'], self.config['auth']['pass']), api_version=2, use_cache=False, strict_cache=False)
# this is nescesary for detecting faulty credentials in yaml
self.fm.architectures.index()
logging.disable(self.loglevel-1)
except:
log.log(log.LOG_ERROR, "Cannot connect to Foreman-API")
sys.exit(1)
def get_subnet(foreman, user, passwd):
""""
Return the subnet name and network for the hosts
:param foreman: URL to the foreman server
:param user: username to login into Foreman
:param passwd: Password to use when logging in
"""
subnet_id = dict((
_iget_properties('subnet_id', foreman=foreman,
user=user, passwd=passwd)
))
frm = Foreman(foreman, (user, passwd))
subnet = frm.show_subnets(subnet_id['subnet_id'])
if subnet:
puts(green(env.host + '=')
+ white(subnet['subnet']['name'])
+ '|' + blue(subnet['subnet']['network']))
else:
puts(red(env.host + ' no subnet found'))
def hostgroups_summary(foreman, user, passwd):
"""
Show a summary of the current used and unused profiles and machines
:param foreman: URL to the foreman server
:param user: username to login into Foreman
:param passwd: Password to use when logging in
"""
frm = Foreman(foreman, (user, passwd))
hgdict = {}
for hg in frm.index_hostgroups(per_page=900):
hgdict[hg['hostgroup']['id']] = hg['hostgroup']['name']
notused = dict(hgdict.iteritems())
groups = []
## Available
hosts = frm.index_hosts(per_page=1000)
puts(green("Total Available hosts: ", True) + green(len(hosts)))
puts(blue("Available hosts by hostgroup:", True))
for next_host in hosts:
gid = next_host['host']['hostgroup_id']
if gid:
groups.append((hgdict[gid], gid))
gid in notused and notused.pop(gid)
else:
groups.append(('No group', 'None'))
if COUNTER:
groups_count = dict(Counter(groups)).items()
else:
dict([(item, groups.count(item))
for item in sorted(set(groups))]).items()
groups_count.sort()
for group, count in groups_count:
gname, gid = group
if gname != 'No group':
puts(blue("\t%s (id=%d) = %s" % (gname, gid, white(count))))
else:
puts(blue("\tNo group = %s" % white(count)))
## Unused
puts(blue("Unused hostgroups:", True))
for gid, name in notused.iteritems():
puts(blue("\t%s (id=%s)" % (name, gid)))
def __init__(self, config_path, debug=False, version='1', log=None):
super(ForemanClient, self).__init__(config_path, debug, log)
self.logger.debug('=> config file: %s' % config_path)
foreman_url = self.get_config('foreman', 'url')
self.logger.debug('=> foreman url: %s' % foreman_url)
foreman_user = self.get_config('foreman', 'user')
foreman_password = self.get_config('foreman', 'password')
self.foreman = Foreman(foreman_url,
(foreman_user, foreman_password),
api_version=2,
version=version,
verify=False)
def __init__(self, username, password, foreman_url):
# initializing vars
self.base_url = foreman_url
self.username = username
self.password = password
self.inventory = self._empty_inventory()
self._cache = self._empty_cache()
try:
self.client = Foreman(self.base_url,
(self.username, self.password))
except ConnectionError, e:
raise Exception(
"It looks like Foreman's API is unreachable. Error "
"was: " + str(e))
def main():
argument_spec = dict(
state=dict(default='present', choices=['absent', 'present']),
url=dict(required=True),
foreman_user=dict(required=True),
foreman_password=dict(required=True, no_log=True),
build=dict(default='false', choices=['true', 'false']),
name=dict(required=True),
ip=dict(required=False),
mac=dict(required=False),
organization_name=dict(required=True),
location_name=dict(required=True),
hostgroup_name=dict(required=False),
ptable_name=dict(required=False),
root_pass=dict(required=False),
host_parameters_attributes=dict(required=False),
interfaces_attributes=dict(required=False),
compute_resource=dict(required=False),
compute_profile=dict(required=False),
network=dict(required=False),
)
module = AnsibleModule(argument_spec)
if not HAS_REQS:
module.fail_json(msg='python-foreman is required for this module')
state = module.params['state']
try:
host_params = dict(module.params)
foreman_client = Foreman(
host_params['url'],
(host_params['foreman_user'], host_params['foreman_password']),
api_version=2)
if state == 'present':
_get_host_state(module, foreman_client)
_create_host(module, foreman_client)
elif state == 'absent':
_get_host_state(module, foreman_client)
_delete_host(module, foreman_client)
except ValueError as e:
module.fail_json(msg=e.message)
def clean_old_certificates(json_file, check_on_fs):
""" This method that will clear all puppet cert for instances that do not still exist """
logging.info("########## Start Cleaning ###########")
# connect to Foreman and ForemanProxy
f = Foreman(FOREMAN_URL, (FOREMAN_USER, FOREMAN_PASSWORD), api_version=2)
fp = ForemanProxy(FOREMAN_PROXY_URL)
host_pattern = [
'ndev', 'nsta', 'nifd', 'npra', 'nifp-es5k', 'nhip', 'nifh', 'win',
'nprd', 'nqa'
]
if not json_file and check_on_fs:
jcerts = check_output(["ls", "-f",
"/var/lib/puppet/ssl/ca/signed/"]).split()
elif not json_file and not check_on_fs:
fp_certs = fp.get_certificates()
jcerts = [c for c in fp_certs if fp_certs[c].get('state') == 'valid']
else:
try:
with open(json_file) as data_file:
jcerts = json.load(data_file)
except Exception as e:
print("Cant't decode json file: {}".format(e))
sys.exit(0)
certs = [
cert.replace(".pem", "") for cert in jcerts
if any(pattern in cert for pattern in host_pattern)
]
foreman_hosts = []
result = foreman_wrapper(f.index_hosts, call_args={"per_page": 1000})
for host in result:
foreman_hosts.append(host["certname"])
certs_to_delete = list(set(certs) - set(foreman_hosts))
for cert in certs_to_delete:
try:
print(" {} will be deleted".format(cert))
fp.delete_certificate(cert)
except Exception as e:
print(" {} couldn't be deleted: {}".format(cert, e))
def gera_config(rack):
id_core1 = None
id_core2 = None
name_sp1 = None
name_sp2 = None
name_sp3 = None
name_sp4 = None
name_core1 = None
name_core2 = None
int_sp1 = None
int_sp2 = None
int_sp3 = None
int_sp4 = None
int_lf1_sp1 = None
int_lf1_sp2 = None
int_lf2_sp3 = None
int_lf2_sp4 = None
int_oob_mgmtlf1 = None
int_oob_mgmtlf2 = None
int_oob_core1 = None
int_oob_core2 = None
int_core1_oob = None
int_core2_oob = None
prefix_spine = 'SPN'
prefix_oob = 'OOB'
# Equipamentos
num_rack = rack.numero
try:
id_lf1 = rack.id_sw1.id
name_lf1 = rack.id_sw1.nome
id_lf2 = rack.id_sw2.id
name_lf2 = rack.id_sw2.nome
id_oob = rack.id_ilo.id
name_oob = rack.id_ilo.nome
except:
raise RackConfigError(None, rack.nome, 'Erro: Rack incompleto.')
# Interface leaf01
try:
interfaces = Interface.search(id_lf1)
for interface in interfaces:
try:
sw = interface.get_switch_and_router_interface_from_host_interface(
None)
if sw.equipamento.nome.split('-')[0] == prefix_spine:
if sw.equipamento.nome.split('-')[2] == '01' or sw.equipamento.nome.split('-')[2] == '1':
int_lf1_sp1 = interface.interface
name_sp1 = sw.equipamento.nome
id_sp1 = sw.equipamento.id
int_sp1 = sw.interface
elif sw.equipamento.nome.split('-')[2] == '02' or sw.equipamento.nome.split('-')[2] == '2':
int_lf1_sp2 = interface.interface
name_sp2 = sw.equipamento.nome
id_sp2 = sw.equipamento.id
int_sp2 = sw.interface
elif sw.equipamento.nome.split('-')[0] == prefix_oob:
int_oob_mgmtlf1 = sw.interface
except:
pass
except InterfaceNotFoundError:
raise RackConfigError(
None, rack.nome, 'Erro ao buscar as interfaces associadas ao Leaf 01.')
if int_sp1 is None or int_sp2 is None or int_oob_mgmtlf1 is None:
raise RackConfigError(
None, rack.nome, 'Erro: As interfaces do Leaf01 nao foram cadastradas.')
# Interface leaf02
try:
interfaces1 = Interface.search(id_lf2)
for interface1 in interfaces1:
try:
sw = interface1.get_switch_and_router_interface_from_host_interface(
None)
if sw.equipamento.nome.split('-')[0] == prefix_spine:
if sw.equipamento.nome.split('-')[2] == '03' or sw.equipamento.nome.split('-')[2] == '3':
int_lf2_sp3 = interface1.interface
name_sp3 = sw.equipamento.nome
id_sp3 = sw.equipamento.id
int_sp3 = sw.interface
elif sw.equipamento.nome.split('-')[2] == '04' or sw.equipamento.nome.split('-')[2] == '4':
int_lf2_sp4 = interface1.interface
name_sp4 = sw.equipamento.nome
id_sp4 = sw.equipamento.id
int_sp4 = sw.interface
elif sw.equipamento.nome.split('-')[0] == prefix_oob:
int_oob_mgmtlf2 = sw.interface
except:
pass
except InterfaceNotFoundError:
raise RackConfigError(
None, rack.nome, 'Erro ao buscar as interfaces associadas ao Leaf 02.')
if int_sp3 is None or int_sp4 is None or int_oob_mgmtlf2 is None:
raise RackConfigError(
None, rack.nome, 'Erro: As interfaces do Leaf02 nao foram cadastradas.')
# Interface OOB
try:
interfaces2 = Interface.search(id_oob)
for interface2 in interfaces2:
try:
sw = interface2.get_switch_and_router_interface_from_host_interface(
None)
if sw.equipamento.nome.split('-')[0] == prefix_oob:
if sw.equipamento.nome.split('-')[2] == '01' or sw.equipamento.nome.split('-')[2] == '1':
int_oob_core1 = interface2.interface
name_core1 = sw.equipamento.nome
int_core1_oob = sw.interface
id_core1 = sw.equipamento.id
elif sw.equipamento.nome.split('-')[2] == '02' or sw.equipamento.nome.split('-')[2] == '2':
int_oob_core2 = interface2.interface
name_core2 = sw.equipamento.nome
int_core2_oob = sw.interface
id_core2 = sw.equipamento.id
except:
pass
except InterfaceNotFoundError:
raise RackConfigError(
None, rack.nome, 'Erro ao buscar as interfaces associadas ao Switch de gerencia.')
if int_oob_core1 is None or int_core1_oob is None or int_oob_core2 is None or int_core2_oob is None:
raise RackConfigError(
None, rack.nome, 'Erro: As interfaces do Switch de gerencia nao foram cadastradas.')
# Roteiro LF01
try:
FILEINLF1 = buscar_roteiro(id_lf1, 'CONFIGURACAO')
except:
raise RackConfigError(
None, rack.nome, 'Erro ao buscar o roteiro do Leaf 01.')
# Roteiro LF02
try:
FILEINLF2 = buscar_roteiro(id_lf2, 'CONFIGURACAO')
except:
raise RackConfigError(
None, rack.nome, 'Erro ao buscar o roteiro do Leaf 02.')
# Roteiro SPN01
try:
FILEINSP1 = buscar_roteiro(id_sp1, 'CONFIGURACAO')
except:
raise RackConfigError(
None, rack.nome, 'Erro ao buscar o roteiro do Spine 01.')
# Roteiro SPN02
try:
FILEINSP2 = buscar_roteiro(id_sp2, 'CONFIGURACAO')
except:
raise RackConfigError(
None, rack.nome, 'Erro ao buscar o roteiro do Spine 02.')
# Roteiro SPN03
try:
FILEINSP3 = buscar_roteiro(id_sp3, 'CONFIGURACAO')
except:
raise RackConfigError(
None, rack.nome, 'Erro ao buscar o roteiro do Spine 03.')
# Roteiro SPN04
try:
FILEINSP4 = buscar_roteiro(id_sp4, 'CONFIGURACAO')
except:
raise RackConfigError(
None, rack.nome, 'Erro ao buscar o roteiro do Spine 04.')
# Roteiro Core 01
try:
FILEINCR1 = buscar_roteiro(id_core1, 'CONFIGURACAO')
except:
raise RackConfigError(
None, rack.nome, 'Erro ao buscar o roteiro do Core 01.')
# Roteiro Core 02
try:
FILEINCR2 = buscar_roteiro(id_core2, 'CONFIGURACAO')
except:
raise RackConfigError(
None, rack.nome, 'Erro ao buscar o roteiro do Core 02.')
# Roteiro OOB
try:
FILEINOOB = buscar_roteiro(id_oob, 'CONFIGURACAO')
except:
raise RackConfigError(
None, rack.nome, 'Erro ao buscar o roteiro do switch de gerencia.')
# Ip LF01
try:
ip_mgmtlf1 = buscar_ip(id_lf1)
except:
raise RackConfigError(
None, rack.nome, 'Erro ao buscar o ip de gerencia do leaf 01.')
# Ip LF02
try:
ip_mgmtlf2 = buscar_ip(id_lf2)
except:
raise RackConfigError(
None, rack.nome, 'Erro ao buscar o ip de gerencia do leaf 02.')
# Ip OOB
try:
ip_mgmtoob = buscar_ip(id_oob)
except:
raise RackConfigError(
None, rack.nome, 'Erro ao buscar o ip de gerencia do oob.')
try:
NETWORKAPI_USE_FOREMAN = int(get_variable('use_foreman'))
NETWORKAPI_FOREMAN_URL = get_variable('foreman_url')
NETWORKAPI_FOREMAN_USERNAME = get_variable('foreman_username')
NETWORKAPI_FOREMAN_PASSWORD = get_variable('foreman_password')
FOREMAN_HOSTS_ENVIRONMENT_ID = get_variable(
'foreman_hosts_environment_id')
except ObjectDoesNotExist:
raise var_exceptions.VariableDoesNotExistException(
'Erro buscando as variáveis relativas ao Foreman.')
# begin - Create Foreman entries for rack switches
if NETWORKAPI_USE_FOREMAN:
foreman = Foreman(NETWORKAPI_FOREMAN_URL, (NETWORKAPI_FOREMAN_USERNAME,
NETWORKAPI_FOREMAN_PASSWORD), api_version=2)
# for each switch, check the switch ip against foreman know networks, finds foreman hostgroup
# based on model and brand and inserts the host in foreman
# if host already exists, delete and recreate with new information
for [switch, mac] in [[rack.id_sw1, rack.mac_sw1], [rack.id_sw2, rack.mac_sw2], [rack.id_ilo, rack.mac_ilo]]:
# Get all foremand subnets and compare with the IP address of the
# switches until find it
if mac is None:
raise RackConfigError(
None, rack.nome, ('Could not create entry for %s. There is no mac address.' % (switch.nome)))
ip = buscar_ip(switch.id)
if ip is None:
raise RackConfigError(
None, rack.nome, ('Could not create entry for %s. There is no management IP.' % (switch.nome)))
switch_cadastrado = 0
for subnet in foreman.subnets.index()['results']:
network = IPNetwork(ip + '/' + subnet['mask']).network
# check if switches ip network is the same as
# subnet['subnet']['network'] e subnet['subnet']['mask']
if network.__str__() == subnet['network']:
subnet_id = subnet['id']
hosts = foreman.hosts.index(search=switch.nome)['results']
if len(hosts) == 1:
foreman.hosts.destroy(id=hosts[0]['id'])
elif len(hosts) > 1:
raise RackConfigError(
None, rack.nome, ('Could not create entry for %s. There are multiple entries with the sam name.' % (switch.nome)))
# Lookup foreman hostgroup
# By definition, hostgroup should be Marca+"_"+Modelo
hostgroup_name = switch.modelo.marca.nome + '_' + switch.modelo.nome
hostgroups = foreman.hostgroups.index(
search=hostgroup_name)
if len(hostgroups['results']) == 0:
raise RackConfigError(None, rack.nome, 'Could not create entry for %s. Could not find hostgroup %s in foreman.' % (
switch.nome, hostgroup_name))
elif len(hostgroups['results']) > 1:
raise RackConfigError(None, rack.nome, 'Could not create entry for %s. Multiple hostgroups %s found in Foreman.' % (
switch.nome, hostgroup_name))
else:
hostgroup_id = hostgroups['results'][0]['id']
host = foreman.hosts.create(host={'name': switch.nome, 'ip': ip, 'mac': mac, 'environment_id': FOREMAN_HOSTS_ENVIRONMENT_ID,
'hostgroup_id': hostgroup_id, 'subnet_id': subnet_id, 'build': 'true', 'overwrite': 'true'})
switch_cadastrado = 1
if not switch_cadastrado:
raise RackConfigError(None, rack.nome, "Unknown error. Could not create entry for %s in foreman." % (switch.nome))
#end - Create Foreman entries for rack switches
var1 = autoprovision_splf(num_rack, FILEINLF1, FILEINLF2, FILEINSP1, FILEINSP2, FILEINSP3, FILEINSP4,
name_lf1, name_lf2, name_oob, name_sp1, name_sp2, name_sp3, name_sp4,
ip_mgmtlf1, ip_mgmtlf2,
int_oob_mgmtlf1, int_oob_mgmtlf2, int_sp1, int_sp2, int_sp3, int_sp4, int_lf1_sp1,
int_lf1_sp2, int_lf2_sp3, int_lf2_sp4)
var2 = autoprovision_coreoob(num_rack, FILEINCR1, FILEINCR2, FILEINOOB,
name_core1, name_core2, name_oob, name_lf1, name_lf2,
ip_mgmtoob, int_oob_core1, int_oob_core2, int_core1_oob, int_core2_oob )
if var1 and var2:
return True
return False
def main():
global debugmode
args = parser.parse_args()
if args.debug:
logger.setLevel(logging.DEBUG)
debugmode = True
if args.quiet:
logger.setLevel(logging.ERROR)
debugmode = False
onlynodes = []
if args.onlynode:
onlynodes = args.onlynode
config = get_config(args.config)
host = config['foreman']['host']
user = config['foreman']['user']
password = config['foreman']['pass']
f = Foreman('https://%s' % host, (user, password), verify=False, api_version=1)
node_ids = []
for node in f.hosts.index(per_page=100000):
node = node['host']
if len(onlynodes) > 0 and node['name'] in onlynodes:
node_ids.append(node['id'])
elif len(onlynodes) == 0:
node_ids.append(node['id'])
facts_query = 'fqdn or memorysize_mb or is_virtual or processorcount or processors::models or serialnumber'
nodes = []
for node_id in node_ids:
host = f.hosts.show(id=node_id)['host']
if len(onlynodes) > 0 and host['name'] not in onlynodes:
continue
host['model'] = f.models.show(id=host['model_id'])
host['os'] = f.operatingsystems.show(id=host['operatingsystem_id'])
facts = f.do_get('/api/hosts/%s/facts?search=%s&per_page=999' % (node_id, facts_query), {})
facts = facts[host['name']] if host['name'] in facts else {}
disks = f.do_get('/api/hosts/%s/facts?search=disks&per_page=999' % node_id, {})
disks = disks[host['name']] if host['name'] in disks else {}
ec2_metadata = f.do_get('/api/hosts/%s/facts?search=ec2_metadata&per_page=999' % node_id, {})
ec2_metadata = ec2_metadata[host['name']] if host['name'] in ec2_metadata else {}
networking = f.do_get('/api/hosts/%s/facts?search=networking&per_page=999' % node_id, {})
networking = networking[host['name']] if host['name'] in networking else {}
if 'networking::interfaces' in networking:
networking2 = {}
for x in networking:
value = networking[x]
if value:
value = value.replace('"=>', '":')
chain = x.split('::')
last = False
el = 0
obj = networking2
while not last:
key = chain[el:el+1][0]
el += 1
if key not in obj:
obj[key] = {}
obj = obj[key]
if len(chain) == el:
last = True
obj[key] = value
def clean_dict(obj):
for x in obj:
try:
obj[x] = obj[x][x]
except:
pass
try:
if type(obj[x]) == dict:
obj[x] = clean_dict(obj[x])
except:
pass
return obj
networking = {
'networking': json.dumps(clean_dict(networking2['networking']))
}
if facts == {}:
continue
formatted_disks = {}
for key in disks:
splitted = key.split('::')
if len(splitted) == 3:
if splitted[2] == 'size_bytes':
formatted_disks[splitted[1]] = {
'size_bytes': disks[key]
}
# Check to see that we have all data, or set it to '' if not
if facts.has_key('is_virtual'):
_is_virtual = facts['is_virtual']
else:
_is_virtual = False
if facts.has_key('serialnumber'):
_serialnumber = facts['serialnumber']
else:
_serialnumber = ''
if facts.has_key('processors::models'):
_processors_models = ast.literal_eval(facts['processors::models'])
else:
_processors_models = ['']
# prepare correct format
data = {
'hostname': host['name'],
'memorysize_mb': facts['memorysize_mb'],
'fqdn': facts['fqdn'],
'disks': formatted_disks,
'is_virtual': _is_virtual,
'serial_no': _serialnumber,
'physicalprocessorcount': facts['physicalprocessorcount'],
'processorcount': facts['processorcount'],
'processors': {
'models': _processors_models
},
'operatingsystem': host['os']['operatingsystem']['name'],
'operatingsystemrelease': host['os']['operatingsystem']['release_name'],
'macaddress': host['mac'],
'networking': json.loads(networking['networking'].replace('"=>', '":')) if 'networking' in networking else ''
}
if len(ec2_metadata) > 0:
data.update({'ec2_metadata': ec2_metadata})
nodes.append(data)
if args.savenodes:
with open(args.savenodes, 'w') as wnf:
wnf.write(json.dumps(nodes, cls=JSONEncoder, indent=4, sort_keys=True, ensure_ascii=False))
dev42 = device42.Device42(
endpoint=config['device42']['host'],
user=config['device42']['user'],
password=config['device42']['pass'],
logger=logger,
debug=debugmode
)
d42_update(dev42, nodes, config['options'], config.get('static', {}), config.get('mapping', {}),
from_version='4', puppethost=config['foreman']['host'])
return 0
class Client(object):
def __init__(self, config_path, debug=False, version='1', log=None):
self.config = utils.get_config(config_path)
self.logger = utils.get_logger(__name__, self.config, debug, log)
config = self.get_config_section('foreman')
self.logger.debug('=> config file: %s' % config_path)
self.logger.debug('=> foreman url: %s' % config['url'])
self.foreman = Foreman(config['url'],
(config['user'], config['password']),
api_version=2,
version=version,
verify=False)
def get_config(self, section, option):
try:
value = self.config.get(section, option)
return value
except ConfigParser.NoOptionError:
self.logger.debug('=> config file section [%s] missing option %s'
% (section, option))
except ConfigParser.NoSectionError:
self.logger.debug('=> config file missing section %s' % section)
return None
def get_config_section(self, section):
try:
openstack = self.config.items(section)
except ConfigParser.NoSectionError:
self.logger.exception('missing [%s]' % section)
self.logger.critical('Could not find section [%s] in %s', section, self.config_path)
sys.exit(1)
return dict(openstack)
def get_logger(self):
return self.logger
def get_client(self):
return self.foreman
def get_compute_resources(self):
resources = self.foreman.index_computeresources()
found_resources = dict({})
for r in resources['results']:
found_resources[r['name']] = r['id']
return found_resources
def get_host(self, host):
host = self.__set_host(host)
return self.foreman.show_hosts(id=host)
def set_host_build(self, host, build=True):
host = self.__set_host(host)
if len(self.foreman.show_hosts(id=host)) > 0:
self.foreman.update_hosts(id=host, host={'build': build})
def get_hosts(self, search=None):
hosts = self.foreman.index_hosts()
self.logger.debug("=> fetch %s page(s) with a total of %s hosts" %
(hosts['page'], hosts['total']))
return hosts
def create_host(self, host):
if 'name' not in host:
self.logger.critical('host dict missing name')
return
self.logger.debug('=> create new host %s' % host['name'])
result = self.foreman.create_host(host)
self.logger.debug('=> host created: %s' % result)
def create_node(self, name, node_data, region, dry_run=False):
if self.get_host(name):
self.logger.debug('=> node %s found, dropping create' % name)
return
found_resources = self.get_compute_resources()
host = dict()
host['name'] = name
host['build'] = self.__get_node_data('build', node_data, '1')
host['hostgroup_id'] = self.__get_node_data('hostgroup', node_data, '1')
host['compute_profile_id'] = self.__get_node_data('compute_profile', node_data, '1')
host['interfaces_attributes'] = self.__get_node_data(
'interfaces_attributes', node_data, {})
host['compute_attributes'] = self.__get_node_data(
'compute_attributes', node_data, {})
host['host_parameters_attributes'] = self.__get_node_data(
'host_parameters_attributes', node_data, {})
if 'mac' in node_data:
host['mac'] = node_data['mac']
if 'compute_resource' in node_data:
compute_resource = '%s-%s' % (region, node_data['compute_resource'])
if compute_resource in found_resources:
host['compute_resource_id'] = found_resources[compute_resource]
else:
self.logger.critical('=> compute resource %s not found' % compute_resource)
return
elif 'mac' not in node_data:
self.logger.critical('=> mac or compute resource are mandatory for %s' % name)
return
if not dry_run:
result = self.foreman.create_hosts(host)
if not result:
self.log_error('Could not create host. Check production.log on foreman host!')
return
if 'mac' not in node_data:
self.foreman.hosts.power(id=result['name'], power_action='start')
self.logger.debug('=> create host %s' % result)
else:
self.logger.debug('=> dry run: host config %s' % host)
def __set_host(self, host):
if not host:
self.host = None
return
domain = self.config.get('openstack', 'domain')
if domain and not '.' in host:
self.logger.debug("=> prepend %s to %s" % (domain, host))
host = host + '.' + domain
return host
@staticmethod
def log_error(msg, code=0):
sys.stderr.write("%s\n" % msg)
if code > 0:
sys.exit(code)
@staticmethod
def __get_node_data(var, node_data, default=None):
if var in node_data:
return node_data[var]
else:
return default
class ForemanClient(Client):
per_page = 100
def __init__(self, config_path, debug=False, version='1', log=None):
super(ForemanClient, self).__init__(config_path, debug, log)
self.logger.debug('=> config file: %s' % config_path)
foreman_url = self.get_config('foreman', 'url')
self.logger.debug('=> foreman url: %s' % foreman_url)
foreman_user = self.get_config('foreman', 'user')
foreman_password = self.get_config('foreman', 'password')
self.foreman = Foreman(foreman_url, (foreman_user, foreman_password),
api_version=2,
version=version,
verify=False)
def set_per_page(self, per_page):
self.per_page = per_page
def get_config(self, section, option):
try:
value = self.config.get(section, option)
return value
except ConfigParser.NoOptionError:
self.logger.debug('=> config file section [%s] missing option %s' %
(section, option))
except ConfigParser.NoSectionError:
self.logger.debug('=> config file missing section %s' % section)
return None
def get_config_section(self, section):
try:
openstack = self.config.items(section)
except ConfigParser.NoSectionError:
self.logger.debug('missing [%s]' % section)
self.logger.debug('Could not find section [%s] in %s', section,
self.config_path)
sys.exit(1)
return dict(openstack)
def get_location(self):
locations = self.foreman.index_locations()
location_id = False
for l in locations['results']:
if l['name'] == 'Default Location':
location_id = l['id']
return location_id
def get_organization(self):
organizations = self.foreman.index_organizations()
organization_id = False
for o in organizations['results']:
if o['name'] == 'Default Organization':
organization_id = o['id']
return organization_id
def get_logger(self):
return self.logger
def get_client(self):
return self.foreman
def get_compute_resources(self):
resources = self.foreman.index_computeresources()
found_resources = dict({})
for r in resources['results']:
found_resources[r['name']] = r['id']
return found_resources
def get_compute_profiles(self):
profiles = self.foreman.index_computeprofiles()
found_profiles = dict({})
for p in profiles['results']:
found_profiles[p['name']] = p['id']
return found_profiles
def get_profile_id(self, profile_name):
profile = self.foreman.show_computeprofiles(profile_name)
return profile['id']
def get_host(self, host):
host = self.__set_host(host)
return self.foreman.show_hosts(id=host)
def get_fact(self, host, fact):
host = self.__set_host(host)
facts = self.get_facts(host)
fact = facts['results'][host][fact]
return fact
def get_facts(self, host_id):
host = self.__set_host(host_id)
return self.foreman.hosts.fact_values_index(host_id=host,
per_page=self.per_page)
def set_host_build(self, host, build=True):
host = self.__set_host(host)
if len(self.foreman.show_hosts(id=host)) > 0:
self.foreman.update_hosts(id=host, host={'build': build})
def get_hosts(self, search=None):
hosts = self.foreman.index_hosts(per_page=self.per_page)
self.logger.debug("=> fetch %s page(s) with a total of %s hosts" %
(hosts['page'], hosts['total']))
return hosts
def create_host(self, host):
if 'name' not in host:
self.logger.debug('host dict missing name')
return
self.logger.debug('=> create new host %s' % host['name'])
result = self.foreman.create_host(host)
self.logger.debug('=> host created: %s' % result)
def create_node(self, name, node_data, region):
if self.get_host(name):
self.logger.debug('=> node %s found, dropping create' % name)
return
found_resources = self.get_compute_resources()
host = dict()
host['name'] = name
host['build'] = self.__get_node_data('build', node_data, '1')
host['hostgroup_id'] = self.__get_node_data('hostgroup', node_data,
'1')
host['compute_profile_id'] = self.get_profile_id(
self.__get_node_data('compute_profile', node_data, 'small'))
host['organization_id'] = self.get_organization()
host['location_id'] = self.get_location()
host['interfaces_attributes'] = self.__get_node_data(
'interfaces_attributes', node_data, {})
host['compute_attributes'] = self.__get_node_data(
'compute_attributes', node_data, {})
host['host_parameters_attributes'] = self.__get_node_data(
'host_parameters_attributes', node_data, {})
if 'mac' in node_data:
host['mac'] = node_data['mac']
if 'compute_resource' in node_data:
compute_resource = '%s-%s' % (region,
node_data['compute_resource'])
if compute_resource in found_resources:
host['compute_resource_id'] = found_resources[compute_resource]
else:
self.logger.debug('=> compute resource %s not found' %
compute_resource)
return
elif 'mac' not in node_data:
self.logger.debug(
'=> mac or compute resource are mandatory for %s' % name)
return
if not self.dry_run:
result = self.foreman.create_hosts(host)
if not result:
self.log_error(
'Could not create host. Check production.log on foreman host!'
)
return
if 'mac' not in node_data:
self.foreman.hosts.power(id=result['name'],
power_action='start')
self.logger.debug('=> create host %s' % result)
else:
self.logger.debug('=> dry run: host config %s' % host)
def delete_node(self, host):
host = self.__set_host(host)
if not self.dry_run:
result = self.foreman.destroy_hosts(host)
if not result:
self.log_error('Could not delete host.')
return
self.logger.debug('=> deleted node %s' % host)
else:
self.logger.debug('=> dry run: deleted node %s' % host)
def __set_host(self, host):
if not host:
self.host = None
return
domain = self.config.get('openstack', 'domain')
if domain and not '.' in host:
self.logger.debug("=> prepend %s to %s" % (domain, host))
host = host + '.' + domain
return host
@staticmethod
def log_error(msg, code=0):
sys.stderr.write("%s\n" % msg)
if code > 0:
sys.exit(code)
@staticmethod
def __get_node_data(var, node_data, default=None):
if var in node_data:
return node_data[var]
else:
return default
def generate(self):
api_definition = self.cli_args.api_definition
if not api_definition:
api_definition_list = glob.glob(API_DEFINITION_GLOB_PATH)
# find newest api def (by name)
api_definition = max(api_definition_list)
self.client = Foreman('https://{}/'.format(self.cli_args.server),
auth=(self.cli_args.username, self.cli_args.password),
api_version=2,
use_cache=True,
cache_dir=api_definition)
else:
defs_path = os.path.join(api_definition, 'definitions')
api_def_paths = glob.glob('{}/*-v{}.json'.format(defs_path, self.api_version))
for p in api_def_paths:
# Filename fetched from fetch_api() has the following pattern:
# <version>-<api_version>.json
#
# Example:
# Satellite-v2.json
# 1.15.1-v2.json
#
# We need to pass in <version> and <api_version> into the Foreman()
# constructure, so the following code pulls those variables from
# the filename.
api_def = os.path.basename(p)
api_def = api_def.replace('.json', '')
api_def_parts = api_def.split('-')
self.version = api_def_parts[0]
self.api_version = api_def_parts[1].replace('v', '')
break
self.client = Foreman('https://usingcache/',
version=self.version,
api_version=2,
use_cache=True,
cache_dir=api_definition)
# all methods of the client
unused = {'methods': [],
'unknown_extras': []}
results = {'resources': [],
'special_attributes': []}
special_attributes = ['api_version', 'version']
for attr in inspect.getmembers(self.client):
if inspect.ismethod(attr[1]):
unused['methods'].append(attr)
elif isinstance(attr[1], Resource):
if attr[0] == "":
continue
results['resources'].append(attr)
elif attr[0] in special_attributes:
results['special_attributes'].append(attr)
else:
unused['unknown_extras'].append(attr)
# get all of the methods
method_defs = self.gather_method_defs(results['resources'])
# convert the methods into actions
self.render_method_defs_into_actions(method_defs)
self.render_method_defs_into_table(method_defs)
def connectToHost(host, host_user, host_pwd):
apiurl = "http://" + host
#insecure -> skips SSL check
api = Foreman(apiurl, (host_user, host_pwd), api_version=2)
return api
#!/usr/bin/python3
#
# Create records in Foreman from CSV
#
# Foreman pkg docs: http://python-foreman.readthedocs.io/en/latest/basic/
# Foreman api docs: https://theforeman.org/api/1.14/index.html
import sys
import fileinput
from foreman.client import Foreman
with open("password_foreman.txt") as f:
password_foreman = f.readline().rstrip()
fore = Foreman(url="https://foreman-crn.acis.ufl.edu",
api_version=2,
auth=('mjcollin', password_foreman))
for line in fileinput.input():
fields = line.split(",")
fqdn = fields[0]
hostname = fields[1]
ip = fields[2]
mac = fields[3]
fore.hosts.create(
host={
"name": fqdn,
"ip": ip,
"mac": mac,
"build": True,
from getpass import getpass
from foreman.client import Foreman
f = Foreman('https://192.168.99.100:8443', ('admin', 'changeme'))
print(f.index_hosts())
print(f.show_hosts(id=1))
res = f.create_hosts(host={'name': 'mynewhost', 'ip': '192.168.1.1', 'mac': '00:00:00:00:00:00'})
class ForemanClient(Client):
per_page = 100
def __init__(self, config_path, debug=False, version='1', log=None):
super(ForemanClient, self).__init__(config_path, debug, log)
self.logger.debug('=> config file: %s' % config_path)
foreman_url = self.get_config('foreman', 'url')
self.logger.debug('=> foreman url: %s' % foreman_url)
foreman_user = self.get_config('foreman', 'user')
foreman_password = self.get_config('foreman', 'password')
self.foreman = Foreman(foreman_url,
(foreman_user, foreman_password),
api_version=2,
version=version,
verify=False)
def set_per_page(self, per_page):
self.per_page = per_page
def get_config(self, section, option):
try:
value = self.config.get(section, option)
return value
except ConfigParser.NoOptionError:
self.logger.debug('=> config file section [%s] missing option %s'
% (section, option))
except ConfigParser.NoSectionError:
self.logger.debug('=> config file missing section %s' % section)
return None
def get_config_section(self, section):
try:
openstack = self.config.items(section)
except ConfigParser.NoSectionError:
self.logger.debug('missing [%s]' % section)
self.logger.debug('Could not find section [%s] in %s', section, self.config_path)
sys.exit(1)
return dict(openstack)
def get_logger(self):
return self.logger
def get_client(self):
return self.foreman
def get_compute_resources(self):
resources = self.foreman.index_computeresources()
found_resources = dict({})
for r in resources['results']:
found_resources[r['name']] = r['id']
return found_resources
def get_compute_profiles(self):
profiles = self.foreman.index_computeprofiles()
found_profiles = dict({})
for p in profiles['results']:
found_profiles[p['name']] = p['id']
return found_profiles
def get_profile_id(self, profile_name):
profile = self.foreman.show_computeprofiles(profile_name)
return profile['id']
def get_host(self, host):
host = self.__set_host(host)
return self.foreman.show_hosts(id=host)
def get_fact(self, host, fact):
host = self.__set_host(host)
facts = self.get_facts(host)
fact = facts['results'][host][fact]
return fact
def get_facts(self, host_id):
host = self.__set_host(host_id)
return self.foreman.hosts.fact_values_index(host_id=host, per_page=self.per_page)
def set_host_build(self, host, build=True):
host = self.__set_host(host)
if len(self.foreman.show_hosts(id=host)) > 0:
self.foreman.update_hosts(id=host, host={'build': build})
def get_hosts(self, search=None):
hosts = self.foreman.index_hosts(per_page=self.per_page)
self.logger.debug("=> fetch %s page(s) with a total of %s hosts" %
(hosts['page'], hosts['total']))
return hosts
def create_host(self, host):
if 'name' not in host:
self.logger.debug('host dict missing name')
return
self.logger.debug('=> create new host %s' % host['name'])
result = self.foreman.create_host(host)
self.logger.debug('=> host created: %s' % result)
def create_node(self, name, node_data, region):
if self.get_host(name):
self.logger.debug('=> node %s found, dropping create' % name)
return
found_resources = self.get_compute_resources()
host = dict()
host['name'] = name
host['build'] = self.__get_node_data('build', node_data, '1')
host['hostgroup_id'] = self.__get_node_data('hostgroup', node_data, '1')
host['compute_profile_id'] = self.get_profile_id(
self.__get_node_data('compute_profile',
node_data,
'small'))
host['interfaces_attributes'] = self.__get_node_data(
'interfaces_attributes', node_data, {})
host['compute_attributes'] = self.__get_node_data(
'compute_attributes', node_data, {})
host['host_parameters_attributes'] = self.__get_node_data(
'host_parameters_attributes', node_data, {})
if 'mac' in node_data:
host['mac'] = node_data['mac']
if 'compute_resource' in node_data:
compute_resource = '%s-%s' % (region, node_data['compute_resource'])
if compute_resource in found_resources:
host['compute_resource_id'] = found_resources[compute_resource]
else:
self.logger.debug('=> compute resource %s not found' % compute_resource)
return
elif 'mac' not in node_data:
self.logger.debug('=> mac or compute resource are mandatory for %s' % name)
return
if not self.dry_run:
result = self.foreman.create_hosts(host)
if not result:
self.log_error('Could not create host. Check production.log on foreman host!')
return
if 'mac' not in node_data:
self.foreman.hosts.power(id=result['name'], power_action='start')
self.logger.debug('=> create host %s' % result)
else:
self.logger.debug('=> dry run: host config %s' % host)
def delete_node(self, host):
host = self.__set_host(host)
if not self.dry_run:
result = self.foreman.destroy_hosts(host)
if not result:
self.log_error('Could not delete host.')
return
self.logger.debug('=> deleted node %s' % host)
else:
self.logger.debug('=> dry run: deleted node %s' % host)
def __set_host(self, host):
if not host:
self.host = None
return
domain = self.config.get('openstack', 'domain')
if domain and not '.' in host:
self.logger.debug("=> prepend %s to %s" % (domain, host))
host = host + '.' + domain
return host
@staticmethod
def log_error(msg, code=0):
sys.stderr.write("%s\n" % msg)
if code > 0:
sys.exit(code)
@staticmethod
def __get_node_data(var, node_data, default=None):
if var in node_data:
return node_data[var]
else:
return default
def create_host(cfg, host):
stime = time.time()
log.info('Creating host: %s' % host['name'])
log.debug('Host %s: %s' % (host['name'], host))
if 'timeout' in cfg:
timeout = int(cfg['timeout'])
else:
timeout = 60
if 'timeout_post' in cfg:
timeout_post = int(cfg['timeout_post'])
else:
timeout_post = 600
if 'timeout_delete' in cfg:
timeout_delete = int(cfg['timeout_delete'])
else:
timeout_delete = 600
if 'use_cache' in cfg:
use_cache = cfg['use_cache']
else:
use_cache = True
if 'verify' in cfg:
verify = cfg['verify']
else:
verify = False
foreman_api = Foreman(url=cfg['server'],
auth=(cfg['username'], cfg['password']),
api_version=cfg['api_version'],
timeout=timeout,
timeout_post=timeout_post,
timeout_delete=timeout_delete,
use_cache=use_cache,
verify=verify)
host_dict = dict()
# quick hack to check if hosts exists
# TODO: check if domain provided via yaml/hostgroup
_results = foreman_api.hosts.index(search='name ~ %s.%%' %
host['name'])['results']
if _results:
log.warn('Host %s already exists with id: %s' %
(host['name'], _results[0]['id']))
return True
for key in host:
if key == 'hostgroup':
hostgroup_id = foreman_api.hostgroups.show(
id=host['hostgroup'])['id']
if hostgroup_id:
host_dict['hostgroup_id'] = hostgroup_id
else:
log.warn('Could not match an id for hostgroup: %s' %
host['hostgroup'])
elif key == 'subnet':
subnet_id = foreman_api.subnets.show(id=host['subnet'])['id']
if subnet_id:
host_dict['subnet_id'] = subnet_id
else:
log.warn('Could not match an id for subnet: %s' %
host['subnet'])
elif key == 'compute_profile':
compute_profile_id = foreman_api.compute_profiles.show(
id=host['compute_profile'])['id']
if compute_profile_id:
host_dict['compute_profile_id'] = compute_profile_id
else:
log.warn('Could not match an id for compute_profile: %s' %
host['compute_profile'])
elif key == 'host_parameters':
host_dict['host_parameters_attributes'] = host['host_parameters']
else:
host_dict[key] = host[key]
log.debug('%s host_dict: %s' % (host['name'], host_dict))
try:
foreman_api.hosts.create(host=host_dict)
except ForemanException, e:
log.error('%s ForemanException: %s' % (host['name'], e))
item['owner'] = owner
item['rsa_fp'] = rsa['fpsha1']
item['rsa_keytype'] = rsa['keytype']
item['dsa_fp'] = dsa['fpsha1']
item['dsa_keytype'] = dsa['keytype']
if ecdsa:
item['ecdsa_fp'] = ecdsa['fpsha1']
item['ecdsa_keytype'] = ecdsa['keytype']
keylist.append(item)
else:
from foreman.client import Foreman
if not opts.foremanpassword:
password = getpass()
f = Foreman(opts.foremanurl, (opts.foremanusername, password))
key_map = {
'dsa': ('sshdsakey', 'ssh-dss'),
'rsa': ('sshrsakey', 'ssh-rsa'),
'ecdsa': ('sshecdsakey', 'ssh-ecdsa'),
'ed25519': ('sshed25519key', 'ssh-ed25519'),
}
for keytype in key_map:
fact_name, ssh_key_type = key_map[keytype]
facts = f.fact_values.index(999, search="name=%s" % fact_name)
for host in facts:
key = create_sshfp(host, ssh_key_type, facts[host][fact_name])
item = {}
item['%s_fp' % keytype] = key['fpsha1']
item['%s_keytype' % keytype] = key['keytype']
def api_foreman(rack):
try:
NETWORKAPI_FOREMAN_URL = get_variable("foreman_url")
NETWORKAPI_FOREMAN_USERNAME = get_variable("foreman_username")
NETWORKAPI_FOREMAN_PASSWORD = get_variable("foreman_password")
FOREMAN_HOSTS_ENVIRONMENT_ID = get_variable(
"foreman_hosts_environment_id")
except ObjectDoesNotExist:
raise var_exceptions.VariableDoesNotExistException(
"Erro buscando as variáveis relativas ao Foreman.")
foreman = Foreman(
NETWORKAPI_FOREMAN_URL,
(NETWORKAPI_FOREMAN_USERNAME, NETWORKAPI_FOREMAN_PASSWORD),
api_version=2)
# for each switch, check the switch ip against foreman know networks, finds foreman hostgroup
# based on model and brand and inserts the host in foreman
# if host already exists, delete and recreate with new information
for [switch, mac] in [[rack.id_sw1, rack.mac_sw1],
[rack.id_sw2, rack.mac_sw2],
[rack.id_ilo, rack.mac_ilo]]:
# Get all foremand subnets and compare with the IP address of the switches until find it
if mac == None:
raise RackConfigError(
None, rack.nome,
("Could not create entry for %s. There is no mac address." %
switch.nome))
ip = _buscar_ip(switch.id)
if ip == None:
raise RackConfigError(
None, rack.nome,
("Could not create entry for %s. There is no management IP." %
switch.nome))
switch_cadastrado = 0
for subnet in foreman.subnets.index()['results']:
network = IPNetwork(ip + '/' + subnet['mask']).network
# check if switches ip network is the same as subnet['subnet']['network'] e subnet['subnet']['mask']
if network.__str__() == subnet['network']:
subnet_id = subnet['id']
hosts = foreman.hosts.index(search=switch.nome)['results']
if len(hosts) == 1:
foreman.hosts.destroy(id=hosts[0]['id'])
elif len(hosts) > 1:
raise RackConfigError(None, rack.nome, (
"Could not create entry for %s. There are multiple entries "
"with the sam name." % switch.nome))
# Lookup foreman hostgroup
# By definition, hostgroup should be Marca+"_"+Modelo
hostgroup_name = switch.modelo.marca.nome + "_" + switch.modelo.nome
hostgroups = foreman.hostgroups.index(search=hostgroup_name)
if len(hostgroups['results']) == 0:
raise RackConfigError(
None, rack.nome,
"Could not create entry for %s. Could not find hostgroup %s "
"in foreman." % (switch.nome, hostgroup_name))
elif len(hostgroups['results']) > 1:
raise RackConfigError(
None, rack.nome,
"Could not create entry for %s. Multiple hostgroups %s found"
" in Foreman." % (switch.nome, hostgroup_name))
else:
hostgroup_id = hostgroups['results'][0]['id']
host = foreman.hosts.create(
host={
'name': switch.nome,
'ip': ip,
'mac': mac,
'environment_id': FOREMAN_HOSTS_ENVIRONMENT_ID,
'hostgroup_id': hostgroup_id,
'subnet_id': subnet_id,
'build': 'true',
'overwrite': 'true'
})
switch_cadastrado = 1
if not switch_cadastrado:
raise RackConfigError(
None, rack.nome,
"Unknown error. Could not create entry for %s in foreman." %
switch.nome)
def clean_old_host():
""" Method call by cron to clean instances """
logging.info("########## Start Cleaning ###########")
metrics = {
'hosts_ok': {
'description':
'count of hosts which have a report in the defined delay',
'value': 0
},
'hosts_deleted': {
'description':
'count of hosts successfully deleted from foreman puppet and ds',
'value': 0
},
'hosts_skipped': {
'description':
'count of hosts skipped because they are still in EC2',
'value': 0
},
'hosts_delete_failed': {
'description':
'count hosts unsuccessfully deleted from foreman puppet and ds',
'value': 0
},
}
# connect to Foreman and ForemanProxy
f = Foreman(FOREMAN_URL, (FOREMAN_USER, FOREMAN_PASSWORD), api_version=2)
fp = ForemanProxy(FOREMAN_PROXY_URL)
# Connect to the DS
try:
ds = AwsDs(LDAP_HOST, COMPUTERS_BASE_DN, BIND_USER_DN, BIND_PASSWORD)
except ldap.INVALID_CREDENTIALS:
raise "Your username or password is incorrect."
# Get the the current date
currentdate = datetime.datetime.utcnow()
# check for all host
instance_id_dict = foreman_wrapper(
f.do_get,
call_args={
'url': '/api/fact_values?&search=+name+%3D+ec2_instance_id',
'kwargs': {
'per_page': 1000
}
})
result = foreman_wrapper(f.index_hosts, call_args={"per_page": 1000})
for host in result:
# get the compile date
lastcompile = None
if host["last_compile"]:
lastcompile = host["last_compile"]
elif host["last_report"]:
lastcompile = host["last_report"]
elif host["created_at"]:
lastcompile = host["created_at"]
# Convert the string date to datetime format
if not host["last_compile"] and not host["last_report"] and host[
"created_at"]:
logging.info(
"Can't retrieve last compile/report date for {}, will use create time ({})"
.format(host["certname"], host["created_at"]))
hostdate = datetime.datetime.strptime(lastcompile,
'%Y-%m-%dT%H:%M:%S.%fZ')
# Get the delta between the last puppet repport and the current date
elapsed = currentdate - hostdate
# if the deta is more than $delay days we delete the host
if elapsed > datetime.timedelta(hours=int(DELAY)):
# Make the following 2 call only at the end in order to avoid useless consuming API call
try:
instance_id = instance_id_dict[host['name']]['ec2_instance_id']
is_terminated = (
get_ec2_instance_state(instance_id) == 'terminated')
except KeyError:
if host['ip']:
is_terminated = (get_ec2_instance_state(
'', ip=host['ip']) == 'terminated')
elif host['mac']:
is_terminated = (get_ec2_instance_state(
'', ip=host['ip'], mac=host['mac']) == 'terminated')
else:
logging.warning(
"Can't retrieve EC2 id or ip, skipping {}".format(
host["certname"]))
metrics["hosts_skipped"]["value"] += 1
continue
except Exception as e:
logging.warning(
"Can't retrieve EC2 state, skipping {} : {}".format(
host["certname"], e))
metrics["hosts_skipped"]["value"] += 1
continue
if is_terminated:
try:
logging.info(
"I will destroy the server {} because the last report was {}"
.format(host["certname"], str(lastcompile)))
# destroy the host in foreman
f.destroy_hosts(id=host["id"])
# remove the certificate in puppet
fp.delete_certificate(host["certname"])
# remove host in the DS
ds.delete_computer(host["certname"])
metrics["hosts_deleted"]["value"] += 1
except Exception as e:
logging.error("Something went wrong : {}".format(e))
metrics["hosts_delete_failed"]["value"] += 1
else:
metrics["hosts_skipped"]["value"] += 1
else:
metrics["hosts_ok"]["value"] += 1
logging.debug("{} OK: Last puppet's run : {}".format(
host["certname"], lastcompile))
logging.info("Push metrics to prometheus")
push_metrics(metrics)
'''
from getpass import getpass
import psycopg2
from foreman.client import Foreman
import csv, time
# create log file
logfile = "log" + time.strftime("_%m_%d_%Y-%H_%M_%S") + ".csv"
with open(logfile, "w") as f:
writer = csv.writer(f)
writer.writerow(["hostname", "hostgroup"])
# create connection object to foreman and object for all nodes
username = raw_input("Enter username: "******"dbname='puppetdb' user='******' host='devitpuppet.tsi.lan' password='******'")
cur = conn.cursor()
cur.execute("""select h.name,hg.name from hosts h left join hostgroups hg on h.hostgroup_id=hg.id""")
rows = cur.fetchall()
# iterate through foreman nodes and add to necessary hostgroup
def clean_ds():
""" This is a 'one time use' method that will clear from the DS all instances that doesn't still exist """
# Retrieve config from ENV
# Stats
saved = 0
deleted = 0
# connect to Foreman and ForemanProxy
f = Foreman(FOREMAN_URL, (FOREMAN_USER, FOREMAN_PASSWORD), api_version=2)
# Connect to the DS
try:
ds = AwsDs(LDAP_HOST, COMPUTERS_BASE_DN, BIND_USER_DN, BIND_PASSWORD)
except ldap.INVALID_CREDENTIALS:
raise "Your username or password is incorrect."
# Get all host from foreman
page = 1
result = []
last_len = 1
while last_len > 0:
tmp_result = f.index_hosts(per_page="1000", page=str(page))['results']
last_len = len(tmp_result)
page += 1
result += tmp_result
foreman_hosts = {host["certname"]: host["ip"] for host in result}
# Get all ds computer
ds_computers = []
for c_dn, attr in ds.computers:
if 'dNSHostName' in attr and re.match('.*\.cloud\.coveo\.com$',
attr['dNSHostName'][0]):
ds_computers.append(attr['dNSHostName'][0].lower())
continue
else:
ds_computers.append(build_from_cn(attr['cn'][0]))
to_delete = ds_computers
"""
to_delete = []
# (Optional) filter which instances should be cleaned
for ds_computer in ds_computers:
if re.match('^npra-al.*', ds_computer) or re.match('^npra-aw.*', ds_computer):
to_delete.append(ds_computer)
"""
# Exlude host that exist in foreman to the list of instances retrieve from the DS
for foreman_host in foreman_hosts.keys():
for i, ds_computer in enumerate(to_delete):
if re.match('^{}.*'.format(foreman_host), ds_computer):
del to_delete[i]
for host in to_delete:
try:
ip_address = socket.gethostbyname(host)
found = True
except:
found = False
# Make the following 2 call only at the end in order to avoid useless consuming API call
try:
if found:
is_terminated = (get_ec2_instance_state(
'', ip=ip_address) == 'terminated')
if not is_terminated:
print(
"{} is not terminated, ignoring this instance".format(
host))
saved += 1
continue
logging.info("I will destroy the server {}".format(host))
# remove host in the DS
ds.delete_computer(host)
deleted += 1
except Exception as e:
logging.error("Something went wrong : {}".format(e))
logging.info(
"{} instances deleted\n{} instances saved\n{} instances in foreman\n".
format(deleted, saved, len(foreman_hosts)))