def test_search_item(self):
self.assertIn('test_1', Search.list())
q = Search('test_1')
self.assertIn('item_1', q)
self.assertIn('item_2', q)
self.assertEqual(q['item_1']['raw_data']['test_attr'], 1)
item = q['item_1'].object
self.assertIsInstance(item, DataBagItem)
self.assertEqual(item['test_attr'], 1)
def test_slice(self):
s = Search('node')[1:2]
self.assertEqual(len(s), 1)
self.assertGreaterEqual(s.total, 3)
s2 = s[1:2]
self.assertEqual(len(s2), 1)
self.assertGreaterEqual(s2.total, 3)
self.assertNotEqual(s[0]['name'], s2[0]['name'])
s3 = Search('node')[2:3]
self.assertEqual(len(s3), 1)
self.assertGreaterEqual(s3.total, 3)
self.assertEqual(s2[0]['name'], s3[0]['name'])
def opencenter_endpoint(chef, name='test', os='ubuntu'):
validate_environment(chef, name=name, os=os)
env = env_format % (name, os)
query = "in_use:\"server\" AND chef_environment:%s" % env
server = next(Node(node['name']) for node in Search('node').query(query))
ep_url = "https://%s:8443" % server['ipaddress']
return OpenCenterEndpoint(ep_url, user="******", password="******")
def chef_roledefs(api=None, hostname_attr='fqdn'):
"""Build a Fabric roledef dictionary from a Chef server.
Example:
from fabric.api import env, run, roles
from chef.fabric import chef_roledefs
env.roledefs = chef_roledefs()
@roles('web_app')
def mytask():
run('uptime')
hostname_attr is the attribute in the chef node that holds the real hostname.
to refer to a nested attribute, separate the levels with '.'.
for example 'ec2.public_hostname'
"""
api = api or ChefAPI.get_global() or autoconfigure()
if not api:
raise ChefError('Unable to load Chef API configuration')
roledefs = {}
for row in Search('role', api=api):
name = row['name']
roledefs[name] = Roledef(name, api, hostname_attr)
return roledefs
def test_search_item(self):
self.assertIn('test_1', Search.list(self.api))
q = Search('test_1', self.api)
self.assertIn('item_1', q)
self.assertIn('item_2', q)
self.assertEqual(q['item_1']['raw_data']['test_attr'], 1)
item = q['item_1'].object
self.assertIsInstance(item, DataBagItem)
self.assertEqual(item['test_attr'], 1)
def node_search(cls, query, environment=None, tries=10):
api = autoconfigure()
if environment:
api = environment.local_api
search = None
while not search and tries > 0:
search = Search("node", api=api).query(query)
sleep(10)
tries = tries - 1
return (n.object for n in search)
def handle(event, _context):
"""Lambda Handler"""
log_event(event)
node_name = None
node_ip = None
# Remove from one of the chef servers
for URL in CHEF_SERVER_URLS:
with ChefAPI(URL, CHEF_PEM, CHEF_USERNAME):
instance_id = get_instance_id(event)
try:
search = Search('node', 'ec2_instance_id:' + instance_id)
except ChefServerNotFoundError as err:
LOGGER.error(err)
return False
if len(search) != 0:
for instance in search:
node_name = instance.object.name
node = Node(node_name)
node_ip = node['ipaddress']
client = Client(node_name)
try:
node.delete()
client.delete()
LOGGER.info(
'=====SUCCESSFULLY REMOVED INSTANCE FROM CHEF SERVER===== {}'
.format(URL))
break
except ChefServerNotFoundError as err:
LOGGER.error(err)
return False
else:
LOGGER.info(
'===Instance does not appear to be Chef Server managed.=== {}'
.format(URL))
# Remove from Spacewalk
spacewalk_cleanup(node_ip)
# Remove from DNS
dns_cleanup(node_name)
# Remove from AD
active_directory_cleanup(node_name)
# Remove fom Solarwinds
solarwinds_cleanup(node_ip, node_name)
# Remove from Chef Automate
chef_automate_cleanup(node_name)
def get_chef_hosts(self):
def get_public_ip(chef_node):
if 'cloud' in chef_node.get('automatic', {}):
return chef_node.get('automatic', {}).get('cloud', {}).get('public_ipv4')
else:
return chef_node.get('automatic', {}).get('ipaddress')
for i in xrange(5):
try:
search_result = Search('node', rows=10000, api=self.api)
if search_result:
return {get_public_ip(node): node for node in search_result if
get_public_ip(node) and IP(get_public_ip(node)).iptype() != 'PRIVATE'}
except ChefServerError:
time.sleep(5)
def load_known_ips(self):
self.logger.debug("Loading public IP list from chef")
nodes = list(Search('node', '*:*', rows=2000, api=self.chef_api))
cloud_ips = [
node.get("automatic", {}).get("cloud", {}).get("public_ips", [])
for node in nodes
]
phy_ifaces = sum([
node["automatic"].get("network", {}).get(
"interfaces", {}).values() for node in nodes
], [])
phy_ips = [i.get("addresses", {}).keys() for i in phy_ifaces]
self.logger.debug("Loading public IP list from AWS")
aws_machines = self.edda_client.soft_clean().query(
"/api/v2/view/instances;_expand")
aws_ips = [m.get("publicIpAddress") for m in aws_machines]
return set(sum(cloud_ips, []) + sum(phy_ips, []) + aws_ips) #flatten
def node_search(cls, query, environment=None, tries=10):
"""
Performs a node search query on the chef server
:param query: search query to request
:type query: string
:param environment: Environment the query should be
:type environment: ChefEnvironment
:rtype: Iterator (chef.Node)
"""
api = autoconfigure()
if environment:
api = environment.local_api
search = None
while not search and tries > 0:
search = Search("node", api=api).query(query)
sleep(10)
tries = tries - 1
return (n.object for n in search)
def do_run(self):
since = self.edda_client._since or 0
chef_hosts = {
row['automatic']['cloud']['public_ipv4']: row['name']
for row in Search('node', 'ec2:*', rows=1000, api=self.api) if
'cloud' in row.get('automatic', {}) and # only store EC2 instances
'public_ipv4' in row['automatic'][
'cloud'] # which have public IP address
}
# print chef_hosts
for machine in self.edda_client.clean().query(
"/api/v2/view/instances;_expand"):
launchTime = int(machine.get("launchTime", 0))
# convert list of tags to a more readable dict
tags = {
tag['key']: tag['value']
for tag in machine.get('tags', [])
if 'key' in tag and 'value' in tag
}
if machine['publicIpAddress'] not in chef_hosts and launchTime >= since and \
tags.get('service_name', None) not in self.excluded_instances and \
machine['instanceId'] not in self.status['first_seen']:
# found a non-chef managed host which has not been seen before and which is not excluded
self.status['first_seen'][machine['instanceId']] = launchTime
yield {
"plugin_name":
self.plugin_name,
"id":
"%s (%s / %s)" %
(machine['instanceId'], machine['publicIpAddress'],
machine['privateIpAddress']),
"details": [{
'tags':
tags,
'keyName':
machine.get('keyName', None),
'securityGroups':
machine.get('securityGroups', [])
}]
}
def get_chef_hosts(self):
for i in xrange(5):
try:
search_result = Search('node',
'ec2:*',
rows=2000,
api=self.api)
if search_result:
return {
row['automatic']['cloud']['public_ipv4']: row['name']
for row in search_result
if 'cloud' in row.get('automatic', {})
and # only store EC2 instances
'public_ipv4' in row['automatic'][
'cloud'] # which have public IP address
}
except ChefServerError:
time.sleep(5)
return None
def test_list(self):
searches = Search.list()
self.assertIn('node', searches)
self.assertIn('role', searches)
def fqdn_for_instance_id(instance_id):
query = "ec2_instance_id:%s" % instance_id
return Search("node", query)[0]['automatic']['fqdn']
def __call__(self):
for row in Search('node', 'roles:' + self.name, api=self.api):
yield row.object.attributes.get_dotted(self.hostname_attr)
def instance_id_for_fqdn(fqdn):
query = "fqdn:%s" % fqdn
nodes = Search("node", query)
return nodes[0]["automatic"]["ec2"]["instance_id"]
def test_start(self):
s = Search('node', start=1)
self.assertEqual(len(s), s.total - 1)
self.assertGreaterEqual(s.total, 3)
def test_search_call(self):
s = Search('node')('role:test_1')
self.assertGreaterEqual(len(s), 2)
self.assertIn('test_1', s)
self.assertNotIn('test_2', s)
self.assertIn('test_3', s)
def test_search_sort_desc(self):
s = Search('node', sort='test_sort_key desc')
self.assertGreater(s.index('test_1'), s.index('test_3'))
def test_search_sort_asc(self):
s = Search('node', sort='test_sort_key asc')
self.assertLess(s.index('test_1'), s.index('test_3'))
def validate_environment(chef, name='test', os='ubuntu'):
env = env_format % (name, os)
if not Search("environment").query("name:%s" % env):
print "environment %s not found" % env
sys.exit(1)
def test_list(self):
searches = Search.list()
self.assertIn('node', searches)
self.assertIn('role', searches)
def command(self):
# Initialization
self.api = _get_chef_api(self.settings.get('chef.url'),
toChefUsername(self.options.chef_username),
self.options.chef_pem, False,
self.settings.get('chef.version'))
self.auth_user = self.db.adminusers.find_one(
{'username': self.options.chef_username})
if self.auth_user is None:
logger.error('The administrator user must exist in MongoDB')
return
self.db = self.pyramid.db
ous = []
groups = []
computers = []
# Get packages policy ID
packages_policy = self.db.policies.find_one({"slug": "package_res"})
if packages_policy is None:
logger.error('Can\'t detect "package_res" policy!')
return
if 'pkgs_to_remove' in packages_policy["schema"]["properties"]:
logger.error(
"The 'package_res' policy in the system is deprecated, please update to new package policy!"
)
return
logger.info('Packages policy ID: %s' % (str(packages_policy['_id'])))
# Get all nodes with old package policy data
logger.info('Getting all nodes with old package policy data...')
path_to_find = "policies.%s.pkgs_to_remove" % (str(
packages_policy['_id']))
old_policy_nodes = self.db.nodes.find({
'$query': {
path_to_find: {
'$exists': True
}
},
'$orderby': {
"path": 1
}
})
updated_nodes = []
for node in old_policy_nodes:
logger.info('Updating node %s ...' % (str(node['_id'])))
updated_nodes.append(str(node['_id']))
logger.debug('Packages to add: %s' % (str(node['policies'][str(
packages_policy['_id'])]['package_list'])))
logger.debug('Packages to remove: %s' % (str(node['policies'][str(
packages_policy['_id'])]['pkgs_to_remove'])))
# Join the lists
package_list = []
for package_name in node['policies'][str(
packages_policy['_id'])]['package_list']:
package_list.append({
'name': package_name,
'version': 'current',
'action': 'add'
})
for package_name in node['policies'][str(
packages_policy['_id'])]['pkgs_to_remove']:
package_list.append({
'name': package_name,
'version': 'current',
'action': 'remove'
})
if 'pkgs_to_remove' in node['policies'][str(
packages_policy['_id'])]:
del node['policies'][str(
packages_policy['_id'])]['pkgs_to_remove']
node['policies'][str(
packages_policy['_id'])]['package_list'] = package_list
# Update policies
self.db.nodes.update({'_id': node['_id']},
{'$set': {
'policies': node['policies']
}})
logger.debug('Joined list: %s' % (str(node['policies'][str(
packages_policy['_id'])]['package_list'])))
if node['type'] == 'ou':
ous.append(node)
elif node['type'] == 'group':
groups.append(node)
elif node['type'] == 'computer':
computers.append(node)
logger.info('%s nodes were updated!' % (len(updated_nodes)))
# We only go through the highest level OUs.
# Therefore, we eliminate intermediate OUs and
# then do not recalculate the policies
# for the same node several times.
for ou in ous:
parents = [
ObjectId(oid) for oid in ou['path'].split(',') if oid != 'root'
]
if any(o['_id'] in parents for o in ous):
ous.remove(ou)
# Users that are not under an OU or GROUP that have the migrated policy
for computer in computers:
parents = [
ObjectId(oid) for oid in computer['path'].split(',')
if oid != 'root'
]
if any(o['_id'] in parents for o in ous):
computers.remove(computer)
elif any(computer['_id'] in group['members'] for group in groups):
computers.remove(computer)
# Recalculating policies for OU
for ou in ous:
old = deepcopy(ou)
del old["policies"][str(packages_policy['_id'])]
object_changed(self.auth_user, 'ou', ou, old)
# Recalculating policies for GROUP
for group in groups:
old = deepcopy(group)
del old["policies"][str(packages_policy['_id'])]
object_changed(self.auth_user, 'group', group, old)
# Recalculating policies for USER
for computer in computers:
old = deepcopy(computer)
del old["policies"][str(packages_policy['_id'])]
object_changed(self.auth_user, 'computer', computer, old)
# Recalculate policies for Chef nodes
for node_id in ChefNode.list():
node = ChefNode(node_id, self.api)
logger.info('Checking node: %s' % (node_id))
if ("gecos_ws_mgmt" in node.attributes) and (
"software_mgmt" in node.attributes["gecos_ws_mgmt"]) and (
"package_res"
in node.attributes["gecos_ws_mgmt"]["software_mgmt"]):
if "pkgs_to_remove" in node.attributes["gecos_ws_mgmt"][
"software_mgmt"]["package_res"]:
logger.debug(
"Chef node %s contains a pkgs_to_remove value!" %
(node_id))
# Remove pkgs_to_remove from mongodb node
logger.info("Remove 'pkgs_to_remove' attribute!")
try:
del node.attributes["gecos_ws_mgmt"]["software_mgmt"][
"package_res"]["pkgs_to_remove"]
node.save()
except:
logger.warn(
"Problem deleting pkgs_to_remove value from node: %s"
% (node_id))
if not "package_list" in node.attributes["gecos_ws_mgmt"][
"software_mgmt"]["package_res"]:
logger.error(
"Chef node %s doesn\'t contains a package_list value!"
% (node_id))
continue
package_list = node.attributes["gecos_ws_mgmt"][
"software_mgmt"]["package_res"]["package_list"]
for element in package_list:
if not 'action' in element:
logger.debug(
'Chef node: %s doesn\'t have an action value in package_res! (package_list:%s)'
% (node_id, str(package_list)))
break
# Final check
bad_nodes = Search('node',
"pkgs_to_remove:*",
rows=1000,
start=0,
api=self.api)
for node in bad_nodes:
logger.warn('Detected bad node: %s' % (node.object.name))
gecos_node = self.db.nodes.find_one(
{"node_chef_id": node.object.name})
if gecos_node is None:
logger.warn('Can\'t find node in MongoDB for: %s' %
(node.object.name))
else:
logger.warn(
'For an unknown reason a computer called %s wasn\'t updated!'
% (gecos_node['name']))
logger.info('END ;)')
def test_rows(self):
s = Search('node', rows=1)
self.assertEqual(len(s), 1)
self.assertGreaterEqual(s.total, 3)
def test_object(self):
s = Search('node', 'name:test_1')
self.assertEqual(len(s), 1)
node = s[0].object
self.assertEqual(node.name, 'test_1')
self.assertEqual(node.run_list, ['role[test_1]'])
# Save the parsed arguments
results = parser.parse_args()
results.chef_client_pem = results.chef_client_pem.replace(
'~', os.getenv("HOME"))
# Load chef and razor apis
with ChefAPI(results.chef_url, results.chef_client_pem, results.chef_client):
razor = razor_api(results.razor_ip)
server = []
dashboard = []
agents = []
# Make sure environment exists, if not create one
env = "%s-%s-opencenter" % (results.name, results.os)
if not Search("environment").query("name:%s" % env):
print "Making environment: %s " % env
Environment.create(env)
# Gather the servers in the environment into their roles
nodes = Search('node').query("name:qa-%s-pool* AND chef_environment:%s" %
(results.os, env))
for n in nodes:
node = Node(n['name'])
#print "Found: %s " % node.name
#print node.attributes['in_use']
if node.attributes['in_use'] == "server":
server.append(node.name)
elif node.attributes['in_use'] == "dashboard":
dashboard.append(node.name)
elif node.attributes['in_use'] == "agent":
def getNodesByRole(role):
return Search("node", "roles:%s" % (role))
def test_search_all(self):
s = Search('node')
self.assertGreaterEqual(len(s), 3)
self.assertIn('test_1', s)
self.assertIn('test_2', s)
self.assertIn('test_3', s)
assert isValidAndNonEmptyString(environ.get('AWS_ACCESS_KEY_ID')), "AWS_ACCESS_KEY_ID must be set"
assert isValidAndNonEmptyString(environ.get('AWS_SECRET_ACCESS_KEY')), "AWS_SECRET_ACCESS_KEY must be set"
api = autoconfigure()
print("connected {0} chef url: ".format(api.url))
session = boto3.Session(region_name=args.aws_region)
ec2 = session.resource('ec2')
print("connected to ec2 with key {0}".format(environ.get('AWS_ACCESS_KEY_ID')))
ec2_alive_instances = ec2.instances.filter(Filters=[{'Name': 'instance-state-name', 'Values': ['stopping','pending','initializing','running','stopped']}])
ec2_alive_instance_ids = map(lambda ec2_instance: ec2_instance.instance_id, ec2_alive_instances)
chef_nodes = Search('node', "chef_environment:{0}".format(args.chef_environment))
chef_node_names = map(lambda chef_node: chef_node.object.name, chef_nodes)
def is_ec2_node_defined_in_region(ec2_instance_id_to_be_found, ec2_instance_ids):
is_ec2_node_defined = False
for ec2_instance_id in ec2_instance_ids:
if(ec2_instance_id == ec2_instance_id_to_be_found):
is_ec2_node_defined = True
return is_ec2_node_defined
if (len(chef_nodes) == 0):
print("No chef nodes registered... doing nothing")
else:
print("{0} chef nodes found".format(len(chef_nodes)))
for chef_node_row in chef_nodes:
chef_node_name = chef_node_row.object.name
