def test_search_item(self): self.assertIn('test_1', Search.list()) q = Search('test_1') self.assertIn('item_1', q) self.assertIn('item_2', q) self.assertEqual(q['item_1']['raw_data']['test_attr'], 1) item = q['item_1'].object self.assertIsInstance(item, DataBagItem) self.assertEqual(item['test_attr'], 1)
def test_slice(self): s = Search('node')[1:2] self.assertEqual(len(s), 1) self.assertGreaterEqual(s.total, 3) s2 = s[1:2] self.assertEqual(len(s2), 1) self.assertGreaterEqual(s2.total, 3) self.assertNotEqual(s[0]['name'], s2[0]['name']) s3 = Search('node')[2:3] self.assertEqual(len(s3), 1) self.assertGreaterEqual(s3.total, 3) self.assertEqual(s2[0]['name'], s3[0]['name'])
def opencenter_endpoint(chef, name='test', os='ubuntu'): validate_environment(chef, name=name, os=os) env = env_format % (name, os) query = "in_use:\"server\" AND chef_environment:%s" % env server = next(Node(node['name']) for node in Search('node').query(query)) ep_url = "https://%s:8443" % server['ipaddress'] return OpenCenterEndpoint(ep_url, user="******", password="******")
def chef_roledefs(api=None, hostname_attr='fqdn'): """Build a Fabric roledef dictionary from a Chef server. Example: from fabric.api import env, run, roles from chef.fabric import chef_roledefs env.roledefs = chef_roledefs() @roles('web_app') def mytask(): run('uptime') hostname_attr is the attribute in the chef node that holds the real hostname. to refer to a nested attribute, separate the levels with '.'. for example 'ec2.public_hostname' """ api = api or ChefAPI.get_global() or autoconfigure() if not api: raise ChefError('Unable to load Chef API configuration') roledefs = {} for row in Search('role', api=api): name = row['name'] roledefs[name] = Roledef(name, api, hostname_attr) return roledefs
def test_search_item(self): self.assertIn('test_1', Search.list(self.api)) q = Search('test_1', self.api) self.assertIn('item_1', q) self.assertIn('item_2', q) self.assertEqual(q['item_1']['raw_data']['test_attr'], 1) item = q['item_1'].object self.assertIsInstance(item, DataBagItem) self.assertEqual(item['test_attr'], 1)
def node_search(cls, query, environment=None, tries=10): api = autoconfigure() if environment: api = environment.local_api search = None while not search and tries > 0: search = Search("node", api=api).query(query) sleep(10) tries = tries - 1 return (n.object for n in search)
def handle(event, _context): """Lambda Handler""" log_event(event) node_name = None node_ip = None # Remove from one of the chef servers for URL in CHEF_SERVER_URLS: with ChefAPI(URL, CHEF_PEM, CHEF_USERNAME): instance_id = get_instance_id(event) try: search = Search('node', 'ec2_instance_id:' + instance_id) except ChefServerNotFoundError as err: LOGGER.error(err) return False if len(search) != 0: for instance in search: node_name = instance.object.name node = Node(node_name) node_ip = node['ipaddress'] client = Client(node_name) try: node.delete() client.delete() LOGGER.info( '=====SUCCESSFULLY REMOVED INSTANCE FROM CHEF SERVER===== {}' .format(URL)) break except ChefServerNotFoundError as err: LOGGER.error(err) return False else: LOGGER.info( '===Instance does not appear to be Chef Server managed.=== {}' .format(URL)) # Remove from Spacewalk spacewalk_cleanup(node_ip) # Remove from DNS dns_cleanup(node_name) # Remove from AD active_directory_cleanup(node_name) # Remove fom Solarwinds solarwinds_cleanup(node_ip, node_name) # Remove from Chef Automate chef_automate_cleanup(node_name)
def get_chef_hosts(self): def get_public_ip(chef_node): if 'cloud' in chef_node.get('automatic', {}): return chef_node.get('automatic', {}).get('cloud', {}).get('public_ipv4') else: return chef_node.get('automatic', {}).get('ipaddress') for i in xrange(5): try: search_result = Search('node', rows=10000, api=self.api) if search_result: return {get_public_ip(node): node for node in search_result if get_public_ip(node) and IP(get_public_ip(node)).iptype() != 'PRIVATE'} except ChefServerError: time.sleep(5)
def load_known_ips(self): self.logger.debug("Loading public IP list from chef") nodes = list(Search('node', '*:*', rows=2000, api=self.chef_api)) cloud_ips = [ node.get("automatic", {}).get("cloud", {}).get("public_ips", []) for node in nodes ] phy_ifaces = sum([ node["automatic"].get("network", {}).get( "interfaces", {}).values() for node in nodes ], []) phy_ips = [i.get("addresses", {}).keys() for i in phy_ifaces] self.logger.debug("Loading public IP list from AWS") aws_machines = self.edda_client.soft_clean().query( "/api/v2/view/instances;_expand") aws_ips = [m.get("publicIpAddress") for m in aws_machines] return set(sum(cloud_ips, []) + sum(phy_ips, []) + aws_ips) #flatten
def node_search(cls, query, environment=None, tries=10): """ Performs a node search query on the chef server :param query: search query to request :type query: string :param environment: Environment the query should be :type environment: ChefEnvironment :rtype: Iterator (chef.Node) """ api = autoconfigure() if environment: api = environment.local_api search = None while not search and tries > 0: search = Search("node", api=api).query(query) sleep(10) tries = tries - 1 return (n.object for n in search)
def do_run(self): since = self.edda_client._since or 0 chef_hosts = { row['automatic']['cloud']['public_ipv4']: row['name'] for row in Search('node', 'ec2:*', rows=1000, api=self.api) if 'cloud' in row.get('automatic', {}) and # only store EC2 instances 'public_ipv4' in row['automatic'][ 'cloud'] # which have public IP address } # print chef_hosts for machine in self.edda_client.clean().query( "/api/v2/view/instances;_expand"): launchTime = int(machine.get("launchTime", 0)) # convert list of tags to a more readable dict tags = { tag['key']: tag['value'] for tag in machine.get('tags', []) if 'key' in tag and 'value' in tag } if machine['publicIpAddress'] not in chef_hosts and launchTime >= since and \ tags.get('service_name', None) not in self.excluded_instances and \ machine['instanceId'] not in self.status['first_seen']: # found a non-chef managed host which has not been seen before and which is not excluded self.status['first_seen'][machine['instanceId']] = launchTime yield { "plugin_name": self.plugin_name, "id": "%s (%s / %s)" % (machine['instanceId'], machine['publicIpAddress'], machine['privateIpAddress']), "details": [{ 'tags': tags, 'keyName': machine.get('keyName', None), 'securityGroups': machine.get('securityGroups', []) }] }
def get_chef_hosts(self): for i in xrange(5): try: search_result = Search('node', 'ec2:*', rows=2000, api=self.api) if search_result: return { row['automatic']['cloud']['public_ipv4']: row['name'] for row in search_result if 'cloud' in row.get('automatic', {}) and # only store EC2 instances 'public_ipv4' in row['automatic'][ 'cloud'] # which have public IP address } except ChefServerError: time.sleep(5) return None
def test_list(self): searches = Search.list() self.assertIn('node', searches) self.assertIn('role', searches)
def fqdn_for_instance_id(instance_id): query = "ec2_instance_id:%s" % instance_id return Search("node", query)[0]['automatic']['fqdn']
def __call__(self): for row in Search('node', 'roles:' + self.name, api=self.api): yield row.object.attributes.get_dotted(self.hostname_attr)
def instance_id_for_fqdn(fqdn): query = "fqdn:%s" % fqdn nodes = Search("node", query) return nodes[0]["automatic"]["ec2"]["instance_id"]
def test_start(self): s = Search('node', start=1) self.assertEqual(len(s), s.total - 1) self.assertGreaterEqual(s.total, 3)
def test_search_call(self): s = Search('node')('role:test_1') self.assertGreaterEqual(len(s), 2) self.assertIn('test_1', s) self.assertNotIn('test_2', s) self.assertIn('test_3', s)
def test_search_sort_desc(self): s = Search('node', sort='test_sort_key desc') self.assertGreater(s.index('test_1'), s.index('test_3'))
def test_search_sort_asc(self): s = Search('node', sort='test_sort_key asc') self.assertLess(s.index('test_1'), s.index('test_3'))
def validate_environment(chef, name='test', os='ubuntu'): env = env_format % (name, os) if not Search("environment").query("name:%s" % env): print "environment %s not found" % env sys.exit(1)
def command(self): # Initialization self.api = _get_chef_api(self.settings.get('chef.url'), toChefUsername(self.options.chef_username), self.options.chef_pem, False, self.settings.get('chef.version')) self.auth_user = self.db.adminusers.find_one( {'username': self.options.chef_username}) if self.auth_user is None: logger.error('The administrator user must exist in MongoDB') return self.db = self.pyramid.db ous = [] groups = [] computers = [] # Get packages policy ID packages_policy = self.db.policies.find_one({"slug": "package_res"}) if packages_policy is None: logger.error('Can\'t detect "package_res" policy!') return if 'pkgs_to_remove' in packages_policy["schema"]["properties"]: logger.error( "The 'package_res' policy in the system is deprecated, please update to new package policy!" ) return logger.info('Packages policy ID: %s' % (str(packages_policy['_id']))) # Get all nodes with old package policy data logger.info('Getting all nodes with old package policy data...') path_to_find = "policies.%s.pkgs_to_remove" % (str( packages_policy['_id'])) old_policy_nodes = self.db.nodes.find({ '$query': { path_to_find: { '$exists': True } }, '$orderby': { "path": 1 } }) updated_nodes = [] for node in old_policy_nodes: logger.info('Updating node %s ...' % (str(node['_id']))) updated_nodes.append(str(node['_id'])) logger.debug('Packages to add: %s' % (str(node['policies'][str( packages_policy['_id'])]['package_list']))) logger.debug('Packages to remove: %s' % (str(node['policies'][str( packages_policy['_id'])]['pkgs_to_remove']))) # Join the lists package_list = [] for package_name in node['policies'][str( packages_policy['_id'])]['package_list']: package_list.append({ 'name': package_name, 'version': 'current', 'action': 'add' }) for package_name in node['policies'][str( packages_policy['_id'])]['pkgs_to_remove']: package_list.append({ 'name': package_name, 'version': 'current', 'action': 'remove' }) if 'pkgs_to_remove' in node['policies'][str( packages_policy['_id'])]: del node['policies'][str( packages_policy['_id'])]['pkgs_to_remove'] node['policies'][str( packages_policy['_id'])]['package_list'] = package_list # Update policies self.db.nodes.update({'_id': node['_id']}, {'$set': { 'policies': node['policies'] }}) logger.debug('Joined list: %s' % (str(node['policies'][str( packages_policy['_id'])]['package_list']))) if node['type'] == 'ou': ous.append(node) elif node['type'] == 'group': groups.append(node) elif node['type'] == 'computer': computers.append(node) logger.info('%s nodes were updated!' % (len(updated_nodes))) # We only go through the highest level OUs. # Therefore, we eliminate intermediate OUs and # then do not recalculate the policies # for the same node several times. for ou in ous: parents = [ ObjectId(oid) for oid in ou['path'].split(',') if oid != 'root' ] if any(o['_id'] in parents for o in ous): ous.remove(ou) # Users that are not under an OU or GROUP that have the migrated policy for computer in computers: parents = [ ObjectId(oid) for oid in computer['path'].split(',') if oid != 'root' ] if any(o['_id'] in parents for o in ous): computers.remove(computer) elif any(computer['_id'] in group['members'] for group in groups): computers.remove(computer) # Recalculating policies for OU for ou in ous: old = deepcopy(ou) del old["policies"][str(packages_policy['_id'])] object_changed(self.auth_user, 'ou', ou, old) # Recalculating policies for GROUP for group in groups: old = deepcopy(group) del old["policies"][str(packages_policy['_id'])] object_changed(self.auth_user, 'group', group, old) # Recalculating policies for USER for computer in computers: old = deepcopy(computer) del old["policies"][str(packages_policy['_id'])] object_changed(self.auth_user, 'computer', computer, old) # Recalculate policies for Chef nodes for node_id in ChefNode.list(): node = ChefNode(node_id, self.api) logger.info('Checking node: %s' % (node_id)) if ("gecos_ws_mgmt" in node.attributes) and ( "software_mgmt" in node.attributes["gecos_ws_mgmt"]) and ( "package_res" in node.attributes["gecos_ws_mgmt"]["software_mgmt"]): if "pkgs_to_remove" in node.attributes["gecos_ws_mgmt"][ "software_mgmt"]["package_res"]: logger.debug( "Chef node %s contains a pkgs_to_remove value!" % (node_id)) # Remove pkgs_to_remove from mongodb node logger.info("Remove 'pkgs_to_remove' attribute!") try: del node.attributes["gecos_ws_mgmt"]["software_mgmt"][ "package_res"]["pkgs_to_remove"] node.save() except: logger.warn( "Problem deleting pkgs_to_remove value from node: %s" % (node_id)) if not "package_list" in node.attributes["gecos_ws_mgmt"][ "software_mgmt"]["package_res"]: logger.error( "Chef node %s doesn\'t contains a package_list value!" % (node_id)) continue package_list = node.attributes["gecos_ws_mgmt"][ "software_mgmt"]["package_res"]["package_list"] for element in package_list: if not 'action' in element: logger.debug( 'Chef node: %s doesn\'t have an action value in package_res! (package_list:%s)' % (node_id, str(package_list))) break # Final check bad_nodes = Search('node', "pkgs_to_remove:*", rows=1000, start=0, api=self.api) for node in bad_nodes: logger.warn('Detected bad node: %s' % (node.object.name)) gecos_node = self.db.nodes.find_one( {"node_chef_id": node.object.name}) if gecos_node is None: logger.warn('Can\'t find node in MongoDB for: %s' % (node.object.name)) else: logger.warn( 'For an unknown reason a computer called %s wasn\'t updated!' % (gecos_node['name'])) logger.info('END ;)')
def test_rows(self): s = Search('node', rows=1) self.assertEqual(len(s), 1) self.assertGreaterEqual(s.total, 3)
def test_object(self): s = Search('node', 'name:test_1') self.assertEqual(len(s), 1) node = s[0].object self.assertEqual(node.name, 'test_1') self.assertEqual(node.run_list, ['role[test_1]'])
# Save the parsed arguments results = parser.parse_args() results.chef_client_pem = results.chef_client_pem.replace( '~', os.getenv("HOME")) # Load chef and razor apis with ChefAPI(results.chef_url, results.chef_client_pem, results.chef_client): razor = razor_api(results.razor_ip) server = [] dashboard = [] agents = [] # Make sure environment exists, if not create one env = "%s-%s-opencenter" % (results.name, results.os) if not Search("environment").query("name:%s" % env): print "Making environment: %s " % env Environment.create(env) # Gather the servers in the environment into their roles nodes = Search('node').query("name:qa-%s-pool* AND chef_environment:%s" % (results.os, env)) for n in nodes: node = Node(n['name']) #print "Found: %s " % node.name #print node.attributes['in_use'] if node.attributes['in_use'] == "server": server.append(node.name) elif node.attributes['in_use'] == "dashboard": dashboard.append(node.name) elif node.attributes['in_use'] == "agent":
def getNodesByRole(role): return Search("node", "roles:%s" % (role))
def test_search_all(self): s = Search('node') self.assertGreaterEqual(len(s), 3) self.assertIn('test_1', s) self.assertIn('test_2', s) self.assertIn('test_3', s)
assert isValidAndNonEmptyString(environ.get('AWS_ACCESS_KEY_ID')), "AWS_ACCESS_KEY_ID must be set" assert isValidAndNonEmptyString(environ.get('AWS_SECRET_ACCESS_KEY')), "AWS_SECRET_ACCESS_KEY must be set" api = autoconfigure() print("connected {0} chef url: ".format(api.url)) session = boto3.Session(region_name=args.aws_region) ec2 = session.resource('ec2') print("connected to ec2 with key {0}".format(environ.get('AWS_ACCESS_KEY_ID'))) ec2_alive_instances = ec2.instances.filter(Filters=[{'Name': 'instance-state-name', 'Values': ['stopping','pending','initializing','running','stopped']}]) ec2_alive_instance_ids = map(lambda ec2_instance: ec2_instance.instance_id, ec2_alive_instances) chef_nodes = Search('node', "chef_environment:{0}".format(args.chef_environment)) chef_node_names = map(lambda chef_node: chef_node.object.name, chef_nodes) def is_ec2_node_defined_in_region(ec2_instance_id_to_be_found, ec2_instance_ids): is_ec2_node_defined = False for ec2_instance_id in ec2_instance_ids: if(ec2_instance_id == ec2_instance_id_to_be_found): is_ec2_node_defined = True return is_ec2_node_defined if (len(chef_nodes) == 0): print("No chef nodes registered... doing nothing") else: print("{0} chef nodes found".format(len(chef_nodes))) for chef_node_row in chef_nodes: chef_node_name = chef_node_row.object.name