def delete_chef_node(self, nodeId):
try:
pychef.Node(nodeId, api=self.chef_api).delete(api=self.chef_api)
pychef.Client(nodeId, api=self.chef_api).delete(api=self.chef_api)
return True
except Exception as e:
return e
def check_cluster_health(self, callback_url):
import chef
cluster_name = self.config_manager.get_clustername()
nodes = chef.Search(
'node',
'tags:rally_node AND name:*%s' % cluster_name,
api=self.chef_api
)
if not nodes:
err_msg = "Cannot find Rally node!"
logging.info(err_msg)
raise Exception(err_msg)
rally_node_name = None
for node in nodes:
rally_node_name = node.object.name
break
rally_node = chef.Node(rally_node_name, api=self.chef_api)
rally_node_ip = rally_node['ipaddress']
command = self.config_manager.get_adapter_health_check_cmd()
command = command.replace('$cluster_name', cluster_name)
option = '--url %s --clustername %s' % (callback_url, cluster_name)
command = ' '.join((command, option))
username, pwd = self.config_manager.get_server_credentials()
util.execute_cli_by_ssh(
command, rally_node_ip, username=username,
password=pwd, nowait=True
)
def handle(event, _context):
"""Lambda Handler"""
log_event(event)
# If you're using a self signed certificate change
# the ssl_verify argument to False
with chef.ChefAPI(CHEF_SERVER_URL,
get_pem(),
USERNAME,
ssl_verify=VERIFY_SSL):
instance_id = get_instance_id(event)
try:
search = chef.Search('node', 'ec2_instance_id:' + instance_id)
except ChefServerNotFoundError as err:
LOGGER.error(err)
return False
if len(search) != 0:
for instance in search:
node = chef.Node(instance.object.name)
client = chef.Client(instance.object.name)
try:
node.delete()
LOGGER.info('===Node Delete: SUCCESS===')
client.delete()
LOGGER.info('===Client Delete: SUCCESS===')
return True
except ChefServerNotFoundError as err:
LOGGER.error(err)
return False
else:
LOGGER.info(
'=Instance does not appear to be Chef Server managed.=')
return True
def _clean_host_attributes(self, config, target_system):
"""clean node attributes about target system."""
import chef
node_name = self._get_node_name(config['fullname'])
client_name = self._get_client_name(config['fullname'])
node = chef.Node(node_name, api=self.api_)
roles_per_target_system = node.get('roles_per_target_system', {})
if target_system in roles_per_target_system:
del roles_per_target_system[target_system]
node['roles_per_target_system'] = roles_per_target_system
if not roles_per_target_system:
try:
node.delete()
client = chef.Client(client_name, api=self.api_)
client.delete()
logging.debug('delete %s for host %s ', target_system,
node_name)
except Exception as error:
logging.debug('failed to delete %s for host %s: %s',
target_system, node_name, error)
else:
node.run_list = []
for _, roles in roles_per_target_system.items():
for role in roles:
node.run_list.append('role[%s]' % role)
node.save()
logging.debug('node %s is updated for %s', node_name,
target_system)
def get_data(self):
nodes = []
chefapi = chef.ChefAPI(chef_url, chef_key, chef_usr)
for name in chef.Node.list(api=chefapi):
node = chef.Node(name)
nodes.append(initChefNode(node))
nodes.sort(key=op.attrgetter('id'))
return nodes
def process_virtual(chefapi,
data,
config_file,
internal_dns_suffix,
force_update=False):
assign_dict = {}
# find all internal nodes
internal_nodes = chef.Search('node',
'role:%s' % INTERNAL_NODE_ROLE,
api=chefapi)
nodes = itertools.cycle(list(internal_nodes))
# walk all virtual machines, slotting them to internal_nodes
for machine in data['virtual']:
node_to_assign = nodes.next()
node_name = node_to_assign['name']
logger.info("Assigning virtual machine '%s' to node %s'" %
(machine['name'], node_name))
if not assign_dict.has_key(node_name):
assign_dict[node_name] = []
assign_dict[node_name].append(machine)
# create chef entry for the virtual machine itself
virtual_name = machine['name'] + '.' + internal_dns_suffix
chef_node_obj = chef.Node(name=virtual_name, api=chefapi)
chef_node_obj.run_list = machine['chef']['run_list']
chef_node_obj.save()
for node in assign_dict:
attrib_dict = {
'internal-node': {
'virtual_machines': assign_dict[node]
}
}
chef_node_obj = chef.Node(name=node, api=chefapi)
chef_node_obj.normal.update(attrib_dict)
chef_node_obj.save()
def delete_chef_nodes(cert_path, user_name, nodes):
for node in nodes:
if "packer" not in node:
print(node)
with chef.ChefAPI('https://cbs-chef.hl.aws/organizations/cbs',
cert_path,
user_name,
ssl_verify=False):
n = chef.Node(node)
n.delete()
def perform(self, cm):
node = chef.Node(self.node_id, api=cm.chefapi)
dotted_attr = \
attribute if isinstance(self.attribute, str) \
else '.'.join(self.attribute) if hasattr(self.attribute, '__iter__') \
else util.f_raise(TypeError(
'Unknown attribute specification: {0}'.format(self.attribute)))
try:
return node.attributes.get_dotted(dotted_attr)
except KeyError:
raise KeyError('Unresolved node attribute: %s', dotted_attr)
def perform(self, cm):
node_id = self.instance_data['node_id']
log.debug("[CM] Querying node state for %r", node_id)
node = chef.Node(node_id, api=cm.chefapi)
if self.chef_exists(cm, node):
if 'ohai_time' in node.attributes:
return status.READY
else:
return status.PENDING
else:
return status.UNKNOWN
def get_data(self):
try:
node_id = self.kwargs['id']
chefapi = chef.ChefAPI(chef_url, chef_key, chef_usr)
node = chef.Node(node_id, api=chefapi)
except Exception:
#redirect = self.get_redirect_url()
exceptions.handle(self.request,
_('Unable to retrieve node details.'),
redirect=redirect)
return node
def configure_couchbase(self):
with self.chef_api:
self.chef_node = chef.Node(self.name)
username = self.chef_node.attributes.get_dotted(
'couchbase.server.username')
password = self.chef_node.attributes.get_dotted(
'couchbase.server.password')
memory_quota = self.chef_node.attributes.get_dotted(
'couchbase.server.'
'memory_quota_mb')
port = self.chef_node.attributes.get_dotted('couchbase.server.'
'port')
uri = 'http://{hostname}:{port}/pools/default/buckets'.format(
hostname=self.hostname, port=port)
payload = {
'authType': 'sasl',
'bucketType': 'memcached',
'name': self.bucket_name,
'ramQuotaMB': memory_quota
}
auth = (username, password)
while True:
try:
r = requests.post(uri, auth=auth, data=payload)
break
except requests.exceptions.ConnectionError:
self.log.error('Failed to connect to Couchbase')
self.log.debug('Re-trying in 10 seconds')
time.sleep(10)
if r.status_code == 202:
self.log.info('Created memcached bucket {bucket}'.format(
bucket=self.bucket_name))
else:
self.log.error('Failed to create memcached bucket '
'{bucket}'.format(bucket=self.bucket_name))
self.log.error('Recieved status code {code} for '
'Couchbase'.format(code=r.status_code))
self.log.error(
'Recieved response {response}'.format(response=r.json()))
def test_add_roles(self):
import chef
test_node = chef.Node('testnode', api=self.chef_test_api)
test_node.run_list.append('role[test_role_a]')
test_node.save()
self._register(test_node)
input_roles = ['test_role_1', 'test_role_2', 'test_role_a']
self.test_chef.add_roles(test_node, input_roles)
expected_roles = [('role[%s]' % role) for role in input_roles]
self.assertSetEqual(set(expected_roles), set(test_node.run_list))
def perform(self, cm):
log.info("[CM] Registering node: %r", self.resolved_node_definition['name'])
self.ensure_role(cm)
n = chef.Node(cm.node_name(self.resolved_node_definition),
api=cm.chefapi)
n.chef_environment = self.resolved_node_definition['infra_id']
n.run_list = self.assemble_run_list(cm)
self.assemble_attributes(n.normal)
n.save()
log.debug("[CM] Done")
def delete_node(hostname):
with chef.ChefAPI(CHEF_SERVER_URL, PEM_KEY, USERNAME):
try:
# remove client from chef server
client = chef.Client(hostname)
client.delete()
logger.info('Successfully deleted client %s', hostname)
# remove node from chef server
node = chef.Node(hostname)
node.delete()
logger.info('Successfully deleted node %s', hostname)
except ChefServerNotFoundError as err:
logger.error(err)
def lambda_handler(event, context):
print("Event received: " + json.dumps(event))
for record in event['Records']:
if 'Sns' in record:
message = json.loads(record['Sns']['Message'])
if message['Event'] == 'autoscaling:EC2_INSTANCE_TERMINATE':
instance_id = message['EC2InstanceId']
print("instance_id = " + instance_id)
try:
chef_api = chef.autoconfigure()
except:
raise Exception('Could not configure Chef!')
try:
rows = chef.Search('node',
'ec2_instance_id:' + instance_id)
except:
raise Exception(
'Could not search for nodes with ec2_instance_id: ' +
instance_id)
for row in rows:
try:
n = chef.Node(row['name'])
except:
raise Exception('Could not fetch node object for ' +
row['name'])
print("node: " + str(n))
try:
c = chef.Client(row['name'])
except:
raise Exception('Could not fetch client object for ' +
row['name'])
print("client: " + str(c))
try:
n.delete()
except:
raise Exception('Could not delete node ' + str(n))
try:
c.delete()
except:
raise Exception('Could not delete client ' + str(n))
else:
raise Exception('Could not process SNS message')
def verify_recipes_in_runlist(step, serv_as, recipes):
recipes = recipes.split(',')
server = getattr(world, serv_as)
host_name = world.get_hostname_by_server_format(server)
chef_api = chef.autoconfigure()
run_list = chef.Node(host_name, api=chef_api).run_list
if len(run_list) != len(recipes):
raise AssertionError(
'Count of recipes in node is another that must be: "%s" != "%s" "%s"'
% (len(run_list), len(recipes), run_list))
if not all(recipe in ','.join(run_list) for recipe in recipes):
raise AssertionError('Recipe "%s" not exist in run list!' % run_list)
def node_collect_data(api, max_records):
log_prefix = 'chef - node:'
if VERBOSE:
print log_prefix + ' collecting data'
all_nodes = list(chef.Node.list())
total_nodes = len(all_nodes)
if VERBOSE:
print '%s found %d nodes' % (log_prefix, total_nodes)
count = 1
node_data = {}
for node in all_nodes:
if VERBOSE:
print '%s loading node %d/%d' % (log_prefix, count, total_nodes)
curr_node_data = chef.Node(node).attributes.to_dict()
if keyable_attribute(curr_node_data, 'fqdn'):
if curr_node_data['fqdn'] in node_data:
print log_prefix + ' duplicate fqdn - overwriting'
node_data[curr_node_data['fqdn']] = curr_node_data
elif keyable_attribute(curr_node_data, 'hostname'):
if curr_node_data['hostname'] in node_data:
print log_prefix + ' duplicate hostname - overwriting'
node_data[curr_node_data['hostname']] = curr_node_data
elif keyable_attribute(curr_node_data, 'ipaddress'):
if curr_node_data['ipaddress'] in node_data:
print log_prefix + ' duplicate ipaddress - overwriting'
node_data[curr_node_data['ipaddress']] = curr_node_data
else:
node_data[node] = curr_node_data
#if 'ipaddress' in curr_node_data:
# node_data[curr_node_data['ipaddress']] = curr_node_data
#else:
# if 'no_ipaddress' not in node_data:
# node_data['no_ipaddress'] = {}
# node_data['no_ipaddress'][node] = curr_node_data
if max_records and count == max_records:
break
count += 1
return node_data
def perform(self, cm):
"""
Delete a node and all associated data from the chef server.
.. todo:: Delete the generated client too.
"""
node_id = cm.node_name(self.instance_data)
log.debug("[CM] Dropping node %r", node_id)
try:
chef.Node(node_id, api=cm.chefapi).delete()
log.debug("[CM] Done")
except Exception as ex:
log.exception('Error dropping node:')
log.info('[CM] Dropping node failed - ignoring.')
def handle(event, _context):
"""Lambda Handler"""
log_event(event)
#Suppress InsecureRequestWarning: Unverified HTTPS request is being made in Python2.6
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
# If you're using a self signed certificate change
# the ssl_verify argument to False
with chef.ChefAPI(CHEF_SERVER_URL,
get_pem(),
USERNAME,
ssl_verify=VERIFY_SSL):
instance_id = get_instance_id(event)
try:
search = chef.Search('node', 'ec2_instance_id:' + instance_id)
except ChefServerNotFoundError as err:
LOGGER.error(err)
return False
if len(search) != 0:
for instance in search:
node = chef.Node(instance.object.name)
client = chef.Client(instance.object.name)
try:
LOGGER.info('About to delete the node named - ' +
node.name)
LOGGER.info('About to delete the client named - ' +
client.name)
if not DEBUG:
node.delete()
LOGGER.info('===Node Delete: SUCCESS===')
client.delete()
LOGGER.info('===Client Delete: SUCCESS===')
else:
LOGGER.info('Would have deleted the node named - ' +
node.name +
' here, but we are in DEBUG mode')
LOGGER.info('Would have deleted the client named - ' +
client.name +
' here, but we are in DEBUG mode')
return True
except ChefServerNotFoundError as err:
LOGGER.error(err)
return False
else:
LOGGER.info(
'=Instance does not appear to be Chef Server managed.=')
return True
def check_node_exists_on_chef_server(step, serv_as, negation):
server = getattr(world, serv_as)
try:
host_name = getattr(world, '%s_chef_node_name' % server.id)
except AttributeError:
host_name = world.get_hostname_by_server_format(server)
setattr(world, '%s_chef_node_name' % server.id, host_name)
LOG.debug('Chef node name: %s' % host_name)
chef_api = chef.autoconfigure()
LOG.debug('Chef api instance: %s' % chef_api)
if not isinstance(chef_api, chef.api.ChefAPI):
raise AssertionError("Can't initialize ChefAPI instance.")
node = chef.Node(host_name, api=chef_api)
node_exists = node.exists
assert not node_exists if negation else node_exists, 'Node %s not in valid state on Chef server' % host_name
def _update_host_attributes(self, config, target_system):
"""chef manage node attributes about target system."""
import chef
node_name = self._get_node_name(config['fullname'])
node = chef.Node(node_name, api=self.api_)
node['cluster'] = self._cluster_databag_name(config['clusterid'])
roles_per_target_system = node.get('roles_per_target_system', {})
roles_per_target_system[target_system] = config['roles']
node['roles_per_target_system'] = roles_per_target_system
node.run_list = []
for _, roles in roles_per_target_system.items():
for role in roles:
node.run_list.append('role[%s]' % role)
node.save()
logging.debug('update %s for host %s', target_system, node_name)
def bake(self):
if self.CHEF_RUNLIST:
chef_path = os.path.expanduser(self.chef_path)
self.chef_api = chef.autoconfigure(chef_path)
with self.chef_api:
try:
node = chef.Node(self.name)
node.delete()
self.log.info('Removed previous chef node "{node}"'.format(
node=self.name))
except chef.exceptions.ChefServerNotFoundError:
pass
except chef.exceptions.ChefServerError as e:
# This gets thrown on chef12 when the client/node does not
# exist.
if str(e) == 'Forbidden':
pass
else:
self.log.error(str(e))
raise e
except Exception as e:
self.log.error(str(e))
raise e
try:
client = chef.Client(self.name)
client = client.delete()
self.log.info(
'Removed previous chef client "{client}"'.format(
client=self.name))
except chef.exceptions.ChefServerNotFoundError:
pass
except chef.exceptions.ChefServerError as e:
# This gets thrown on chef12 when the client/node does not
# exist.
if str(e) == 'Forbidden':
pass
else:
self.log.error(str(e))
raise e
except Exception as e:
self.log.error(str(e))
raise e
def get_convergence_status(self, target_nodes=None, show_all=True):
nodes = {}
now = datetime.datetime.now()
if target_nodes is None:
target_nodes = self.get_chef_nodes()
for node in target_nodes:
try:
last_converge = datetime.datetime.fromtimestamp(
pychef.Node(node,
api=self.chef_api).attributes['ohai_time'])
except:
continue
if not show_all:
if (now - last_converge).seconds > 720:
nodes[node] = last_converge
else:
nodes[node] = last_converge
return nodes
def clean(self):
try:
for node_name in chef.Node.list(api=self.api):
node = chef.Node(node_name, api=self.api)
node.delete()
logging.info('delete node %s', node_name)
except Exception as error:
logging.error('failed to delete some nodes')
logging.exception(error)
try:
for client_name in chef.Client.list(api=self.api):
if client_name in ['chef-webui', 'chef-validator']:
continue
client = chef.Client(client_name, api=self.api)
client.delete()
logging.info('delete client %s', client_name)
except Exception as error:
logging.error('failed to delete some clients')
logging.exception(error)
try:
for env_name in chef.Environment.list(api=self.api):
if env_name == '_default':
continue
env = chef.Environment(env_name, api=self.api)
env.delete()
logging.info('delete env %s', env_name)
except Exception as error:
logging.error('failed to delete some envs')
logging.exception(error)
try:
for databag_name in chef.DataBag.list(api=self.api):
databag = chef.DataBag(databag_name, api=self.api)
for item_name, item in databag.items():
item.delete()
logging.info(
'delete item %s from databag %s',
item_name, databag_name
)
except Exception as error:
logging.error('failed to delete some databag items')
logging.exception(error)
def create_node(chefapi, node_data, internal_dns_suffix, force_update=False):
nodes_list = chef.Node.list(api=chefapi)
#Checking, chef server has this node and will we update it if necessary
node_name = node_data['name'] + '.' + internal_dns_suffix
# calculate actual node name
if node_name in nodes_list and not force_update:
logger.warning("Found node with name '%s'. Skipping it" % node_name)
else:
if role_validator(node_data['chef']['run_list']):
if not node_name in nodes_list:
logger.info("Node with name '%s' not found on chef server" %
node_name)
logger.info("Addind node '%s' to chef server" % node_name)
new_node = chef.Node.create(name=node_name, api=chefapi)
else:
logger.info(
"Node with name '%s' found on chef server and will be updated"
% node_name)
logger.info("Updating node '%s' on chef server" % node_name)
new_node = chef.Node(name=node_name, api=chefapi)
if 'chef_environment' in node_data['chef']:
new_node.chef_environment = node_data['chef'][
'chef_environment']
elif not 'chef_environment' in node_data['chef'] and force_update:
new_node.chef_environment = "_default"
if 'default' in node_data['chef']:
new_node.default = node_data['chef']['default']
elif not 'default' in node_data['chef'] and force_update:
new_node.default = {}
if 'override' in node_data['chef']:
new_node.override = node_data['chef']['override']
elif not 'override' in node_data['chef'] and force_update:
new_node.override = {}
if 'normal' in node_data['chef']:
new_node.normal = node_data['chef']['normal']
elif not 'normal' in node_data['chef'] and force_update:
new_node.normal = {}
new_node.normal['reboot-handler']['post_boot_runlist'] = node_data[
'chef']['run_list']
new_node.save()
def install(request, curr_cluster, server_name, role):
if not commons.logged_in(request):
return HttpResponseRedirect('/cluster/'+curr_cluster+'/')
server = Server.objects.get(name=server_name)
if server.ip == '':
server_data = get_server(request, server.server_id)
for ip in server_data['addresses']['private']:
if ip['version'] == 4:
server.ip = ip['addr']
server.save()
os.system('echo '+server.ip+' '+server.name+'.novalocal >> /etc/hosts')
break
api = get_chef_api()
node = chef.Node(server_name+".novalocal", api)
if "role["+role+"]" not in node.run_list:
if role == 'vm-master':
if server.cluster_in.master != None:
return HttpResponseRedirect('/cluster/'+curr_cluster+'/')
server.cluster_in.master = server
server.cluster_in.save()
update_databag(server, '1')
if 'role[vm-slave]' in node.run_list:
node.run_list.remove("role[vm-slave]")
node.run_list.append("role[vm-master]")
server.is_master = True
server.is_slave = False
node.save()
server.save()
trigger_chef_client(server_name)
elif role == 'vm-slave':
update_databag(server, '0')
if 'role[vm-master]' in node.run_list:
node.run_list.remove("role[vm-master]")
node.run_list.append("role[vm-slave]")
server.is_master = False
server.is_slave = True
node.save()
server.save()
trigger_chef_client(server_name)
trigger_chef_client(server.cluster_in.master.name)
return HttpResponseRedirect('/cluster/'+curr_cluster+'/')
def get(self, host):
"""Add <host> to device class in POD for monitoring"""
deviceclass = self.get_argument('deviceclass',
default='/Server/SSH/Linux/Collector')
#productionState wants an int Prod=1000,PreProd=500,Maint=300,Test=0,Decom=-1
prodstate = str(self.get_argument('prodstate', default="500"))
chefapi = chef.autoconfigure()
master = chef.Node(chefapi.client)
vpcid = master['vpc_id']
privip = master['private_ips'][0]
stackname = master['cluster']['name']
collectors = master['zenoss']['collector-passwords']
zaasdomain = master['zaas']['domain']
for c in collectors:
if c['collector'] == host:
#devicename = c['ip']
devicename = host + '.' + stackname + '.internal.' + zaasdomain
pod = chef.Search('node', 'hostname:POD* AND vpc_id:' + vpcid)
podip = pod[0]['automatic']['ipaddress']
password = pod[0]['normal']['zenoss']['password']
user = '******'
url = 'https://' + podip + '/zport/dmd/device_router'
data = '{"action":"DeviceRouter","method":"addDevice","data":[{"deviceName":"' + devicename + '","deviceClass":"' + deviceclass + '","collector":"localhost","title":"' + host + '","productionState":"' + prodstate + '"}],"tid":1}'
userpass = user + ":" + password
auth = "Basic " + userpass.encode("base64").rstrip()
headers = {
"Authorization": auth,
"Content-Type": "application/json; charset=utf-8"
}
request = tornado.httpclient.HTTPRequest(url=url,
method='POST',
body=data,
headers=headers,
validate_cert=False)
client = tornado.httpclient.AsyncHTTPClient()
response = yield tornado.gen.Task(client.fetch, request)
self.finish(response.body)
def get_create_node(self, node_name, env_name=None):
"""Get chef node
Gets the node if existing, otherwise create one and set its
environment.
:param str node_name: The name for this node.
:param str env_name: The environment name for this node.
"""
import chef
if not self.chef_api:
logging.info("Cannot find ChefAPI object!")
raise Exception("Cannot find ChefAPI object!")
node = chef.Node(node_name, api=self.chef_api)
if node not in chef.Node.list(api=self.chef_api):
if env_name:
node.chef_environment = env_name
node.save()
return node
def converge(env, node = None):
if env == BOOTSTRAP_ENV:
flask.abort(400)
if len(processes(env, node, only_executing = True)) > 0:
return ujson.encode({ 'status': 'converging' })
if node is not None:
nodes = { node: chef.Node(node, api = api), }
else:
nodes = { row.object.name: row.object for row in chef.Search('node', 'chef_environment:' + env, api = api) }
get_command = lambda n: ['ssh', '-o', 'StrictHostKeyChecking=no', n['ipaddress'], 'sudo', 'chef-client']
return _run(
nodes,
get_command,
env = env,
progress_status = 'converging',
)
def install_software_in_node(self, node_name, product_name):
api = chef.autoconfigure('knife.rb')
node = chef.Node(node_name + '.novalocal')
print node.exists
if not node.exists:
print 'node ' + node_name + '.novalocal'
print 'no node in chef server'
time.sleep(20)
print node.exists
if not node.exists:
print 'node ' + node_name + '.novalocal'
print 'no node in chef server'
time.sleep(20)
print node.exists
if not node.exists:
print 'Node not registed in the chef sever'
sys.exit()
node.run_list.append(product_name)
node.save()
print node
return node
