def node_search(query, environment=None, tries=10):
"""Performs a node search query on the chef server.
:param query: search query to request
:type query: string
:param environment: Environment the query should be
:type environment: monster.environments.chef.environment.Environment
:rtype: Iterator (chef.Node)
"""
if environment:
api = environment.local_api
else:
try:
api = chef.autoconfigure(
active.config['secrets']['chef']['knife'])
logger.debug("Using knife.rb found at {}".format(
active.config['secrets']['chef']['knife']))
except KeyError:
api = chef.autoconfigure()
search = None
while not search and tries > 0:
search = chef.Search("node", api=api).query(query)
time.sleep(10)
tries -= 1
return (n.object for n in search)
def local_api():
try:
knife_override = active.config['secrets']['chef']['knife']
logger.debug("Using knife.rb found at {}".format(knife_override))
return chef.autoconfigure(knife_override)
except KeyError:
return chef.autoconfigure()
def __init__(self):
self.parser = self._create_parser()
self.chef_server_url = ""
self.client_key = ""
self.client_name = ""
self.cache_max_age = 3600
self.cache_path = os.path.join(os.path.expanduser('~'),
'.ansible-chef.cache')
self.read_settings()
self.api = chef.autoconfigure()
if self.chef_server_url and self.client_key and self.client_name:
print("Using chef ini values", file=sys.stderr)
self.api = chef.ChefAPI(self.chef_server_url, self.client_key,
self.client_name)
else:
pemfile = os.environ.get('CHEF_PEMFILE')
username = os.environ.get('CHEF_USER')
chef_server_url = os.environ.get('CHEF_SERVER_URL')
if not self.api:
if pemfile is None or username is None or chef_server_url is None:
print(
"Set CHEF_PEMFILE, CHEF_USER and CHEF_SERVER_URL environment vars. They might be located under ~/.chef/knife_local.rb or ~/.chef/knife.rb"
)
exit(0)
self.api = chef.ChefAPI(chef_server_url, pemfile, username)
if not self.api:
print("Could not find chef configuration", file=sys.stderr)
sys.exit(1)
def __init__(self, **kwargs):
super(Installer, self).__init__(**kwargs)
import chef
self.installer_url_ = setting.CHEF_INSTALLER_URL
self.global_databag_name_ = setting.CHEF_GLOBAL_DATABAG_NAME
self.api_ = chef.autoconfigure()
logging.debug('%s instance created', self)
def __init__(self,
SIGNALFX_API_TOKEN,
CONFIG_FILE=DEFAULT_CONFIG_FILE,
LOG_FILE=DEFAULT_LOG_FILE,
SIGNALFX_REST_API=DEFAULT_SIGNALFX_REST_API,
PICKLE_FILE=DEFAULT_PICKLE_FILE,
SLEEP_DURATION=DEFAULT_SLEEP_DURATION,
LOG_HANDLER=DEFAULT_LOG_HANDLER):
self.api = autoconfigure()
self.SIGNALFX_API_TOKEN = SIGNALFX_API_TOKEN
self.CONFIG_FILE = CONFIG_FILE
self.LOG_FILE = LOG_FILE
self.SIGNALFX_REST_API = SIGNALFX_REST_API + '/v1/dimension'
self.PICKLE_FILE = PICKLE_FILE
self.SLEEP_DURATION = SLEEP_DURATION
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.INFO)
if LOG_HANDLER == 'logfile':
self.handler = logging.FileHandler(DEFAULT_LOG_FILE)
else:
self.handler = logging.StreamHandler(sys.stdout)
self.handler.setLevel(logging.INFO)
self.formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
self.handler.setFormatter(self.formatter)
self.logger.addHandler(self.handler)
self.property_name_pattern = re.compile('^[a-zA-Z_][a-zA-Z0-9_-]*$')
self.config = []
self.organization = ''
self.nodes_metadata = []
def get_databag_attributes(self, atlas_set_header, databag_set_name):
"""
Returns all items of a databag given the header and databag name in the atlas config file.
"""
data_bag_attr = {}
base_path = self.ah_obj.get_atlas_config_data("chef_module",
'chef-base-path')
api = chef.autoconfigure(base_path)
chef_databags = self.ah_obj.get_atlas_config_data(
atlas_set_header, databag_set_name)[1]
for databag in chef_databags.keys():
data_bag = DataBag(databag, api)
key_list = {}
items = chef_databags[databag]['items'].keys()
for item_index in items:
key_list = chef_databags[databag]['items'][item_index]['keys']
chef_databag_item = DataBagItem(databag, item_index, api)
for item_keys, item_values in chef_databag_item.iteritems():
if item_keys in key_list:
data_bag_attr[item_keys] = item_values
elif type(item_values) == dict:
data_bag_attr[item_keys] = {}
for key in key_list:
attr_values = self.ah_obj.get_nested_attribute_values(
item_values, key)
data_bag_attr[item_keys][key] = attr_values
return data_bag_attr
def watch(self):
api = chef.autoconfigure()
for server in self.queue.get_messages():
instance = self.get_metadata(server)
node = chef.Node(server)
node.run_list.append('role[%s]' % instance)
node.save()
def roles (self, mess, args):
""" Search and return roles """
api = chef.autoconfigure()
roles = ''
for row in chef.Search('role', 'name:*' + args + '*'):
roles += "%s\n" % row.object.name
return(roles)
def __init__(self, inventory):
""" constructor """
# Apparently nothing needed here
self.inv = inventory
#print("Running Autoconfigure . . .")
self.api = chef.autoconfigure()
def set_hosts(stage, role):
api = autoconfigure()
query = 'roles:{project_name}-python-{role}-node AND ' \
'chef_environment:{stage}'.format(project_name=PROJECT_NAME,
stage=stage, role=role)
env.stage = stage
env.hosts = [row.object.attributes.get_dotted('fqdn') for
row in Search('node', query, api=api)]
def from_chef_environment(cls, environment):
"""
Rebuilds a Deployment given a chef environment
:param environment: name of environment
:type environment: string
:rtype: Chef
"""
local_api = autoconfigure()
env = ChefEnvironment(environment, api=local_api)
override = env.override_attributes
default = env.default_attributes
chef_auth = override.get('remote_chef', None)
remote_api = None
if chef_auth and chef_auth["key"]:
remote_api = ChefServer._remote_chef_api(chef_auth)
renv = ChefEnvironment(environment, api=remote_api)
override = renv.override_attributes
default = renv.default_attributes
environment = MonsterChefEnvironment(env.name,
local_api,
description=env.name,
default=default,
override=override,
remote_api=remote_api)
name = env.name
deployment_args = override.get('deployment', {})
features = deployment_args.get('features', {})
os_name = deployment_args.get('os_name', None)
branch = deployment_args.get('branch', None)
status = deployment_args.get('status', "provisioning")
product = deployment_args.get('product', None)
provisioner_name = deployment_args.get('provisioner', "razor")
provisioner = get_provisioner(provisioner_name)
deployment = cls.deployment_config(features,
name,
os_name,
branch,
environment,
provisioner,
status,
product=product)
nodes = deployment_args.get('nodes', [])
for node in (ChefNode(n, local_api) for n in nodes):
if not node.exists:
util.logger.error("Non existant chef node:{0}".format(
node.name))
continue
cnode = MonsterChefNode.from_chef_node(node,
deployment_args['os_name'],
product, environment,
deployment, provisioner,
deployment_args['branch'])
deployment.nodes.append(cnode)
return deployment
def __init__(self, **kwargs):
import chef
super(Installer, self).__init__(**kwargs)
self.installer_url_ = setting.CHEF_INSTALLER_URL
self.global_databag_name_ = setting.CHEF_GLOBAL_DATABAG_NAME
self.api_ = chef.autoconfigure()
self.tmp_databags_ = {}
self.tmp_databag_items_ = {}
logging.debug('%s instance created', self)
def __init__(self):
import chef
try:
self.api = chef.autoconfigure()
except Exception as error:
logging.error('%s failed to autoconfigure', self.NAME)
logging.exception(error)
raise error
logging.debug('%s instance created %s', self.NAME, self.api)
def fromfile(cls, name, template_name, branch, provisioner, template_file,
template_path=None):
"""
Returns a new deployment given a deployment template at path
:param name: name for the deployment
:type name: string
:param name: name of template to use
:type name: string
:param branch: branch of the RCBOPS chef cookbook repo to use
:type branch:: string
:param provisioner: provisioner to use for nodes
:type provisioner: Provisioner
:param path: path to template
:type path: string
:rtype: Chef
"""
local_api = autoconfigure()
if ChefEnvironment(name, api=local_api).exists:
# Use previous dry build if exists
util.logger.info("Using previous deployment:{0}".format(name))
return cls.from_chef_environment(name)
if not template_path:
path = os.path.join(os.path.dirname(__file__),
os.pardir, os.pardir,
'templates/{0}.yaml'.format(
template_file))
else:
path = template_path
template = Config(path)[template_name]
environment = MonsterChefEnvironment(name, local_api, description=name)
os_name = template['os']
product = template['product']
deployment = cls.deployment_config(template['features'], name, os_name,
branch, environment, provisioner,
product=product)
# provision nodes
chef_nodes = provisioner.provision(template, deployment)
for node in chef_nodes:
cnode = MonsterChefNode.from_chef_node(node, os_name, product,
environment, deployment,
provisioner, branch)
provisioner.post_provision(cnode)
deployment.nodes.append(cnode)
# add features
for node, features in zip(deployment.nodes, template['nodes']):
node.add_features(features)
return deployment
def __init__(self):
import chef
try:
self.api = chef.autoconfigure()
except Exception as error:
logging.error('%s failed to autoconfigure', self.NAME)
logging.exception(error)
raise error
logging.debug('%s instance created %s', self.NAME, self.api)
def assert_recipes_in_runlist(server: Server, recipes: tp.List[str]):
host_name = lib_server.get_hostname_by_server_format(server)
chef_api = chef.autoconfigure()
run_list = chef.Node(host_name, api=chef_api).run_list
if len(run_list) != len(recipes):
raise AssertionError('Number of recipes in the node is wrong: "%s" != "%s". Actual runlist in Chef Node: "%s"' %
(len(run_list), len(recipes), run_list))
if not all(recipe in ','.join(run_list) for recipe in recipes):
raise AssertionError('Recipe "%s" is not in the runlist!' % run_list)
def __init__(self, base_path, keypair_pem, user,
run_list, boot_sudo, boot_version, log):
self.base_path = os.path.abspath(base_path)
self.keypair_pem = keypair_pem
self.user = user
self.boot_run_list = run_list
self.boot_sudo = boot_sudo
self.boot_version = boot_version
self.log = log
self.chef = chef.autoconfigure(self.base_path)
def check_required():
"""Check if the required roles are available."""
chef_api = autoconfigure()
chef_roles = Role.list(api=chef_api)
for role in get_required().values():
if check(role, chef_roles):
print(green('Role %s available.') % role)
else:
print(red('Role %s NOT available.') % role)
def __init__(self, base_path, keypair_pem, user, run_list, boot_sudo,
boot_version, log):
self.base_path = os.path.abspath(base_path)
self.keypair_pem = keypair_pem
self.user = user
self.boot_run_list = run_list
self.boot_sudo = boot_sudo
self.boot_version = boot_version
self.log = log
self.chef = chef.autoconfigure(self.base_path)
def node_search(cls, query, environment=None, tries=10):
api = autoconfigure()
if environment:
api = environment.local_api
search = None
while not search and tries > 0:
search = Search("node", api=api).query(query)
sleep(10)
tries = tries - 1
return (n.object for n in search)
def hosts(env=None, name=None):
api = autoconfigure()
if name:
nodes = (node for node in Node.list() if name in Node(node).name)
else:
nodes = (node for node in Node.list() if Node(node).chef_environment == env)
file = open("hosts", "w")
file.write("[hosts]\n")
map(lambda n: file.write("{0}\n".format(Node(n)['ipaddress'])), nodes)
def get_graphite_hosts(environment):
"""Retrieve the hostname for graphite.
:param str environment:
The environment in which to locate the graphite server
"""
api = autoconfigure()
query = 'roles:graphite-server AND ' \
'chef_environment:{0}'.format(environment)
result = Search('node', query, api=api)
return [row.object.attributes.get_dotted('fqdn') for row in result]
def mls_config():
"""Get the MLS configuration object for the current node."""
chef_api = autoconfigure()
if '127.0.0.1:2222' in api.env.hosts:
query = 'hostname:%s' % api.env.hostname
else:
query = 'ipaddress:%s' % api.env.host
for node in Search('node', query, api=chef_api):
try:
return node.object['mls']
except KeyError:
return {}
def check_chef_data(self, data_type):
"""Checks if chef cookbooks/roles/databags are correct.
:param data_type : chef data type
should be one of ['CookBook','DataBag','Role']
:type data_type : string
:param github_url : Latest chef data on stackforge/compass-adapters
:type github_url : string
"""
print "Checking Chef %s......" % (data_type.lower().strip() + 's'),
try:
import chef
except Exception:
self._set_status(0,
"[%s]Error: pychef is not installed." % self.NAME)
return
api = chef.autoconfigure()
if data_type == 'CookBook':
local = set(
[name for name in os.listdir('/var/chef/cookbooks') if name])
try:
server = set(api['/cookbooks'].keys())
except Exception:
self._set_status(
0, "[%s]Error: pychef fails to get cookbooks" % self.NAME)
return
elif data_type == 'Role':
local = set([
name[:-3] for name in os.listdir('/var/chef/roles')
if name.endswith('.rb')
])
try:
server = set(api['/roles'].keys())
except Exception:
self._set_status(
0, "[%s]Error: pychef fails to get roles" % self.NAME)
return
else:
local = set(
[name for name in os.listdir('/var/chef/databags') if name])
try:
server = set(api['/data'].keys())
except Exception:
self._set_status(
0, "[%s]Error: pychef fails to get databags" % self.NAME)
return
diff = server - local
if len(diff) > 0:
self._set_status(0, "[%s]Error: %s diff: %s" % (self.NAME, diff))
def from_chef_environment(cls, environment):
"""
Rebuilds a Deployment given a chef environment
:param environment: name of environment
:type environment: string
:rtype: Chef
"""
local_api = autoconfigure()
environ = ChefEnvironment(environment, api=local_api)
if not environ.exists:
util.logger.error("The specified environment, {0}, does not"
"exist.".format(environment))
exit(1)
override = environ.override_attributes
default = environ.default_attributes
chef_auth = override.get('remote_chef', None)
remote_api = None
if chef_auth and chef_auth["key"]:
remote_api = ChefServer._remote_chef_api(chef_auth)
renv = ChefEnvironment(environment, api=remote_api)
override = renv.override_attributes
default = renv.default_attributes
environment = MonsterChefEnvironment(
environ.name, local_api, description=environ.name,
default=default, override=override, remote_api=remote_api)
name = environ.name
deployment_args = override.get('deployment', {})
features = deployment_args.get('features', {})
os_name = deployment_args.get('os_name', None)
branch = deployment_args.get('branch', None)
status = deployment_args.get('status', "provisioning")
product = deployment_args.get('product', None)
provisioner_name = deployment_args.get('provisioner', "razor2")
provisioner = get_provisioner(provisioner_name)
deployment = cls.deployment_config(features, name, os_name, branch,
environment, provisioner, status,
product=product)
nodes = deployment_args.get('nodes', [])
for node in (ChefNode(n, local_api) for n in nodes):
if not node.exists:
util.logger.error("Non existant chef node:{0}".
format(node.name))
continue
cnode = MonsterChefNode.from_chef_node(node, product, environment,
deployment, provisioner,
deployment_args['branch'])
deployment.nodes.append(cnode)
return deployment
def alert(self, config, msgtype, msg):
"""
Processes an event into an alert
"""
for r in boto.ec2.regions():
if r.name == config['region']:
region = r
break
ec2 = boto.connect_ec2(aws_access_key_id=config['AWS_ACCESS'], aws_secret_access_key=config['AWS_SECRET'], region=region)
api = chef.autoconfigure()
instance_id = None
client = None
node = None
self.logger.debug('chef_handler - Checking Chef Server For node matching: %s'%msg['EC2InstanceId'] , extra=dict(program="autoscale-alert", handler="chef_handler", instance_id=msg['EC2InstanceId'], region=region.name))
chefapiobject = chef.ChefAPI.from_config_file('/etc/chef/client.rb')
nodes = chef.Search('node', 'instance_id:%s'%msg['EC2InstanceId'], api=chefapiobject)
if nodes is not None:
for n in nodes:
try:
node = chef.Node(n["name"])
except:
pass
try:
client = chef.Client(n["name"])
except:
pass
break
if node is not None:
try:
self.logger.debug('chef_handler - Found Node, Deleting.', extra=dict(program="autoscale-alert", handler="chef_handler", instance_id=msg['EC2InstanceId'], region=region.name, action="node-delete"))
node.delete()
self.logger.info('chef_handler - Deleted Node.', extra=dict(program="autoscale-alert", handler="chef_handler", instance_id=msg['EC2InstanceId'], region=region.name, action="node-delete"))
except ChefServerNotFoundError as e:
self.logger.error('chef_handler - Failure Deleting Node.', extra=dict(program="autoscale-alert", handler="chef_handler", instance_id=msg['EC2InstanceId'], region=region.name, action="node-delete"))
pass
if client is not None:
try:
self.logger.debug('chef_handler - Found Client, Deleting.', extra=dict(program="autoscale-alert", handler="chef_handler", instance_id=msg['EC2InstanceId'], region=region.name, action="client-delete"))
client.delete()
self.logger.info('chef_handler - Deleted Client.', extra=dict(program="autoscale-alert", handler="chef_handler", instance_id=msg['EC2InstanceId'], region=region.name, action="client-delete"))
except ChefServerNotFoundError as e:
self.logger.error('chef_handler - Failure Deleting Client.', extra=dict(program="autoscale-alert", handler="chef_handler", instance_id=msg['EC2InstanceId'], region=region.name, action="client-delete"))
raise
def _load_chef_databag(self, databag_name, databag_item_id):
import chef
# TODO : check file existence
chef_api = chef.autoconfigure(Config.CHEF_CONFIGURATION_PATH)
print(('Loading databag item [{:s}:{:s}]'.format(databag_name, databag_item_id)))
databag_item = chef.DataBagItem(databag_name, databag_item_id, api=chef_api)
if not databag_item.exists:
raise Exception('Databag item not found')
if 'instances' not in databag_item:
raise Exception('Empty databag item')
return databag_item['instances']
def verify_recipes_in_runlist(step, serv_as, recipes):
recipes = recipes.split(',')
server = getattr(world, serv_as)
host_name = world.get_hostname_by_server_format(server)
chef_api = chef.autoconfigure()
run_list = chef.Node(host_name, api=chef_api).run_list
if len(run_list) != len(recipes):
raise AssertionError('Count of recipes in node is another that must be: "%s" != "%s" "%s"' %
(len(run_list), len(recipes), run_list))
if not all(recipe in ','.join(run_list) for recipe in recipes):
raise AssertionError('Recipe "%s" not exist in run list!' % run_list)
def collect_data(cfg):
api = chef.autoconfigure()
# get env or node data depending on user input
if cfg['cmdline']['view'] == 'env':
view_data = env_collect_data(api, cfg['cmdline']['max_records'])
else:
view_data = node_collect_data(api, cfg['cmdline']['max_records'])
if cfg['cmdline']['dump_ds']:
print json.dumps(view_data, indent=4)
return view_data
def collect_data(cfg):
api = chef.autoconfigure()
# get env or node data depending on user input
if cfg['cmdline']['view'] == 'env':
view_data = env_collect_data(api, cfg['cmdline']['max_records'])
else:
view_data = node_collect_data(api, cfg['cmdline']['max_records'])
if cfg['cmdline']['dump_ds']:
print json.dumps(view_data, indent=4)
return view_data
def lambda_handler(event, context):
print("Event received: " + json.dumps(event))
for record in event['Records']:
if 'Sns' in record:
message = json.loads(record['Sns']['Message'])
if message['Event'] == 'autoscaling:EC2_INSTANCE_TERMINATE':
instance_id = message['EC2InstanceId']
print("instance_id = " + instance_id)
try:
chef_api = chef.autoconfigure()
except:
raise Exception('Could not configure Chef!')
try:
rows = chef.Search('node',
'ec2_instance_id:' + instance_id)
except:
raise Exception(
'Could not search for nodes with ec2_instance_id: ' +
instance_id)
for row in rows:
try:
n = chef.Node(row['name'])
except:
raise Exception('Could not fetch node object for ' +
row['name'])
print("node: " + str(n))
try:
c = chef.Client(row['name'])
except:
raise Exception('Could not fetch client object for ' +
row['name'])
print("client: " + str(c))
try:
n.delete()
except:
raise Exception('Could not delete node ' + str(n))
try:
c.delete()
except:
raise Exception('Could not delete client ' + str(n))
else:
raise Exception('Could not process SNS message')
def check_chef_data(self, data_type, github_url):
"""Checks if chef cookbooks/roles/databags are correct.
:param data_type : chef data type
should be one of ['CookBook','DataBag','Role']
:type data_type : string
:param github_url : Latest chef data on stackforge/compass-adapters
:type github_url : string
"""
print "Checking Chef %s......" % (data_type.lower().strip() + 's'),
try:
import chef
except Exception:
self._set_status(
0,
"[%s]Error: pychef is not installed." % self.NAME)
return self.get_status()
api = chef.autoconfigure()
github = set([
item['name']
for item in requests.get(github_url).json()
])
if data_type == 'CookBook':
local = set(os.listdir('/var/chef/cookbooks'))
elif data_type == 'Role':
local = set([
name for name, item in chef.Role.list(api=api).iteritems()
])
github = set([
item['name'].replace(".rb", "").replace(".json", "")
for item in requests.get(github_url).json()
])
else:
local = set([
item for item in eval(
'chef.' + data_type + '.list(api=api)'
)
])
logging.info('github %s: %s', data_type, github)
logging.info('local %s: %s', data_type, local)
diff = github - local
if len(diff) <= 0:
return (data_type, [])
else:
return (data_type, list(diff))
def list_nodes(role_list=None):
"""List all available nodes with given roles."""
if not role_list:
role_list = get_required().values()
else:
role_list = role_list.split(';')
chef_api = autoconfigure()
for role in role_list:
print('Role: %s' % role)
query = 'role:%s' % role
for row in Search('node', query, api=chef_api):
print('- %s: %s' % (row['name'], row.object['ipaddress']))
print
def next_client_number(**params):
chef_api = autoconfigure()
query = 'role:%(rolename)s AND chef_environment:%(environment)s' % params
nodes = sorted([
node.object.name for node in Search('node', query, api=chef_api)
])
if len(nodes) < 1:
return 1
last = nodes[-1]
number = last.split('-')[-1]
try:
number = int(number)
except ValueError, e:
raise e
def verify_recipes_in_runlist(step, serv_as, recipes):
recipes = recipes.split(',')
server = getattr(world, serv_as)
host_name = world.get_hostname_by_server_format(server)
chef_api = chef.autoconfigure()
run_list = chef.Node(host_name, api=chef_api).run_list
if len(run_list) != len(recipes):
raise AssertionError(
'Count of recipes in node is another that must be: "%s" != "%s" "%s"'
% (len(run_list), len(recipes), run_list))
if not all(recipe in ','.join(run_list) for recipe in recipes):
raise AssertionError('Recipe "%s" not exist in run list!' % run_list)
def bake(self):
chef_path = os.path.expanduser(self.chef_path)
self.chef_api = chef.autoconfigure(chef_path)
with self.chef_api:
try:
node = chef.Node(self.name)
node.delete()
self.log.info('Removed previous chef node "{node}"'.format(
node = self.name))
except chef.exceptions.ChefServerNotFoundError:
pass
except Exception as e:
self.log.error(str(e))
raise e
try:
client = chef.Client(self.name)
client = client.delete()
self.log.info('Removed previous chef client "{client}"'.format(
client = self.name))
except chef.exceptions.ChefServerNotFoundError:
pass
except Exception as e:
self.log.error(str(e))
raise e
node = chef.Node.create(self.name)
self.chef_node = node
self.log.info('Created new Chef Node "{node}"'.format(
node = self.name))
self.chef_node.chef_environment = self.environment
self.log.info('Set the Chef Environment to "{env}"'.format(
env = self.chef_node.chef_environment))
self.chef_node.run_list = self.CHEF_RUNLIST
self.log.info('Set Chef run list to {list}'.format(
list = self.chef_node.run_list))
self.chef_node.save()
self.log.info('Saved the Chef Node configuration')
def _get_chef_api(self, key=None, client=None):
"""Initializes chef API client."""
import chef
chef_api = None
try:
if key and client:
chef_api = chef.ChefAPI(self.installer_url, key, client)
else:
chef_api = chef.autoconfigure()
except Exception as ex:
err_msg = "Failed to instantiate chef API, error: %s" % ex.message
raise Exception(err_msg)
return chef_api
def assert_node_exists_on_chef_server(server: api.Server, exist: bool = True):
# NOTE: migrated
hostname = lib_server.get_hostname_by_server_format(server)
LOG.debug(f'Chef node name: {hostname}')
chef_api = chef.autoconfigure()
if not isinstance(chef_api, chef.api.ChefAPI):
raise AssertionError("Can't initialize ChefAPI instance.")
node = chef.Node(hostname, api=chef_api)
if node.exists != exist:
raise AssertionError(f'Server {server.id} with hostname {hostname} in state {node.exists} '
f'on Chef server but need: {exist}')
def __init__(self, settings):
installer_url = settings.get(self.CHEFSERVER_URL, None)
key_dir = settings.get(self.KEY_DIR, None)
client = settings.get(self.CLIENT, None)
try:
if installer_url and key_dir and client:
self.api = chef.ChefAPI(installer_url, key_dir, client)
else:
self.api = chef.autoconfigure()
logging.info('chef client created %s(%s, %s)', installer_url,
key_dir, client)
except Exception as error:
logging.error('failed to create chef client %s(%s, %s)',
installer_url, key_dir, client)
logging.exception(error)
def main():
"""
Do some stuff.
"""
arguments = parse_arguments().parse_args()
name_filters = ["*"] if not arguments.name_filters else arguments.name_filters
api = chef.autoconfigure()
response = nodes_status(
api, name_filters, healthy_only=arguments.healthy_only, healthy_minutes=arguments.healthy_minutes
)
if arguments.output_format == "knife_status":
print_status(response, arguments.sort, healthy_minutes=arguments.healthy_minutes)
elif arguments.output_format == "fqdn_list":
print_fqdn_list(response, arguments.sort)
elif arguments.output_format == "version_list":
print_version_list(response, arguments.sort)
def fqdn_for_node(self, node):
self.node = node
api = autoconfigure()
n = Node(node)
if n:
try:
self.fqdn = n.attributes['ec2']['public_hostname']
except KeyError:
if 'fqdn' in n:
self.fqdn = n['fqdn']
else:
return None
else:
return None
return self.fqdn
def chef_start_api():
"""Horribly patches chef api to skip SSL cert validation"""
from chef.api import ChefRequest
api = chef.autoconfigure()
def _request(method, url, data, headers):
request = ChefRequest(url, data, headers, method=method)
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
return urlopen(request, context=ctx).read()
api._request = _request
return api
def fqdn_for_node(self,node):
self.node = node
api = autoconfigure()
n = Node(node)
if n:
try:
self.fqdn = n.attributes['ec2']['public_hostname']
except KeyError:
if 'fqdn' in n:
self.fqdn = n['fqdn']
else:
return None
else:
return None
return self.fqdn
def lambda_handler(event, context):
'''Handler for Lambda'''
print("Event received: " + json.dumps(event))
for record in event['Records']:
if 'Sns' in record:
message = json.loads(record['Sns']['Message'])
if message['Event'] == 'autoscaling:EC2_INSTANCE_TERMINATE':
instance_id = message['EC2InstanceId']
print("instance_id = " + instance_id)
try:
chef_api = chef.autoconfigure()
except:
raise Exception('Could not configure Chef!')
try:
rows = chef.Search('node', 'ec2_instance_id:' + instance_id)
except:
raise Exception('Could not search for nodes with ec2_instance_id: ' + instance_id)
for row in rows:
try:
n = chef.Node(row['name'])
except:
raise Exception('Could not fetch node object for ' + row['name'])
print("chef-node: " + str(n))
try:
c = chef.Client(row['name'])
except:
raise Exception('Could not fetch client object for ' + row['name'])
print("chef-client: " + str(c))
try:
n.delete()
except:
raise Exception('Could not delete chef-node ' + str(n))
try:
c.delete()
except:
raise Exception('Could not delete chef-client ' + str(n))
else:
raise Exception('Could not process SNS message')
def check_node_exists_on_chef_server(step, serv_as, negation):
server = getattr(world, serv_as)
try:
host_name = getattr(world, '%s_chef_node_name' % server.id)
except AttributeError:
host_name = world.get_hostname_by_server_format(server)
setattr(world, '%s_chef_node_name' % server.id, host_name)
LOG.debug('Chef node name: %s' % host_name)
chef_api = chef.autoconfigure()
LOG.debug('Chef api instance: %s' % chef_api)
if not isinstance(chef_api, chef.api.ChefAPI):
raise AssertionError("Can't initialize ChefAPI instance.")
node = chef.Node(host_name, api=chef_api)
node_exists = node.exists
assert not node_exists if negation else node_exists, 'Node %s not in valid state on Chef server' % host_name
def bake(self):
if self.CHEF_RUNLIST:
chef_path = os.path.expanduser(self.chef_path)
self.chef_api = chef.autoconfigure(chef_path)
with self.chef_api:
try:
node = chef.Node(self.name)
node.delete()
self.log.info('Removed previous chef node "{node}"'.format(
node=self.name))
except chef.exceptions.ChefServerNotFoundError:
pass
except chef.exceptions.ChefServerError as e:
# This gets thrown on chef12 when the client/node does not
# exist.
if str(e) == 'Forbidden':
pass
else:
self.log.error(str(e))
raise e
except Exception as e:
self.log.error(str(e))
raise e
try:
client = chef.Client(self.name)
client = client.delete()
self.log.info(
'Removed previous chef client "{client}"'.format(
client=self.name))
except chef.exceptions.ChefServerNotFoundError:
pass
except chef.exceptions.ChefServerError as e:
# This gets thrown on chef12 when the client/node does not
# exist.
if str(e) == 'Forbidden':
pass
else:
self.log.error(str(e))
raise e
except Exception as e:
self.log.error(str(e))
raise e
def update_data_bag_item(node):
data_bag_item_name = 'deployment_{}'.format(
node.CHEF_ATTRIBUTES['zuun']['deployment'])
data_bag_item_node_data = {
'version': node.mongodb_version,
'conf': base64.b64encode(generate_mongo_conf(node))
}
try:
search_key = node.expanded_replica_set
except AttributeError:
search_key = None
if search_key is None: search_key = node.CHEF_MONGODB_TYPE
data_bag_item = {'replica-sets': {}}
with chef.autoconfigure(node.chef_path):
data_bag = chef.DataBag('zuun')
data_bag_item_exists = False
if data_bag_item_name in data_bag.keys():
node.log.info(
'Data bag item {} already exists; updating (but not overwriting) if required'
.format(data_bag_item_name))
data_bag_item = chef.DataBagItem(data_bag, data_bag_item_name)
data_bag_item_exists = True
else:
node.log.info('Data bag item {} does not exist; creating'.format(
data_bag_item_name))
if node.CHEF_MONGODB_TYPE == 'data':
source = data_bag_item['replica-sets']
source[search_key] = {'data': data_bag_item_node_data}
elif node.CHEF_MONGODB_TYPE == 'config' and search_key != 'config':
source = data_bag_item['replica-sets']
source[search_key] = {'config': data_bag_item_node_data}
else:
source = data_bag_item
source[search_key] = data_bag_item_node_data
if data_bag_item_exists:
data_bag_item.save()
else:
chef.DataBagItem.create('zuun', data_bag_item_name,
**data_bag_item)
def node_search(cls, query, environment=None, tries=10):
"""
Performs a node search query on the chef server
:param query: search query to request
:type query: string
:param environment: Environment the query should be
:type environment: ChefEnvironment
:rtype: Iterator (chef.Node)
"""
api = autoconfigure()
if environment:
api = environment.local_api
search = None
while not search and tries > 0:
search = Search("node", api=api).query(query)
sleep(10)
tries = tries - 1
return (n.object for n in search)
def run(self, terms, variables=None, **kwargs):
# Ensure pychef has been loaded
if not HAS_CHEF:
raise AnsibleError('PyChef needed for lookup plugin, try `pip install pychef`')
for term in terms:
self.parse_kv_args(parse_kv(term))
api_object = chef.autoconfigure()
if not isinstance(api_object, chef.api.ChefAPI):
raise AnsibleError('Unable to connect to Chef Server API.')
data_bag_object = chef.DataBag(self.name)
data_bag_item = data_bag_object[self.item]
return [dict(data_bag_item)]
def run(command):
"""
Handle execution of `command`, a `ChefCommand` subclass
"""
flags, arguments = None, None
try:
flags, arguments = getopt.getopt(sys.argv[1:], command.getopt_options)
except getopt.GetoptError:
print command.getUsage(sys.argv[0])
sys.exit(0)
if len(arguments) != command.argument_count:
print 'error: Wrong argument count'
print command.getUsage(sys.argv[0])
sys.exit(1)
# configures pychef with ~/.chef/knife.rb
if os.environ.has_key("KNIFE_CONFIG"):
api = chef.ChefAPI.from_config_file(os.environ["KNIFE_CONFIG"])
else:
api = chef.autoconfigure()
# let the command handler do its magic on the command line arguments
for i, argument in enumerate(arguments):
argument = command.parseAndSearch(argument, api)
arguments[i] = argument
# ChefCommand.parseAndSearch() sets argument.node if it matched against a chef node
# display the nodes we're about to connect to
for argument in arguments:
if getattr(argument, 'node', None):
node = argument.node
# print some information on the chosen node and connect to it
print 'Node %s:' % (node['name'],)
print '\trun_list: %s' % (', '.join(node['run_list']),)
print '\troles: %s' % (', '.join(node['automatic']['roles']),)
print '\trecipes: %s' % (', '.join(node['automatic']['recipes']),)
if 'ec2' in node['automatic']:
print '\tEC2 instance type %s' % (node['automatic']['ec2']['instance_type'],)
print
extra_args = [item for sublist in flags for item in sublist]
command.invoke(extra_args, arguments)
def main():
"""
Do some stuff.
"""
arguments = parse_arguments().parse_args()
name_filters = ['*'] if not arguments.name_filters else \
arguments.name_filters
api = chef.autoconfigure()
response = nodes_status(api,
name_filters,
healthy_only=arguments.healthy_only,
healthy_minutes=arguments.healthy_minutes)
if arguments.output_format == 'knife_status':
print_status(response,
arguments.sort,
healthy_minutes=arguments.healthy_minutes)
elif arguments.output_format == 'fqdn_list':
print_fqdn_list(response, arguments.sort)
elif arguments.output_format == 'version_list':
print_version_list(response, arguments.sort)
def get(self, host):
"""Add <host> to device class in POD for monitoring"""
deviceclass = self.get_argument('deviceclass',
default='/Server/SSH/Linux/Collector')
#productionState wants an int Prod=1000,PreProd=500,Maint=300,Test=0,Decom=-1
prodstate = str(self.get_argument('prodstate', default="500"))
chefapi = chef.autoconfigure()
master = chef.Node(chefapi.client)
vpcid = master['vpc_id']
privip = master['private_ips'][0]
stackname = master['cluster']['name']
collectors = master['zenoss']['collector-passwords']
zaasdomain = master['zaas']['domain']
for c in collectors:
if c['collector'] == host:
#devicename = c['ip']
devicename = host + '.' + stackname + '.internal.' + zaasdomain
pod = chef.Search('node', 'hostname:POD* AND vpc_id:' + vpcid)
podip = pod[0]['automatic']['ipaddress']
password = pod[0]['normal']['zenoss']['password']
user = '******'
url = 'https://' + podip + '/zport/dmd/device_router'
data = '{"action":"DeviceRouter","method":"addDevice","data":[{"deviceName":"' + devicename + '","deviceClass":"' + deviceclass + '","collector":"localhost","title":"' + host + '","productionState":"' + prodstate + '"}],"tid":1}'
userpass = user + ":" + password
auth = "Basic " + userpass.encode("base64").rstrip()
headers = {
"Authorization": auth,
"Content-Type": "application/json; charset=utf-8"
}
request = tornado.httpclient.HTTPRequest(url=url,
method='POST',
body=data,
headers=headers,
validate_cert=False)
client = tornado.httpclient.AsyncHTTPClient()
response = yield tornado.gen.Task(client.fetch, request)
self.finish(response.body)
def __init__(self):
self.parser = self._create_parser()
self.chef_server_url = ""
self.client_key = ""
self.client_name = ""
self.cache_max_age = 3600
self.cache_path = os.path.join(os.path.expanduser('~'),
'.ansible-chef.cache')
self.read_settings()
if self.chef_server_url and self.client_key and self.client_name:
print("Using chef ini values", file=sys.stderr)
self.api = chef.ChefAPI(self.chef_server_url, self.client_key,
self.client_name)
else:
print("Trying chef autoconfiguration", file=sys.stderr)
self.api = chef.autoconfigure()
if not self.api:
print("Could not find chef configuration", file=sys.stderr)
sys.exit(1)
def install_software_in_node(self, node_name, product_name):
api = chef.autoconfigure('knife.rb')
node = chef.Node(node_name + '.novalocal')
print node.exists
if not node.exists:
print 'node ' + node_name + '.novalocal'
print 'no node in chef server'
time.sleep(20)
print node.exists
if not node.exists:
print 'node ' + node_name + '.novalocal'
print 'no node in chef server'
time.sleep(20)
print node.exists
if not node.exists:
print 'Node not registed in the chef sever'
sys.exit()
node.run_list.append(product_name)
node.save()
print node
return node
def fromfile(cls, name, branch, config, path=None):
if not path:
path = os.path.join(os.path.dirname(__file__),
os.pardir,
'deployment_templates/default.yaml')
template = Config(path)[name]
local_api = autoconfigure()
chef = Chef(name, local_api, description=name)
razor = razor_api(config['razor']['ip'])
os_name = template['os']
product = template['product']
name = template['name']
deployment = cls.deployment_config(template['os-features'],
template['rpcs-features'], name,
os_name, branch, config, chef,
razor)
for features in template['nodes']:
node = cls.node_config(deployment, features, os_name, product,
chef, razor, branch)
deployment.nodes.append(node)
return deployment
from chef import autoconfigure, Node
import sys
# Gets rid of warnings about ignoring the ssl cert
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
ID = sys.argv[1]
api = autoconfigure()
headers = {}
path = '/pushy/jobs/' + ID
print "JOB STATUS:\n" + api.api_request('GET', path, headers)
def get_chef_api():
api = chef.autoconfigure()
# chef.ChefAPI
return api
LOG_FILE = None,
LOG_FORMAT = '%(asctime)s %(name)s\t%(levelname)s\t%(message)s',
LOG_LEVEL = logging.INFO,
ENABLE_BOOTSTRAP = True,
)
BOOTSTRAP_ENV = '__chefdash_bootstrap__'
if distutils.spawn.find_executable('knife'):
bootstrap_enabled = True
else:
bootstrap_enabled = False
login_manager = flask.ext.login.LoginManager(app)
api = chef.autoconfigure()
def handler(environ, start_response):
handled = False
path = environ['PATH_INFO']
if path.startswith('/feed/'):
ws = environ.get('wsgi.websocket')
if ws:
handle_websocket(ws, path[6:])
handled = True
if not handled:
return app(environ, start_response)
websockets = {}
