def __init__(self):
self.parser = self._create_parser()
self.chef_server_url = ""
self.client_key = ""
self.client_name = ""
self.cache_max_age = 3600
self.cache_path = os.path.join(os.path.expanduser('~'),
'.ansible-chef.cache')
self.read_settings()
self.api = chef.autoconfigure()
if self.chef_server_url and self.client_key and self.client_name:
print("Using chef ini values", file=sys.stderr)
self.api = chef.ChefAPI(self.chef_server_url, self.client_key,
self.client_name)
else:
pemfile = os.environ.get('CHEF_PEMFILE')
username = os.environ.get('CHEF_USER')
chef_server_url = os.environ.get('CHEF_SERVER_URL')
if not self.api:
if pemfile is None or username is None or chef_server_url is None:
print(
"Set CHEF_PEMFILE, CHEF_USER and CHEF_SERVER_URL environment vars. They might be located under ~/.chef/knife_local.rb or ~/.chef/knife.rb"
)
exit(0)
self.api = chef.ChefAPI(chef_server_url, pemfile, username)
if not self.api:
print("Could not find chef configuration", file=sys.stderr)
sys.exit(1)
def handle(event, _context):
"""Lambda Handler"""
log_event(event)
# If you're using a self signed certificate change
# the ssl_verify argument to False
with chef.ChefAPI(CHEF_SERVER_URL,
get_pem(),
USERNAME,
ssl_verify=VERIFY_SSL):
instance_id = get_instance_id(event)
try:
search = chef.Search('node', 'ec2_instance_id:' + instance_id)
except ChefServerNotFoundError as err:
LOGGER.error(err)
return False
if len(search) != 0:
for instance in search:
node = chef.Node(instance.object.name)
client = chef.Client(instance.object.name)
try:
node.delete()
LOGGER.info('===Node Delete: SUCCESS===')
client.delete()
LOGGER.info('===Client Delete: SUCCESS===')
return True
except ChefServerNotFoundError as err:
LOGGER.error(err)
return False
else:
LOGGER.info(
'=Instance does not appear to be Chef Server managed.=')
return True
def edit_chef_environment(server_url, cert, username, environment, attributes):
"""
_edit_chef_environment_
For a given chef server, edit the override attributes in the named
environment making changes for each dotted key:value pair in attributes
Eg:
attributes = {
'application.app_version': 'x.y.z',
'application.some_url': 'http://www.google.com'
}
will result in changes:
env.override_attributes['application']['app_version'] = 'x.y.z'
env.override_attributes['application']['some_url'] = 'http://www.google.com'
:param server_url: URL of your chef server
:param cert: Client.pem file location
:param username: Chef username matching the cert
:param environment: name of chef environment eg 'production'
:param attributes: dict of dotted attribute key value pairs to change
in the env
"""
with chef.ChefAPI(server_url, cert, username):
env = chef.Environment(environment)
overrides = env.override_attributes
LOGGER.info("Editing Chef Server Environment: {} as {}".format(
environment, username))
for attr, new_value in attributes.iteritems():
LOGGER.info(" => Setting {}={}".format(attr, new_value))
set_dotted(overrides, attr, new_value)
env.save()
def active(bot, trigger):
'''Display the current active set for an environment'''
env_input = trigger.group(2)
if env_input is None:
bot.say('Please give me a chef environment')
return
url = bot.config.chef.url
key = bot.config.chef.key
username = bot.config.chef.username
api = chef.ChefAPI(url, key, username)
api.set_default()
all_envs = chef.Environment.list().names
if env_input in ('prod', 'production', 'all'):
#Select all production environments
envs = [env for env in all_envs if 'production-' in env]
if env_input == 'all':
envs.append('staging')
elif env_input in all_envs:
envs = [env_input]
elif 'production-' + env_input in all_envs:
envs = ['production-' + env_input]
else:
bot.say("{} isn't an environment I recognize".format(env_input))
return
for env_name in sorted(envs):
env = chef.Environment(env_name)
if 'active_set' in env.default_attributes['heat']:
bot.say('{}: Current active set is {}'.format(
env_name,
env.default_attributes['heat']['active_set'].upper()))
else:
bot.say('There is no active set for {}'.format(env_name))
def authenticate(self):
"""
Authenticate to chef server
@return: True on success, False otherwise
@rtype: boolean
"""
ret = False
# log message
content = {'chef-url': self.__url,
'chef-port': self.__port,
'login':"******" % self.__login,
'cmd': 'authenticate' }
tpl = self.template(name=self.__class__.__name__.upper(), content=content )
self.logRequest(msg="authenticate", details=tpl )
try:
self.__server = chef.ChefAPI(url="%s:%s" % (self.__url, self.__port),
key=self.__key,
client=self.__login,
headers={},
ssl_verify=self.__certCheck)
content = {'cmd': 'authenticated' }
tpl = self.template(name=self.__class__.__name__.upper(), content=content )
self.logResponse(msg="authenticated", details=tpl )
ret = True
except Exception as e: # log message
content = { "chef-error": "%s" % e, "cmd": "authenticate" }
tpl = self.template(name=self.__class__.__name__.upper(), content=content )
self.logResponse(msg="authenticate error", details=tpl )
return ret
def get_data(self):
nodes = []
chefapi = chef.ChefAPI(chef_url, chef_key, chef_usr)
for name in chef.Node.list(api=chefapi):
node = chef.Node(name)
nodes.append(initChefNode(node))
nodes.sort(key=op.attrgetter('id'))
return nodes
def environments(bot, trigger):
'''Display a list of chef environments'''
url = bot.config.chef.url
key = bot.config.chef.key
username = bot.config.chef.username
api = chef.ChefAPI(url, key, username)
api.set_default()
envs = chef.Environment.list().names
bot.say('Chef environments: {}'.format(', '.join(
chef.Environment.list().names)))
def delete_chef_nodes(cert_path, user_name, nodes):
for node in nodes:
if "packer" not in node:
print(node)
with chef.ChefAPI('https://cbs-chef.hl.aws/organizations/cbs',
cert_path,
user_name,
ssl_verify=False):
n = chef.Node(node)
n.delete()
def __init__(self, endpoint, auth_data, **cfg):
if not auth_data:
msg = "Authorisation information is not set for the target Chef Server ("+str(endpoint)+")!"
log.error(msg)
raise Exception(msg)
config = dict()
config['client'] = auth_data['client_name']
config['key'] = auth_data['client_key']
config['url'] = endpoint
self.chefapi = chef.ChefAPI(**config)
def get_data(self):
try:
node_id = self.kwargs['id']
chefapi = chef.ChefAPI(chef_url, chef_key, chef_usr)
node = chef.Node(node_id, api=chefapi)
except Exception:
#redirect = self.get_redirect_url()
exceptions.handle(self.request,
_('Unable to retrieve node details.'),
redirect=redirect)
return node
def delete_node(hostname):
with chef.ChefAPI(CHEF_SERVER_URL, PEM_KEY, USERNAME):
try:
# remove client from chef server
client = chef.Client(hostname)
client.delete()
logger.info('Successfully deleted client %s', hostname)
# remove node from chef server
node = chef.Node(hostname)
node.delete()
logger.info('Successfully deleted node %s', hostname)
except ChefServerNotFoundError as err:
logger.error(err)
def chef_noop(bot, trigger):
'''Display a list of noop hosts in a chef environment'''
env_name = trigger.group(2)
if env_name is None:
bot.say('Please give me a chef environment')
return
url = bot.config.chef.url
key = bot.config.chef.key
username = bot.config.chef.username
api = chef.ChefAPI(url, key, username)
api.set_default()
env = chef.Environment(env_name)
noop = env.default_attributes['base']['noop']
bot.say('The following nodes have noop set: {}'.format(', '.join(noop)))
def list_nodes(server_url,
cert,
username,
query,
attribute='name',
format_str=None):
"""
_list_nodes_
Get a list of chef nodes using a chef node seach query.
Use attribute to select the attribute from the matching nodes
and optionally provide a format string to drop the result into
to create well formatted node names
Example:
list_nodes(server_url , cert, 'server_user',
"role:some_role AND environment:prod",
attribute='automatic.ip.public'
)
Will list all nodes with some_role in the prod environment
and return the node['automatic']['ip']['public'] attribute for each node
list_nodes(server_url, cert, 'server_url',
"role:some_role AND environment:prod",
attribute='name', format_str='{}.cloudant.com')
Will extract the name attribute for each node and return that value expanded
in the format_str
:param server_url: Chef Server URL
:param cert: path to PEM cert for chef server auth
:param username: Chef Server username
:param query: chef node search query string
:param attribute: Attribute name to be extracted from node data,
can be a dot.delimited.style name
:param format_str: optional format string to apply to each result
for each node.
"""
result = []
with chef.ChefAPI(server_url, cert, username):
for row in chef.search.Search('node', query):
attr = get_dotted(row, attribute)
if format_str is not None:
attr = format_str.format(attr)
result.append(attr)
LOGGER.info("Node Found:{}".format(attr))
return result
def handle(event, _context):
"""Lambda Handler"""
log_event(event)
#Suppress InsecureRequestWarning: Unverified HTTPS request is being made in Python2.6
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
# If you're using a self signed certificate change
# the ssl_verify argument to False
with chef.ChefAPI(CHEF_SERVER_URL,
get_pem(),
USERNAME,
ssl_verify=VERIFY_SSL):
instance_id = get_instance_id(event)
try:
search = chef.Search('node', 'ec2_instance_id:' + instance_id)
except ChefServerNotFoundError as err:
LOGGER.error(err)
return False
if len(search) != 0:
for instance in search:
node = chef.Node(instance.object.name)
client = chef.Client(instance.object.name)
try:
LOGGER.info('About to delete the node named - ' +
node.name)
LOGGER.info('About to delete the client named - ' +
client.name)
if not DEBUG:
node.delete()
LOGGER.info('===Node Delete: SUCCESS===')
client.delete()
LOGGER.info('===Client Delete: SUCCESS===')
else:
LOGGER.info('Would have deleted the node named - ' +
node.name +
' here, but we are in DEBUG mode')
LOGGER.info('Would have deleted the client named - ' +
client.name +
' here, but we are in DEBUG mode')
return True
except ChefServerNotFoundError as err:
LOGGER.error(err)
return False
else:
LOGGER.info(
'=Instance does not appear to be Chef Server managed.=')
return True
def __init__(self, settings):
installer_url = settings.get(self.CHEFSERVER_URL, None)
key_dir = settings.get(self.KEY_DIR, None)
client = settings.get(self.CLIENT, None)
try:
if installer_url and key_dir and client:
self.api = chef.ChefAPI(installer_url, key_dir, client)
else:
self.api = chef.autoconfigure()
logging.info('chef client created %s(%s, %s)', installer_url,
key_dir, client)
except Exception as error:
logging.error('failed to create chef client %s(%s, %s)',
installer_url, key_dir, client)
logging.exception(error)
def _get_chef_api(self, key=None, client=None):
"""Initializes chef API client."""
import chef
chef_api = None
try:
if key and client:
chef_api = chef.ChefAPI(self.installer_url, key, client)
else:
chef_api = chef.autoconfigure()
except Exception as ex:
err_msg = "Failed to instantiate chef API, error: %s" % ex.message
raise Exception(err_msg)
return chef_api
def edit_chef_role(server_url, cert, username, rolename, attributes):
"""
_edit_chef_role_
For a given chef server, edit the override attributes in the named role
making changes for each dotted key:value pair in attributes.
"""
with chef.ChefAPI(server_url, cert, username):
role = chef.Role(rolename)
LOGGER.info("Editing Chef Server Role: {} as {}".format(
rolename, username))
overrides = role.override_attributes
for attr, new_value in attributes.iteritems():
LOGGER.info(" => Setting {}={}".format(attr, new_value))
set_dotted(overrides, attr, new_value)
role.save()
def authenticate(self):
"""
Connect to the chef server and verify communication.
"""
if self.api:
return
server = self.settings['url']
key_file = self.settings['key_file']
client = self.settings['client']
verify_ssl = self.settings.get('verify_ssl', True) in TrueValues
self.api = chef.ChefAPI(server,
key_file,
client,
ssl_verify=verify_ssl)
chef.Role.list(
api=self.api) # will raise error if authentication failure
def active(bot, trigger):
'''Display the current active set for an environment'''
env_name = trigger.group(2)
if env_name is None:
bot.say('Please give me a chef environment')
return
url = bot.config.chef.url
key = bot.config.chef.key
username = bot.config.chef.username
api = chef.ChefAPI(url, key, username)
api.set_default()
envs = chef.Environment.list().names
if env_name not in envs:
bot.say("{} isn't an environment I recognize".format(env_name))
return
env = chef.Environment(env_name)
if 'active_set' in env.default_attributes['heat']:
bot.say('Current active set is {}'.format(
env.default_attributes['heat']['active_set'].upper()))
else:
bot.say('There is no active set for {}'.format(env_name))
def __init__(self):
self.parser = self._create_parser()
self.chef_server_url = ""
self.client_key = ""
self.client_name = ""
self.cache_max_age = 3600
self.cache_path = os.path.join(os.path.expanduser('~'),
'.ansible-chef.cache')
self.read_settings()
if self.chef_server_url and self.client_key and self.client_name:
print("Using chef ini values", file=sys.stderr)
self.api = chef.ChefAPI(self.chef_server_url, self.client_key,
self.client_name)
else:
print("Trying chef autoconfiguration", file=sys.stderr)
self.api = chef.autoconfigure()
if not self.api:
print("Could not find chef configuration", file=sys.stderr)
sys.exit(1)
import json
import urllib
from hashlib import sha1
from itertools import groupby
from operator import itemgetter
import chef
import consul
import requests
orchestrator_address = 'dbmng-mysql-orchestrator0a.42.wixprod.net'
clusters_url = "http://{}:3000/api/clusters-info".format(orchestrator_address)
response = urllib.urlopen(clusters_url)
clusters = json.loads(response.read())
#api = chef.autoconfigure()
api = chef.ChefAPI('https://chef.wixpress.com', '/etc/chef/client.pem',
'dbmng-mysql-orchestrator0a.42.wixprod.net')
c = consul.Consul(host='sys-hdc2-master0a.42.wixprod.net')
def get_db_user(artifact_id, access_type="rw"):
db_user = artifact_id.split(".")[-1].split("-")
db_user = ''.join(map(lambda s: s[:2], db_user)) + '_' + access_type
return db_user
def get_mysql_password(username, production=True):
m = hashlib.md5()
m.update(username + ('!!' if production else '??'))
password = m.hexdigest()
password = '******' + sha1(sha1(password).digest()).hexdigest()
return password
def get_client_key(hostname):
with chef.ChefAPI(CHEF_SERVER_URL, PEM_KEY, USERNAME):
client = chef.Client.create(hostname)
logger.info(client)
logger.info(client.private_key)
return client.private_key
def deploy(ctx, env_id, env_config_json=None):
"""Deploy project to AWS."""
# get target deployment environment configuration
if env_config_json is not None:
print("using provided target deployment environment configuration...")
env_config = json.loads(env_config_json)
else:
print("loading target deployment environment configuration...")
with open('envs-config.json') as envs_config_file:
env_config = json.load(envs_config_file)[env_id]
# get AWS account id
print("getting AWS account id...")
aws_account_id = boto3.client('sts').get_caller_identity()['Account']
# get shared resources stack outputs
print("getting shared AWS resources stack outputs...")
cloudformation = boto3.client('cloudformation',
region_name=env_config['region'])
res = cloudformation.describe_stacks(StackName=f'etl-pm-shared-{env_id}')
aws_resources = {
**{
v['OutputKey']: v['OutputValue']
for v in res['Stacks'][0]['Outputs']
}
}
# deploy CDK app
print("deploying AWS resources...")
ctx.run(
(f'cdk deploy'
f' --role-arn arn:aws:iam::{aws_account_id}:role/IAS-CloudFormation-Deploy-Role'
f' --require-approval never'),
env={
'CDK_DEPLOY_ENV': env_id,
'CDK_DEPLOY_REGION': env_config['region'],
'CDK_DEPLOY_ENV_CONFIG': json.dumps(env_config)
})
# get backend resources stack outputs
print("getting backend AWS resources stack outputs...")
res = cloudformation.describe_stacks(StackName=f'{APP_NAME}-{env_id}')
aws_resources.update({
**{
v['OutputKey']: v['OutputValue']
for v in res['Stacks'][0]['Outputs']
}
})
# prepare EMR assets package
print("assembling EMR assets package...")
with ctx.cd('emr'):
ctx.run(
'rm -rf ../build/assets/emr'
' && mkdir -p ../build/assets/emr'
' && cp *.py ../build/assets/emr'
' && zip -r ../build/assets/emr/packages.zip $(ls -d */) -i \\*.py'
)
# upload assets to S3
print("uploading assets to S3...")
with open('version.properties') as version_file:
deployment_version = [
v.rstrip().split('=')[1] for v in version_file
if v.startswith('version=')
][0]
region_designator = ''.join(part[0]
for part in env_config['region'].split('-'))
ctx.run(
f'aws s3 sync --delete build/assets/'
f' s3://iasrf-vcs-mixed-{region_designator}-de-{env_id}/pm/airflow/{APP_NAME}/v{deployment_version}/'
)
# prepare DAGs package
print("assembling DAGs package...")
ctx.run(
f'mkdir -p build/airflow'
f' && rm -f build/airflow/{APP_NAME}-{env_id}.zip'
f' && zip -r build/airflow/{APP_NAME}-{env_id}.zip pipeline.py dags -i \\*.py'
)
env_config['env_id'] = env_id
env_config['deployment_version'] = deployment_version
env_config['alarms_topic_name'] = aws_resources['AlarmsTopicName']
env_config['notifications_topic_name'] = aws_resources[
'NotificationsTopicName']
env_config['jas_bucket_name'] = aws_resources['DataLakeJASBucketName']
env_config['jas_bucket_data_prefix'] = aws_resources[
'DataLakeJASBucketDataPrefix']
env_config['jas_mart_bucket_name'] = aws_resources[
'DataLakeMartBucketName']
env_config['jas_mart_bucket_data_prefix'] = aws_resources[
'DataLakeMartBucketDataPrefix']
env_config['itermediate_bucket_name'] = aws_resources[
'IntermediateBucketName']
env_config['pipeline_state_table_name'] = aws_resources[
'PipelineStateTableName']
with open('build/airflow/env-config.json', 'w') as env_config_file:
json.dump(env_config, env_config_file, indent=2)
with ctx.cd('build/airflow'):
ctx.run(f'zip -g {APP_NAME}-{env_id}.zip env-config.json')
# get Airflow nodes
print("getting Airflow nodes from Chef...")
with chef.ChefAPI(CHEF_API_URL, os.environ['CHEF_API_KEY'], CHEF_API_USER):
airflow_nodes = [
node.object.name for node in chef.Search(
'node', f"tags:{env_config['airflow_cluster_id']}.{env_id}")
]
print(f"got Airflow nodes: {airflow_nodes}")
# upload the DAGs package to Airflow nodes
for node in airflow_nodes:
print(f"deploying DAGs package to {node}...")
ctx.run(
f'scp -o StrictHostKeyChecking=no build/airflow/{APP_NAME}-{env_id}.zip'
f' {AIRFLOW_SSH_USER}@{node}:/var/lib/airflow/dags/')
def __init__(self):
self.knife = util.get_knife_creds()
self.chef_api = pychef.ChefAPI(self.knife['chef_server_url'],
self.knife['client_key'],
self.knife['node_name'])
import chef
import urllib3
import json
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
nodeShow = {}
readFile = open('D:\PyChef.json', 'r')
content = json.load(readFile)
readFile.close()
url = content["aws_chef_server"]["url"]
key = content["aws_chef_server"]["key"]
client = content["aws_chef_server"]["client"]
ssl_verify = content["aws_chef_server"]["ssl_verify"]
with chef.ChefAPI(url, key, client, ssl_verify):
#with chef.ChefAPI('https://10.1.1.35:443/organizations/govindarajanv', '../../../chef/.chef/govindarajanv.key', 'govindarajanv', ssl_verify=False):
#with chef.ChefAPI.from_config_file("../../../chef/.chef/knife.rb"):
for node in chef.Node.list():
#print ("fetching node " + node)
nodeObj = chef.Node(node)
#print ("Run List of a node:", nodeObj.run_list)
nodeShow[node] = nodeObj.run_list
# Prints chef environment from node object
#print (nodeObj.chef_environment)
#Prints default attributes
#print (nodeObj.attributes.values())
#print (nodeShow)
print("List of nodes with their run lists:")
print("===================================")
def _get_testchefapi(self):
import chef
return chef.ChefAPI(config_data.test_chef_url,
config_data.test_client_key,
config_data.test_client)
def _get_testchefapi(self):
import chef
return chef.ChefAPI(TEST_CHEF_URL, TEST_CLIENT_KEY_PATH,
TEST_CLIENT_NAME)
def get_chef_nodes(cert_path, user_name):
with chef.ChefAPI('https://cbs-chef.hl.aws/organizations/cbs',
cert_path,
user_name,
ssl_verify=False):
return [str(node) for node in chef.Node.list()]
