def __init__(self): self.parser = self._create_parser() self.chef_server_url = "" self.client_key = "" self.client_name = "" self.cache_max_age = 3600 self.cache_path = os.path.join(os.path.expanduser('~'), '.ansible-chef.cache') self.read_settings() self.api = chef.autoconfigure() if self.chef_server_url and self.client_key and self.client_name: print("Using chef ini values", file=sys.stderr) self.api = chef.ChefAPI(self.chef_server_url, self.client_key, self.client_name) else: pemfile = os.environ.get('CHEF_PEMFILE') username = os.environ.get('CHEF_USER') chef_server_url = os.environ.get('CHEF_SERVER_URL') if not self.api: if pemfile is None or username is None or chef_server_url is None: print( "Set CHEF_PEMFILE, CHEF_USER and CHEF_SERVER_URL environment vars. They might be located under ~/.chef/knife_local.rb or ~/.chef/knife.rb" ) exit(0) self.api = chef.ChefAPI(chef_server_url, pemfile, username) if not self.api: print("Could not find chef configuration", file=sys.stderr) sys.exit(1)
def handle(event, _context): """Lambda Handler""" log_event(event) # If you're using a self signed certificate change # the ssl_verify argument to False with chef.ChefAPI(CHEF_SERVER_URL, get_pem(), USERNAME, ssl_verify=VERIFY_SSL): instance_id = get_instance_id(event) try: search = chef.Search('node', 'ec2_instance_id:' + instance_id) except ChefServerNotFoundError as err: LOGGER.error(err) return False if len(search) != 0: for instance in search: node = chef.Node(instance.object.name) client = chef.Client(instance.object.name) try: node.delete() LOGGER.info('===Node Delete: SUCCESS===') client.delete() LOGGER.info('===Client Delete: SUCCESS===') return True except ChefServerNotFoundError as err: LOGGER.error(err) return False else: LOGGER.info( '=Instance does not appear to be Chef Server managed.=') return True
def edit_chef_environment(server_url, cert, username, environment, attributes): """ _edit_chef_environment_ For a given chef server, edit the override attributes in the named environment making changes for each dotted key:value pair in attributes Eg: attributes = { 'application.app_version': 'x.y.z', 'application.some_url': 'http://www.google.com' } will result in changes: env.override_attributes['application']['app_version'] = 'x.y.z' env.override_attributes['application']['some_url'] = 'http://www.google.com' :param server_url: URL of your chef server :param cert: Client.pem file location :param username: Chef username matching the cert :param environment: name of chef environment eg 'production' :param attributes: dict of dotted attribute key value pairs to change in the env """ with chef.ChefAPI(server_url, cert, username): env = chef.Environment(environment) overrides = env.override_attributes LOGGER.info("Editing Chef Server Environment: {} as {}".format( environment, username)) for attr, new_value in attributes.iteritems(): LOGGER.info(" => Setting {}={}".format(attr, new_value)) set_dotted(overrides, attr, new_value) env.save()
def active(bot, trigger): '''Display the current active set for an environment''' env_input = trigger.group(2) if env_input is None: bot.say('Please give me a chef environment') return url = bot.config.chef.url key = bot.config.chef.key username = bot.config.chef.username api = chef.ChefAPI(url, key, username) api.set_default() all_envs = chef.Environment.list().names if env_input in ('prod', 'production', 'all'): #Select all production environments envs = [env for env in all_envs if 'production-' in env] if env_input == 'all': envs.append('staging') elif env_input in all_envs: envs = [env_input] elif 'production-' + env_input in all_envs: envs = ['production-' + env_input] else: bot.say("{} isn't an environment I recognize".format(env_input)) return for env_name in sorted(envs): env = chef.Environment(env_name) if 'active_set' in env.default_attributes['heat']: bot.say('{}: Current active set is {}'.format( env_name, env.default_attributes['heat']['active_set'].upper())) else: bot.say('There is no active set for {}'.format(env_name))
def authenticate(self): """ Authenticate to chef server @return: True on success, False otherwise @rtype: boolean """ ret = False # log message content = {'chef-url': self.__url, 'chef-port': self.__port, 'login':"******" % self.__login, 'cmd': 'authenticate' } tpl = self.template(name=self.__class__.__name__.upper(), content=content ) self.logRequest(msg="authenticate", details=tpl ) try: self.__server = chef.ChefAPI(url="%s:%s" % (self.__url, self.__port), key=self.__key, client=self.__login, headers={}, ssl_verify=self.__certCheck) content = {'cmd': 'authenticated' } tpl = self.template(name=self.__class__.__name__.upper(), content=content ) self.logResponse(msg="authenticated", details=tpl ) ret = True except Exception as e: # log message content = { "chef-error": "%s" % e, "cmd": "authenticate" } tpl = self.template(name=self.__class__.__name__.upper(), content=content ) self.logResponse(msg="authenticate error", details=tpl ) return ret
def get_data(self): nodes = [] chefapi = chef.ChefAPI(chef_url, chef_key, chef_usr) for name in chef.Node.list(api=chefapi): node = chef.Node(name) nodes.append(initChefNode(node)) nodes.sort(key=op.attrgetter('id')) return nodes
def environments(bot, trigger): '''Display a list of chef environments''' url = bot.config.chef.url key = bot.config.chef.key username = bot.config.chef.username api = chef.ChefAPI(url, key, username) api.set_default() envs = chef.Environment.list().names bot.say('Chef environments: {}'.format(', '.join( chef.Environment.list().names)))
def delete_chef_nodes(cert_path, user_name, nodes): for node in nodes: if "packer" not in node: print(node) with chef.ChefAPI('https://cbs-chef.hl.aws/organizations/cbs', cert_path, user_name, ssl_verify=False): n = chef.Node(node) n.delete()
def __init__(self, endpoint, auth_data, **cfg): if not auth_data: msg = "Authorisation information is not set for the target Chef Server ("+str(endpoint)+")!" log.error(msg) raise Exception(msg) config = dict() config['client'] = auth_data['client_name'] config['key'] = auth_data['client_key'] config['url'] = endpoint self.chefapi = chef.ChefAPI(**config)
def get_data(self): try: node_id = self.kwargs['id'] chefapi = chef.ChefAPI(chef_url, chef_key, chef_usr) node = chef.Node(node_id, api=chefapi) except Exception: #redirect = self.get_redirect_url() exceptions.handle(self.request, _('Unable to retrieve node details.'), redirect=redirect) return node
def delete_node(hostname): with chef.ChefAPI(CHEF_SERVER_URL, PEM_KEY, USERNAME): try: # remove client from chef server client = chef.Client(hostname) client.delete() logger.info('Successfully deleted client %s', hostname) # remove node from chef server node = chef.Node(hostname) node.delete() logger.info('Successfully deleted node %s', hostname) except ChefServerNotFoundError as err: logger.error(err)
def chef_noop(bot, trigger): '''Display a list of noop hosts in a chef environment''' env_name = trigger.group(2) if env_name is None: bot.say('Please give me a chef environment') return url = bot.config.chef.url key = bot.config.chef.key username = bot.config.chef.username api = chef.ChefAPI(url, key, username) api.set_default() env = chef.Environment(env_name) noop = env.default_attributes['base']['noop'] bot.say('The following nodes have noop set: {}'.format(', '.join(noop)))
def list_nodes(server_url, cert, username, query, attribute='name', format_str=None): """ _list_nodes_ Get a list of chef nodes using a chef node seach query. Use attribute to select the attribute from the matching nodes and optionally provide a format string to drop the result into to create well formatted node names Example: list_nodes(server_url , cert, 'server_user', "role:some_role AND environment:prod", attribute='automatic.ip.public' ) Will list all nodes with some_role in the prod environment and return the node['automatic']['ip']['public'] attribute for each node list_nodes(server_url, cert, 'server_url', "role:some_role AND environment:prod", attribute='name', format_str='{}.cloudant.com') Will extract the name attribute for each node and return that value expanded in the format_str :param server_url: Chef Server URL :param cert: path to PEM cert for chef server auth :param username: Chef Server username :param query: chef node search query string :param attribute: Attribute name to be extracted from node data, can be a dot.delimited.style name :param format_str: optional format string to apply to each result for each node. """ result = [] with chef.ChefAPI(server_url, cert, username): for row in chef.search.Search('node', query): attr = get_dotted(row, attribute) if format_str is not None: attr = format_str.format(attr) result.append(attr) LOGGER.info("Node Found:{}".format(attr)) return result
def handle(event, _context): """Lambda Handler""" log_event(event) #Suppress InsecureRequestWarning: Unverified HTTPS request is being made in Python2.6 requests.packages.urllib3.disable_warnings(InsecureRequestWarning) # If you're using a self signed certificate change # the ssl_verify argument to False with chef.ChefAPI(CHEF_SERVER_URL, get_pem(), USERNAME, ssl_verify=VERIFY_SSL): instance_id = get_instance_id(event) try: search = chef.Search('node', 'ec2_instance_id:' + instance_id) except ChefServerNotFoundError as err: LOGGER.error(err) return False if len(search) != 0: for instance in search: node = chef.Node(instance.object.name) client = chef.Client(instance.object.name) try: LOGGER.info('About to delete the node named - ' + node.name) LOGGER.info('About to delete the client named - ' + client.name) if not DEBUG: node.delete() LOGGER.info('===Node Delete: SUCCESS===') client.delete() LOGGER.info('===Client Delete: SUCCESS===') else: LOGGER.info('Would have deleted the node named - ' + node.name + ' here, but we are in DEBUG mode') LOGGER.info('Would have deleted the client named - ' + client.name + ' here, but we are in DEBUG mode') return True except ChefServerNotFoundError as err: LOGGER.error(err) return False else: LOGGER.info( '=Instance does not appear to be Chef Server managed.=') return True
def __init__(self, settings): installer_url = settings.get(self.CHEFSERVER_URL, None) key_dir = settings.get(self.KEY_DIR, None) client = settings.get(self.CLIENT, None) try: if installer_url and key_dir and client: self.api = chef.ChefAPI(installer_url, key_dir, client) else: self.api = chef.autoconfigure() logging.info('chef client created %s(%s, %s)', installer_url, key_dir, client) except Exception as error: logging.error('failed to create chef client %s(%s, %s)', installer_url, key_dir, client) logging.exception(error)
def _get_chef_api(self, key=None, client=None): """Initializes chef API client.""" import chef chef_api = None try: if key and client: chef_api = chef.ChefAPI(self.installer_url, key, client) else: chef_api = chef.autoconfigure() except Exception as ex: err_msg = "Failed to instantiate chef API, error: %s" % ex.message raise Exception(err_msg) return chef_api
def edit_chef_role(server_url, cert, username, rolename, attributes): """ _edit_chef_role_ For a given chef server, edit the override attributes in the named role making changes for each dotted key:value pair in attributes. """ with chef.ChefAPI(server_url, cert, username): role = chef.Role(rolename) LOGGER.info("Editing Chef Server Role: {} as {}".format( rolename, username)) overrides = role.override_attributes for attr, new_value in attributes.iteritems(): LOGGER.info(" => Setting {}={}".format(attr, new_value)) set_dotted(overrides, attr, new_value) role.save()
def authenticate(self): """ Connect to the chef server and verify communication. """ if self.api: return server = self.settings['url'] key_file = self.settings['key_file'] client = self.settings['client'] verify_ssl = self.settings.get('verify_ssl', True) in TrueValues self.api = chef.ChefAPI(server, key_file, client, ssl_verify=verify_ssl) chef.Role.list( api=self.api) # will raise error if authentication failure
def active(bot, trigger): '''Display the current active set for an environment''' env_name = trigger.group(2) if env_name is None: bot.say('Please give me a chef environment') return url = bot.config.chef.url key = bot.config.chef.key username = bot.config.chef.username api = chef.ChefAPI(url, key, username) api.set_default() envs = chef.Environment.list().names if env_name not in envs: bot.say("{} isn't an environment I recognize".format(env_name)) return env = chef.Environment(env_name) if 'active_set' in env.default_attributes['heat']: bot.say('Current active set is {}'.format( env.default_attributes['heat']['active_set'].upper())) else: bot.say('There is no active set for {}'.format(env_name))
def __init__(self): self.parser = self._create_parser() self.chef_server_url = "" self.client_key = "" self.client_name = "" self.cache_max_age = 3600 self.cache_path = os.path.join(os.path.expanduser('~'), '.ansible-chef.cache') self.read_settings() if self.chef_server_url and self.client_key and self.client_name: print("Using chef ini values", file=sys.stderr) self.api = chef.ChefAPI(self.chef_server_url, self.client_key, self.client_name) else: print("Trying chef autoconfiguration", file=sys.stderr) self.api = chef.autoconfigure() if not self.api: print("Could not find chef configuration", file=sys.stderr) sys.exit(1)
import json import urllib from hashlib import sha1 from itertools import groupby from operator import itemgetter import chef import consul import requests orchestrator_address = 'dbmng-mysql-orchestrator0a.42.wixprod.net' clusters_url = "http://{}:3000/api/clusters-info".format(orchestrator_address) response = urllib.urlopen(clusters_url) clusters = json.loads(response.read()) #api = chef.autoconfigure() api = chef.ChefAPI('https://chef.wixpress.com', '/etc/chef/client.pem', 'dbmng-mysql-orchestrator0a.42.wixprod.net') c = consul.Consul(host='sys-hdc2-master0a.42.wixprod.net') def get_db_user(artifact_id, access_type="rw"): db_user = artifact_id.split(".")[-1].split("-") db_user = ''.join(map(lambda s: s[:2], db_user)) + '_' + access_type return db_user def get_mysql_password(username, production=True): m = hashlib.md5() m.update(username + ('!!' if production else '??')) password = m.hexdigest() password = '******' + sha1(sha1(password).digest()).hexdigest() return password
def get_client_key(hostname): with chef.ChefAPI(CHEF_SERVER_URL, PEM_KEY, USERNAME): client = chef.Client.create(hostname) logger.info(client) logger.info(client.private_key) return client.private_key
def deploy(ctx, env_id, env_config_json=None): """Deploy project to AWS.""" # get target deployment environment configuration if env_config_json is not None: print("using provided target deployment environment configuration...") env_config = json.loads(env_config_json) else: print("loading target deployment environment configuration...") with open('envs-config.json') as envs_config_file: env_config = json.load(envs_config_file)[env_id] # get AWS account id print("getting AWS account id...") aws_account_id = boto3.client('sts').get_caller_identity()['Account'] # get shared resources stack outputs print("getting shared AWS resources stack outputs...") cloudformation = boto3.client('cloudformation', region_name=env_config['region']) res = cloudformation.describe_stacks(StackName=f'etl-pm-shared-{env_id}') aws_resources = { **{ v['OutputKey']: v['OutputValue'] for v in res['Stacks'][0]['Outputs'] } } # deploy CDK app print("deploying AWS resources...") ctx.run( (f'cdk deploy' f' --role-arn arn:aws:iam::{aws_account_id}:role/IAS-CloudFormation-Deploy-Role' f' --require-approval never'), env={ 'CDK_DEPLOY_ENV': env_id, 'CDK_DEPLOY_REGION': env_config['region'], 'CDK_DEPLOY_ENV_CONFIG': json.dumps(env_config) }) # get backend resources stack outputs print("getting backend AWS resources stack outputs...") res = cloudformation.describe_stacks(StackName=f'{APP_NAME}-{env_id}') aws_resources.update({ **{ v['OutputKey']: v['OutputValue'] for v in res['Stacks'][0]['Outputs'] } }) # prepare EMR assets package print("assembling EMR assets package...") with ctx.cd('emr'): ctx.run( 'rm -rf ../build/assets/emr' ' && mkdir -p ../build/assets/emr' ' && cp *.py ../build/assets/emr' ' && zip -r ../build/assets/emr/packages.zip $(ls -d */) -i \\*.py' ) # upload assets to S3 print("uploading assets to S3...") with open('version.properties') as version_file: deployment_version = [ v.rstrip().split('=')[1] for v in version_file if v.startswith('version=') ][0] region_designator = ''.join(part[0] for part in env_config['region'].split('-')) ctx.run( f'aws s3 sync --delete build/assets/' f' s3://iasrf-vcs-mixed-{region_designator}-de-{env_id}/pm/airflow/{APP_NAME}/v{deployment_version}/' ) # prepare DAGs package print("assembling DAGs package...") ctx.run( f'mkdir -p build/airflow' f' && rm -f build/airflow/{APP_NAME}-{env_id}.zip' f' && zip -r build/airflow/{APP_NAME}-{env_id}.zip pipeline.py dags -i \\*.py' ) env_config['env_id'] = env_id env_config['deployment_version'] = deployment_version env_config['alarms_topic_name'] = aws_resources['AlarmsTopicName'] env_config['notifications_topic_name'] = aws_resources[ 'NotificationsTopicName'] env_config['jas_bucket_name'] = aws_resources['DataLakeJASBucketName'] env_config['jas_bucket_data_prefix'] = aws_resources[ 'DataLakeJASBucketDataPrefix'] env_config['jas_mart_bucket_name'] = aws_resources[ 'DataLakeMartBucketName'] env_config['jas_mart_bucket_data_prefix'] = aws_resources[ 'DataLakeMartBucketDataPrefix'] env_config['itermediate_bucket_name'] = aws_resources[ 'IntermediateBucketName'] env_config['pipeline_state_table_name'] = aws_resources[ 'PipelineStateTableName'] with open('build/airflow/env-config.json', 'w') as env_config_file: json.dump(env_config, env_config_file, indent=2) with ctx.cd('build/airflow'): ctx.run(f'zip -g {APP_NAME}-{env_id}.zip env-config.json') # get Airflow nodes print("getting Airflow nodes from Chef...") with chef.ChefAPI(CHEF_API_URL, os.environ['CHEF_API_KEY'], CHEF_API_USER): airflow_nodes = [ node.object.name for node in chef.Search( 'node', f"tags:{env_config['airflow_cluster_id']}.{env_id}") ] print(f"got Airflow nodes: {airflow_nodes}") # upload the DAGs package to Airflow nodes for node in airflow_nodes: print(f"deploying DAGs package to {node}...") ctx.run( f'scp -o StrictHostKeyChecking=no build/airflow/{APP_NAME}-{env_id}.zip' f' {AIRFLOW_SSH_USER}@{node}:/var/lib/airflow/dags/')
def __init__(self): self.knife = util.get_knife_creds() self.chef_api = pychef.ChefAPI(self.knife['chef_server_url'], self.knife['client_key'], self.knife['node_name'])
import chef import urllib3 import json urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) nodeShow = {} readFile = open('D:\PyChef.json', 'r') content = json.load(readFile) readFile.close() url = content["aws_chef_server"]["url"] key = content["aws_chef_server"]["key"] client = content["aws_chef_server"]["client"] ssl_verify = content["aws_chef_server"]["ssl_verify"] with chef.ChefAPI(url, key, client, ssl_verify): #with chef.ChefAPI('https://10.1.1.35:443/organizations/govindarajanv', '../../../chef/.chef/govindarajanv.key', 'govindarajanv', ssl_verify=False): #with chef.ChefAPI.from_config_file("../../../chef/.chef/knife.rb"): for node in chef.Node.list(): #print ("fetching node " + node) nodeObj = chef.Node(node) #print ("Run List of a node:", nodeObj.run_list) nodeShow[node] = nodeObj.run_list # Prints chef environment from node object #print (nodeObj.chef_environment) #Prints default attributes #print (nodeObj.attributes.values()) #print (nodeShow) print("List of nodes with their run lists:") print("===================================")
def _get_testchefapi(self): import chef return chef.ChefAPI(config_data.test_chef_url, config_data.test_client_key, config_data.test_client)
def _get_testchefapi(self): import chef return chef.ChefAPI(TEST_CHEF_URL, TEST_CLIENT_KEY_PATH, TEST_CLIENT_NAME)
def get_chef_nodes(cert_path, user_name): with chef.ChefAPI('https://cbs-chef.hl.aws/organizations/cbs', cert_path, user_name, ssl_verify=False): return [str(node) for node in chef.Node.list()]