def handle(event, _context):
"""Lambda Handler"""
log_event(event)
# If you're using a self signed certificate change
# the ssl_verify argument to False
with chef.ChefAPI(CHEF_SERVER_URL,
get_pem(),
USERNAME,
ssl_verify=VERIFY_SSL):
instance_id = get_instance_id(event)
try:
search = chef.Search('node', 'ec2_instance_id:' + instance_id)
except ChefServerNotFoundError as err:
LOGGER.error(err)
return False
if len(search) != 0:
for instance in search:
node = chef.Node(instance.object.name)
client = chef.Client(instance.object.name)
try:
node.delete()
LOGGER.info('===Node Delete: SUCCESS===')
client.delete()
LOGGER.info('===Client Delete: SUCCESS===')
return True
except ChefServerNotFoundError as err:
LOGGER.error(err)
return False
else:
LOGGER.info(
'=Instance does not appear to be Chef Server managed.=')
return True
def search_pick_node(search_query, api):
"""
Perform node search for `search_query` with `api`,
lets the user pick interactivly if there are multiple nodes returned
Throws LookupError if can't find a matching node
"""
nodes = [node for node in chef.Search('node', search_query, api=api)]
node = None
# no nodes match the search
if not nodes:
raise exceptions.LookupError("Can't find a node matching %s" % (search_query,))
# more than one node matched the search. let the user choose one
elif len(nodes) > 1:
# sort the results before displaying them
nodes.sort(lambda a,b: cmp(a.object.name.lower(), b.object.name.lower()))
print '%d nodes matched your query:' % (len(nodes),)
for i, current_node in enumerate(nodes):
print "\t%d. %s" % (i+1, current_node.object.name,)
while not node:
selected_node = raw_input('Please select one: ')
if not selected_node.isdigit() or not 1 <= int(selected_node) <= len(nodes):
print 'Invalid selection. Please choose a number between 1 and %d' % (len(nodes),)
else:
node = nodes[int(selected_node) - 1]
print
# only one result. that's our node.
else:
node = nodes[0]
return node
def check_cluster_health(self, callback_url):
import chef
cluster_name = self.config_manager.get_clustername()
nodes = chef.Search(
'node',
'tags:rally_node AND name:*%s' % cluster_name,
api=self.chef_api
)
if not nodes:
err_msg = "Cannot find Rally node!"
logging.info(err_msg)
raise Exception(err_msg)
rally_node_name = None
for node in nodes:
rally_node_name = node.object.name
break
rally_node = chef.Node(rally_node_name, api=self.chef_api)
rally_node_ip = rally_node['ipaddress']
command = self.config_manager.get_adapter_health_check_cmd()
command = command.replace('$cluster_name', cluster_name)
option = '--url %s --clustername %s' % (callback_url, cluster_name)
command = ' '.join((command, option))
username, pwd = self.config_manager.get_server_credentials()
util.execute_cli_by_ssh(
command, rally_node_ip, username=username,
password=pwd, nowait=True
)
def refresh_cache(self):
print("REFRESHING CACHE - COULD TAKE A WHILE", file=sys.stderr)
with self.api:
data = {}
nodes = chef.Search("node")
for n in nodes:
data[n["name"]] = n
self.write_cache(data)
def _list_all_nodes(self):
resp = dict()
with self.api:
environments = [e[u'name'] for e in chef.Search(u'environment')]
nodes = chef.Search(u'node')
for env in environments:
apis = u'{}_api'.format(env)
workers = u'{}_worker'.format(env)
dbs = u'{}_db'.format(env)
queues = u'{}_queues'.format(env)
repose = u'{}_repose'.format(env)
resp[env] = {u'children': [apis, workers, dbs, queues, repose]}
resp[apis] = [n[u'automatic'][u'ipaddress'] for n in nodes if (
n[u'chef_environment'] == env and \
u'roles' in n[u'automatic'].keys() and \
u'barbican-api' in n[u'automatic'][u'roles']
)]
resp[workers] = [n[u'automatic'][u'ipaddress'] for n in nodes \
if (
n[u'chef_environment'] == env and \
u'roles' in n[u'automatic'].keys() and \
u'barbican-worker' in n[u'automatic'][u'roles']
)
]
resp[dbs] = [n[u'automatic'][u'ipaddress'] for n in nodes if (
n[u'chef_environment'] == env and \
u'roles' in n[u'automatic'].keys() and \
u'barbican-db' in n[u'automatic'][u'roles']
)]
resp[queues] = [n[u'automatic'][u'ipaddress'] for n in nodes \
if (
n[u'chef_environment'] == env and \
u'roles' in n[u'automatic'].keys() and \
u'barbican-queue' in n[u'automatic'][u'roles']
)
]
resp[repose] = [n[u'automatic'][u'ipaddress'] for n in nodes \
if (
n[u'chef_environment'] == env and \
u'roles' in n[u'automatic'].keys() and \
u'barbican-repose' in n[u'automatic'][u'roles']
)
]
print(json.dumps(resp))
def env(env):
if env == BOOTSTRAP_ENV:
flask.abort(400)
if len(chef.Search('environment', 'name:' + env, api = api)) == 0:
flask.abort(404)
nodes = list(chef.Search('node', 'chef_environment:%s' % env, api = api))
nodes.sort(key = lambda n: n.object.name)
status, output, converging = get_env_status(env, (n.object.name for n in nodes), progress_status = 'converging')
return flask.render_template(
'env.html',
env = env,
converging = converging,
status = status,
output = output,
nodes = nodes,
)
def role_validator(data):
result = True
for key in data:
if key.find('role') == 0:
role_name = key[key.find('[') + 1:-1]
if len(chef.Search('role', 'name:%s' % role_name)) == 0:
logger.error(
"Role '%s' does not exist on chef server. Skipping node" %
role_name)
result = False
break
return result
def lambda_handler(event, context):
print("Event received: " + json.dumps(event))
for record in event['Records']:
if 'Sns' in record:
message = json.loads(record['Sns']['Message'])
if message['Event'] == 'autoscaling:EC2_INSTANCE_TERMINATE':
instance_id = message['EC2InstanceId']
print("instance_id = " + instance_id)
try:
chef_api = chef.autoconfigure()
except:
raise Exception('Could not configure Chef!')
try:
rows = chef.Search('node',
'ec2_instance_id:' + instance_id)
except:
raise Exception(
'Could not search for nodes with ec2_instance_id: ' +
instance_id)
for row in rows:
try:
n = chef.Node(row['name'])
except:
raise Exception('Could not fetch node object for ' +
row['name'])
print("node: " + str(n))
try:
c = chef.Client(row['name'])
except:
raise Exception('Could not fetch client object for ' +
row['name'])
print("client: " + str(c))
try:
n.delete()
except:
raise Exception('Could not delete node ' + str(n))
try:
c.delete()
except:
raise Exception('Could not delete client ' + str(n))
else:
raise Exception('Could not process SNS message')
def handle(event, _context):
"""Lambda Handler"""
log_event(event)
#Suppress InsecureRequestWarning: Unverified HTTPS request is being made in Python2.6
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
# If you're using a self signed certificate change
# the ssl_verify argument to False
with chef.ChefAPI(CHEF_SERVER_URL,
get_pem(),
USERNAME,
ssl_verify=VERIFY_SSL):
instance_id = get_instance_id(event)
try:
search = chef.Search('node', 'ec2_instance_id:' + instance_id)
except ChefServerNotFoundError as err:
LOGGER.error(err)
return False
if len(search) != 0:
for instance in search:
node = chef.Node(instance.object.name)
client = chef.Client(instance.object.name)
try:
LOGGER.info('About to delete the node named - ' +
node.name)
LOGGER.info('About to delete the client named - ' +
client.name)
if not DEBUG:
node.delete()
LOGGER.info('===Node Delete: SUCCESS===')
client.delete()
LOGGER.info('===Client Delete: SUCCESS===')
else:
LOGGER.info('Would have deleted the node named - ' +
node.name +
' here, but we are in DEBUG mode')
LOGGER.info('Would have deleted the client named - ' +
client.name +
' here, but we are in DEBUG mode')
return True
except ChefServerNotFoundError as err:
LOGGER.error(err)
return False
else:
LOGGER.info(
'=Instance does not appear to be Chef Server managed.=')
return True
def process_virtual(chefapi,
data,
config_file,
internal_dns_suffix,
force_update=False):
assign_dict = {}
# find all internal nodes
internal_nodes = chef.Search('node',
'role:%s' % INTERNAL_NODE_ROLE,
api=chefapi)
nodes = itertools.cycle(list(internal_nodes))
# walk all virtual machines, slotting them to internal_nodes
for machine in data['virtual']:
node_to_assign = nodes.next()
node_name = node_to_assign['name']
logger.info("Assigning virtual machine '%s' to node %s'" %
(machine['name'], node_name))
if not assign_dict.has_key(node_name):
assign_dict[node_name] = []
assign_dict[node_name].append(machine)
# create chef entry for the virtual machine itself
virtual_name = machine['name'] + '.' + internal_dns_suffix
chef_node_obj = chef.Node(name=virtual_name, api=chefapi)
chef_node_obj.run_list = machine['chef']['run_list']
chef_node_obj.save()
for node in assign_dict:
attrib_dict = {
'internal-node': {
'virtual_machines': assign_dict[node]
}
}
chef_node_obj = chef.Node(name=node, api=chefapi)
chef_node_obj.normal.update(attrib_dict)
chef_node_obj.save()
def bootstrap(ip):
if not bootstrap_enabled or not app.config.get('ENABLE_BOOTSTRAP'):
flask.abort(400)
if len(processes(BOOTSTRAP_ENV, ip, only_executing = True)) > 0:
return ujson.encode({ 'status': 'bootstrapping' })
if len(chef.Search('node', 'ipaddress:%s OR fqdn:%s OR hostname:%s' % (ip, ip, ip), api = api)) > 0:
broadcast(BOOTSTRAP_ENV, { 'host': ip, 'status': 'ready', 'data': 'A node already exists at this address.\n' })
return ujson.encode({ 'status': 'ready' })
get_command = lambda ip: ['knife', 'bootstrap', '--sudo', ip]
return _run(
{ ip: ip, },
get_command,
env = BOOTSTRAP_ENV,
progress_status = 'bootstrapping',
)
def chef_nodes(self):
"""Generator method that queries the chef server and yields node dictionaries."""
q = 'name:*'
size = 1000
start = 0
while start is not None:
response = chef.Search('node',
q,
rows=size,
start=start,
api=self.api)
count = 0
for node in response:
count += 1
yield node
if count == 0 or count < size:
start = None
else:
start += size
def get(self, host):
"""Add <host> to device class in POD for monitoring"""
deviceclass = self.get_argument('deviceclass',
default='/Server/SSH/Linux/Collector')
#productionState wants an int Prod=1000,PreProd=500,Maint=300,Test=0,Decom=-1
prodstate = str(self.get_argument('prodstate', default="500"))
chefapi = chef.autoconfigure()
master = chef.Node(chefapi.client)
vpcid = master['vpc_id']
privip = master['private_ips'][0]
stackname = master['cluster']['name']
collectors = master['zenoss']['collector-passwords']
zaasdomain = master['zaas']['domain']
for c in collectors:
if c['collector'] == host:
#devicename = c['ip']
devicename = host + '.' + stackname + '.internal.' + zaasdomain
pod = chef.Search('node', 'hostname:POD* AND vpc_id:' + vpcid)
podip = pod[0]['automatic']['ipaddress']
password = pod[0]['normal']['zenoss']['password']
user = '******'
url = 'https://' + podip + '/zport/dmd/device_router'
data = '{"action":"DeviceRouter","method":"addDevice","data":[{"deviceName":"' + devicename + '","deviceClass":"' + deviceclass + '","collector":"localhost","title":"' + host + '","productionState":"' + prodstate + '"}],"tid":1}'
userpass = user + ":" + password
auth = "Basic " + userpass.encode("base64").rstrip()
headers = {
"Authorization": auth,
"Content-Type": "application/json; charset=utf-8"
}
request = tornado.httpclient.HTTPRequest(url=url,
method='POST',
body=data,
headers=headers,
validate_cert=False)
client = tornado.httpclient.AsyncHTTPClient()
response = yield tornado.gen.Task(client.fetch, request)
self.finish(response.body)
def converge(env, node = None):
if env == BOOTSTRAP_ENV:
flask.abort(400)
if len(processes(env, node, only_executing = True)) > 0:
return ujson.encode({ 'status': 'converging' })
if node is not None:
nodes = { node: chef.Node(node, api = api), }
else:
nodes = { row.object.name: row.object for row in chef.Search('node', 'chef_environment:' + env, api = api) }
get_command = lambda n: ['ssh', '-o', 'StrictHostKeyChecking=no', n['ipaddress'], 'sudo', 'chef-client']
return _run(
nodes,
get_command,
env = env,
progress_status = 'converging',
)
def deploy(ctx, env_id, env_config_json=None):
"""Deploy project to AWS."""
# get target deployment environment configuration
if env_config_json is not None:
print("using provided target deployment environment configuration...")
env_config = json.loads(env_config_json)
else:
print("loading target deployment environment configuration...")
with open('envs-config.json') as envs_config_file:
env_config = json.load(envs_config_file)[env_id]
# get AWS account id
print("getting AWS account id...")
aws_account_id = boto3.client('sts').get_caller_identity()['Account']
# get shared resources stack outputs
print("getting shared AWS resources stack outputs...")
cloudformation = boto3.client('cloudformation',
region_name=env_config['region'])
res = cloudformation.describe_stacks(StackName=f'etl-pm-shared-{env_id}')
aws_resources = {
**{
v['OutputKey']: v['OutputValue']
for v in res['Stacks'][0]['Outputs']
}
}
# deploy CDK app
print("deploying AWS resources...")
ctx.run(
(f'cdk deploy'
f' --role-arn arn:aws:iam::{aws_account_id}:role/IAS-CloudFormation-Deploy-Role'
f' --require-approval never'),
env={
'CDK_DEPLOY_ENV': env_id,
'CDK_DEPLOY_REGION': env_config['region'],
'CDK_DEPLOY_ENV_CONFIG': json.dumps(env_config)
})
# get backend resources stack outputs
print("getting backend AWS resources stack outputs...")
res = cloudformation.describe_stacks(StackName=f'{APP_NAME}-{env_id}')
aws_resources.update({
**{
v['OutputKey']: v['OutputValue']
for v in res['Stacks'][0]['Outputs']
}
})
# prepare EMR assets package
print("assembling EMR assets package...")
with ctx.cd('emr'):
ctx.run(
'rm -rf ../build/assets/emr'
' && mkdir -p ../build/assets/emr'
' && cp *.py ../build/assets/emr'
' && zip -r ../build/assets/emr/packages.zip $(ls -d */) -i \\*.py'
)
# upload assets to S3
print("uploading assets to S3...")
with open('version.properties') as version_file:
deployment_version = [
v.rstrip().split('=')[1] for v in version_file
if v.startswith('version=')
][0]
region_designator = ''.join(part[0]
for part in env_config['region'].split('-'))
ctx.run(
f'aws s3 sync --delete build/assets/'
f' s3://iasrf-vcs-mixed-{region_designator}-de-{env_id}/pm/airflow/{APP_NAME}/v{deployment_version}/'
)
# prepare DAGs package
print("assembling DAGs package...")
ctx.run(
f'mkdir -p build/airflow'
f' && rm -f build/airflow/{APP_NAME}-{env_id}.zip'
f' && zip -r build/airflow/{APP_NAME}-{env_id}.zip pipeline.py dags -i \\*.py'
)
env_config['env_id'] = env_id
env_config['deployment_version'] = deployment_version
env_config['alarms_topic_name'] = aws_resources['AlarmsTopicName']
env_config['notifications_topic_name'] = aws_resources[
'NotificationsTopicName']
env_config['jas_bucket_name'] = aws_resources['DataLakeJASBucketName']
env_config['jas_bucket_data_prefix'] = aws_resources[
'DataLakeJASBucketDataPrefix']
env_config['jas_mart_bucket_name'] = aws_resources[
'DataLakeMartBucketName']
env_config['jas_mart_bucket_data_prefix'] = aws_resources[
'DataLakeMartBucketDataPrefix']
env_config['itermediate_bucket_name'] = aws_resources[
'IntermediateBucketName']
env_config['pipeline_state_table_name'] = aws_resources[
'PipelineStateTableName']
with open('build/airflow/env-config.json', 'w') as env_config_file:
json.dump(env_config, env_config_file, indent=2)
with ctx.cd('build/airflow'):
ctx.run(f'zip -g {APP_NAME}-{env_id}.zip env-config.json')
# get Airflow nodes
print("getting Airflow nodes from Chef...")
with chef.ChefAPI(CHEF_API_URL, os.environ['CHEF_API_KEY'], CHEF_API_USER):
airflow_nodes = [
node.object.name for node in chef.Search(
'node', f"tags:{env_config['airflow_cluster_id']}.{env_id}")
]
print(f"got Airflow nodes: {airflow_nodes}")
# upload the DAGs package to Airflow nodes
for node in airflow_nodes:
print(f"deploying DAGs package to {node}...")
ctx.run(
f'scp -o StrictHostKeyChecking=no build/airflow/{APP_NAME}-{env_id}.zip'
f' {AIRFLOW_SSH_USER}@{node}:/var/lib/airflow/dags/')
