def notifies_hipchat(start_msg, end_msg):
"""
A decorator to post a notification to hipchat at the start and end of this
function.
The `FOO_msg` arguments define template strings that can use the
following variables as context:
* `deployer` The deploying user
* `deployment_name` The deploying environment. Eg. "beta"
* `generation` The generational target. Eg. "live", "pending"
* `git_branch` The current git branch name.
* `duration` The number of wall-clock seconds taken to complete the
decorated method. Note: Only available to `end_msg`.
"""
# Ensure we have the required configs
hipchat_conf = {}
for key in ['api_token', 'room']:
hipchat_conf[key] = env.get('hipchat_%s' % key, None)
for key, value in hipchat_conf.items():
if value is None:
logger.warning(
"No hipchat_%s found. Not notifying.",
key,
)
yield
logger.warning(
"No hipchat_%s found. Not notifying.",
key,
)
return
hipchat_conf['color'] = env.get('hipchat_color', 'green')
hipchat_conf['from'] = env.get('hipchat_from', 'Neckbeard')
# Get our git branchname. Fallback to a SHA if detached head
r = git.Repo('.')
branch_name = r.commit().hexsha[:7]
if not r.head.is_detached:
branch_name = r.active_branch.name
# Build the message
context = {
'deployer': get_deployer(),
'deployment_name': env.get('_deployment_name', 'unknown'),
'generation': _get_gen_target().lower(),
'git_branch': branch_name,
}
message = start_msg % context
_send_hipchat_msg(message, hipchat_conf)
method_start = datetime.now()
yield
duration = datetime.now() - method_start
context['duration'] = duration.seconds
message = end_msg % context
_send_hipchat_msg(message, hipchat_conf)
def _send_deployment_end_newrelic():
'''
API: https://rpm.newrelic.com/accounts/87516/applications/402046/deployments/instructions # noqa
'''
GIT_MOST_RECENT_TWO_TAGS = 'git tag | tail -2'
GIT_MOST_RECENT_COMMIT_MESSAGE = 'git log -1 --format="%s"'
GIT_CURRENT_TAG = 'git describe --tags'
GITHUB_COMPARE_OPERATOR = '...'
NEWRELIC_API_HTTP_METHOD = 'POST'
def generate_github_changelog_url(tags):
return GITHUB_COMPARE_URL % GITHUB_COMPARE_OPERATOR.join(tags)
if env.get('newrelic_api_token', False):
logger.info('Announcing deployment to newrelic')
headers = {
'x-api-key': env.newrelic_api_token,
}
# Description is the generation target, e.g. active, pending
description = '%s ' % (_get_gen_target().lower(), )
params = {
'deployment[application_id]': env.newrelic_application_id,
'deployment[description]': description,
}
# Add user information to deployment
user = get_deployer()
if user:
params['deployment[user]'] = user
# Set the changelog to a github comparison URL
result = local(GIT_MOST_RECENT_TWO_TAGS, capture=True)
if result.return_code == 0:
tags = result.split()
url = generate_github_changelog_url(tags)
params['deployment[changelog]'] = url
# Append the most recent commit message to the description
result = local(GIT_MOST_RECENT_COMMIT_MESSAGE, capture=True)
if result.return_code == 0:
params['deployment[description]'] += result.strip()
# Set the revision to the current tag
result = local(GIT_CURRENT_TAG, capture=True)
if result.return_code == 0:
params['deployment[revision]'] = result.strip()
# Attempt to post the deployment to newrelic
conn = httplib.HTTPSConnection(NEWRELIC_API_HTTP_HOST)
conn.request(
NEWRELIC_API_HTTP_METHOD,
NEWRELIC_API_HTTP_URL,
urllib.urlencode(params),
headers,
)
response = conn.getresponse()
if response.status != 201:
logger.warn('Failed to post deployment to newrelic')
def notifies_hipchat(start_msg, end_msg):
"""
A decorator to post a notification to hipchat at the start and end of this
function.
The `FOO_msg` arguments define template strings that can use the
following variables as context:
* `deployer` The deploying user
* `deployment_name` The deploying environment. Eg. "beta"
* `generation` The generational target. Eg. "live", "pending"
* `git_branch` The current git branch name.
* `duration` The number of wall-clock seconds taken to complete the
decorated method. Note: Only available to `end_msg`.
"""
# Ensure we have the required configs
hipchat_conf = {}
for key in ["api_token", "room"]:
hipchat_conf[key] = env.get("hipchat_%s" % key, None)
for key, value in hipchat_conf.items():
if value is None:
logger.warning("No hipchat_%s found. Not notifying.", key)
yield
logger.warning("No hipchat_%s found. Not notifying.", key)
return
hipchat_conf["color"] = env.get("hipchat_color", "green")
hipchat_conf["from"] = env.get("hipchat_from", "Neckbeard")
# Get our git branchname. Fallback to a SHA if detached head
r = git.Repo(".")
branch_name = r.commit().hexsha[:7]
if not r.head.is_detached:
branch_name = r.active_branch.name
# Build the message
context = {
"deployer": get_deployer(),
"deployment_name": env.get("_deployment_name", "unknown"),
"generation": _get_gen_target().lower(),
"git_branch": branch_name,
}
message = start_msg % context
_send_hipchat_msg(message, hipchat_conf)
method_start = datetime.now()
yield
duration = datetime.now() - method_start
context["duration"] = duration.seconds
message = end_msg % context
_send_hipchat_msg(message, hipchat_conf)
def _send_deployment_done_desktop_notification(pre_deploy_time, deployer):
if not DT_NOTIFY:
return
title = "%(deployment)s %(target)s %(node_name)s deployed"
content = "Took %(seconds)s seconds"
time_diff = datetime.now() - pre_deploy_time
context = {
"deployment": env._deployment_name,
"target": _get_gen_target().lower(),
"node_name": deployer.node_name,
"seconds": time_diff.seconds,
}
notification = pynotify.Notification(title % context, content % context)
notification.show()
def _send_deployment_done_desktop_notification(pre_deploy_time, deployer):
if not DT_NOTIFY:
return
title = "%(deployment)s %(target)s %(node_name)s deployed"
content = "Took %(seconds)s seconds"
time_diff = datetime.now() - pre_deploy_time
context = {
'deployment': env._deployment_name,
'target': _get_gen_target().lower(),
'node_name': deployer.node_name,
'seconds': time_diff.seconds,
}
notification = pynotify.Notification(title % context, content % context)
notification.show()
def _send_deployment_end_newrelic():
"""
API: https://rpm.newrelic.com/accounts/87516/applications/402046/deployments/instructions # noqa
"""
GIT_MOST_RECENT_TWO_TAGS = "git tag | tail -2"
GIT_MOST_RECENT_COMMIT_MESSAGE = 'git log -1 --format="%s"'
GIT_CURRENT_TAG = "git describe --tags"
GITHUB_COMPARE_OPERATOR = "..."
NEWRELIC_API_HTTP_METHOD = "POST"
def generate_github_changelog_url(tags):
return GITHUB_COMPARE_URL % GITHUB_COMPARE_OPERATOR.join(tags)
if env.get("newrelic_api_token", False):
logger.info("Announcing deployment to newrelic")
headers = {"x-api-key": env.newrelic_api_token}
# Description is the generation target, e.g. active, pending
description = "%s " % (_get_gen_target().lower(),)
params = {"deployment[application_id]": env.newrelic_application_id, "deployment[description]": description}
# Add user information to deployment
user = get_deployer()
if user:
params["deployment[user]"] = user
# Set the changelog to a github comparison URL
result = local(GIT_MOST_RECENT_TWO_TAGS, capture=True)
if result.return_code == 0:
tags = result.split()
url = generate_github_changelog_url(tags)
params["deployment[changelog]"] = url
# Append the most recent commit message to the description
result = local(GIT_MOST_RECENT_COMMIT_MESSAGE, capture=True)
if result.return_code == 0:
params["deployment[description]"] += result.strip()
# Set the revision to the current tag
result = local(GIT_CURRENT_TAG, capture=True)
if result.return_code == 0:
params["deployment[revision]"] = result.strip()
# Attempt to post the deployment to newrelic
conn = httplib.HTTPSConnection(NEWRELIC_API_HTTP_HOST)
conn.request(NEWRELIC_API_HTTP_METHOD, NEWRELIC_API_HTTP_URL, urllib.urlencode(params), headers)
response = conn.getresponse()
if response.status != 201:
logger.warn("Failed to post deployment to newrelic")
def view(
environment_name,
configuration_manager,
resource_tracker,
generation=ACTIVE,
):
"""
The view task output status information about all of the cloud resources
associated with a specific generation of a specific deployment.
"""
# Hard-coding everything to work on active for now
env._active_gen = True
generation_target = _get_gen_target()
logger.info("Gathering deployment status")
environment_config = configuration_manager.get_environment_config(
environment_name,
)
deployment = Deployment(
environment_name,
environment_config.get('ec2', {}),
environment_config.get('rds', {}),
environment_config.get('elb', {}),
)
deployment.verify_deployment_state()
logger.info("Gathering nodes")
if generation_target == 'ACTIVE':
nodes = deployment.get_all_active_nodes()
elif generation_target == 'PENDING':
nodes = deployment.get_all_pending_nodes()
else:
nodes = deployment.get_all_old_nodes(is_running=1)
ec2_nodes = []
rds_nodes = []
for node in nodes:
if node.aws_type == 'ec2':
ec2_nodes.append(node)
elif node.aws_type == 'rds':
rds_nodes.append(node)
# Generation output
gen_id = None
if generation_target == 'PENDING':
gen_id = deployment.pending_gen_id
else:
gen_id = deployment.active_gen_id
if gen_id:
if generation_target == 'OLD':
print "%s generation: older than %s\n" % (
generation_target,
gen_id,
)
else:
print "%s generation: %s\n" % (generation_target, gen_id)
else:
print "No %s generation found\n" % generation_target
# Ec2 output
ec2_nodes.sort(key=lambda x: x.is_running)
print "===%s EC2 Node(s)===" % len(ec2_nodes)
if len(ec2_nodes) == 0:
print "No configured nodes"
else:
for node in ec2_nodes:
print "%s" % node.get_status_output()
print ""
# RDS output
rds_nodes.sort(key=lambda x: x.is_running)
print "===%s RDS Node(s)===" % len(rds_nodes)
if len(rds_nodes) == 0:
print "No configured nodes"
else:
for node in rds_nodes:
print "%s" % node.get_status_output()
def terminate(soft=None):
require('_deployment_name')
require('_deployment_confs')
while soft not in ['H', 'S']:
soft = prompt("Hard (permanent) or soft termination? (H/S)")
soft_terminate = bool(soft == 'S')
generation_target = _get_gen_target()
if generation_target == 'ACTIVE':
logger.critical("Can't terminate active generation")
exit(1)
if soft_terminate:
logger.info(
"SOFT terminating %s nodes." % generation_target)
logger.info("They will be removed from operation, but not terminated")
else:
logger.info(
"HARD terminating %s nodes." % generation_target)
logger.info("They will be TERMINATED. This is not reversible")
deployment = Deployment(
env._deployment_name,
env._deployment_confs['ec2'],
env._deployment_confs['rds'],
env._deployment_confs['elb'],
)
if generation_target == 'PENDING':
possible_nodes = deployment.get_all_pending_nodes(is_running=1)
else:
# OLD
possible_nodes = deployment.get_all_old_nodes(is_running=1)
# Filter out the nodes whose termination isn't yet known
# This is an optimization versus just calling `verify_deployment_state`
# directly, since we need the nodes afterwards anyway.
logger.info("Verifying run statuses")
for node in possible_nodes:
node.verify_running_state()
running_nodes = [node for node in possible_nodes if node.is_running]
if not running_nodes:
logger.info(
"No running nodes exist for generation: %s",
generation_target,
)
return
# Print the nodes we're going to terminate
for node in running_nodes:
logger.info("Terminating: %s", node)
confirm = ''
while not confirm in ['Y', 'N']:
confirm = prompt(
"Are you sure you want to TERMINATE these nodes? (Y/N)")
if confirm == 'N':
exit(1)
for node in running_nodes:
node.make_fully_inoperative()
if soft_terminate:
_disable_newrelic_monitoring(node)
else:
node.terminate()
def terminate(soft=None):
require('_deployment_name')
require('_deployment_confs')
while soft not in ['H', 'S']:
soft = prompt("Hard (permanent) or soft termination? (H/S)")
soft_terminate = bool(soft == 'S')
generation_target = _get_gen_target()
if generation_target == 'ACTIVE':
logger.critical("Can't terminate active generation")
exit(1)
if soft_terminate:
logger.info("SOFT terminating %s nodes." % generation_target)
logger.info("They will be removed from operation, but not terminated")
else:
logger.info("HARD terminating %s nodes." % generation_target)
logger.info("They will be TERMINATED. This is not reversible")
deployment = Deployment(
env._deployment_name,
env._deployment_confs['ec2'],
env._deployment_confs['rds'],
env._deployment_confs['elb'],
)
if generation_target == 'PENDING':
possible_nodes = deployment.get_all_pending_nodes(is_running=1)
else:
# OLD
possible_nodes = deployment.get_all_old_nodes(is_running=1)
# Filter out the nodes whose termination isn't yet known
# This is an optimization versus just calling `verify_deployment_state`
# directly, since we need the nodes afterwards anyway.
logger.info("Verifying run statuses")
for node in possible_nodes:
node.verify_running_state()
running_nodes = [node for node in possible_nodes if node.is_running]
if not running_nodes:
logger.info(
"No running nodes exist for generation: %s",
generation_target,
)
return
# Print the nodes we're going to terminate
for node in running_nodes:
logger.info("Terminating: %s", node)
confirm = ''
while not confirm in ['Y', 'N']:
confirm = prompt(
"Are you sure you want to TERMINATE these nodes? (Y/N)")
if confirm == 'N':
exit(1)
for node in running_nodes:
node.make_fully_inoperative()
if soft_terminate:
_disable_newrelic_monitoring(node)
else:
node.terminate()
def override():
"""
Manually fix the generational config for the given generation.
This is required for initial setup of the generational system. We are only
modifying the simpledb records of the instances, not the instances
themselves.
"""
require('_deployment_name')
require('_deployment_confs')
generation_target = _get_gen_target()
deployment = Deployment(
env._deployment_name,
env._deployment_confs['ec2'],
env._deployment_confs['rds'],
env._deployment_confs['elb'],
)
deployment.verify_deployment_state()
if generation_target not in ['ACTIVE', 'PENDING']:
exit(1)
opts = ['Y', 'N']
for aws_type, confs in env._deployment_confs.items():
for node_name, node_confs in confs.items():
if generation_target == 'ACTIVE':
node = deployment.get_active_node(aws_type, node_name)
else:
node = deployment.get_pending_node(aws_type, node_name)
if node:
print "Existing node found for %s: %s\n" % (node_name, node)
replace_node = ''
while replace_node not in opts:
replace_node = prompt("Change this node? (Y/N)")
if replace_node == 'N':
continue
else:
print "No node for %s: %s\n" % (aws_type, node_name)
retire_alter_opts = ['Retire', 'Alter']
retire_alter_response = ''
should_alter_node = False
should_retire_node = False
while retire_alter_response not in retire_alter_opts:
retire_alter_response = prompt(
"Retire or Alter node? (Retire/Alter)")
if retire_alter_response == 'Retire':
should_retire_node = True
else:
should_alter_node = True
if should_alter_node:
# Prompt if the node doesn't already exist
if not node:
add_node = ''
while add_node not in opts:
add_node = prompt(
'No node record found for <%s>-%s. Add one? '
'(Y/N)' % (aws_type, node_name)
)
if add_node == 'N':
should_alter_node = False
if should_retire_node and not node:
logger.critical(
"No node record found. Can't retire a non-existent node.")
continue
if should_alter_node:
_override_node(
node, deployment, aws_type, node_name)
elif should_retire_node:
logger.info("Retiring: %s", node)
confirm = ''
while confirm not in opts:
confirm = prompt(
"Are you sure you want to RETIRE this node? (Y/N)")
if confirm == 'Y':
node.make_fully_inoperative()
node.retire()
def override():
"""
Manually fix the generational config for the given generation.
This is required for initial setup of the generational system. We are only
modifying the simpledb records of the instances, not the instances
themselves.
"""
require('_deployment_name')
require('_deployment_confs')
generation_target = _get_gen_target()
deployment = Deployment(
env._deployment_name,
env._deployment_confs['ec2'],
env._deployment_confs['rds'],
env._deployment_confs['elb'],
)
deployment.verify_deployment_state()
if generation_target not in ['ACTIVE', 'PENDING']:
exit(1)
opts = ['Y', 'N']
for aws_type, confs in env._deployment_confs.items():
for node_name, node_confs in confs.items():
if generation_target == 'ACTIVE':
node = deployment.get_active_node(aws_type, node_name)
else:
node = deployment.get_pending_node(aws_type, node_name)
if node:
print "Existing node found for %s: %s\n" % (node_name, node)
replace_node = ''
while replace_node not in opts:
replace_node = prompt("Change this node? (Y/N)")
if replace_node == 'N':
continue
else:
print "No node for %s: %s\n" % (aws_type, node_name)
retire_alter_opts = ['Retire', 'Alter']
retire_alter_response = ''
should_alter_node = False
should_retire_node = False
while retire_alter_response not in retire_alter_opts:
retire_alter_response = prompt(
"Retire or Alter node? (Retire/Alter)")
if retire_alter_response == 'Retire':
should_retire_node = True
else:
should_alter_node = True
if should_alter_node:
# Prompt if the node doesn't already exist
if not node:
add_node = ''
while add_node not in opts:
add_node = prompt(
'No node record found for <%s>-%s. Add one? '
'(Y/N)' % (aws_type, node_name))
if add_node == 'N':
should_alter_node = False
if should_retire_node and not node:
logger.critical(
"No node record found. Can't retire a non-existent node.")
continue
if should_alter_node:
_override_node(node, deployment, aws_type, node_name)
elif should_retire_node:
logger.info("Retiring: %s", node)
confirm = ''
while confirm not in opts:
confirm = prompt(
"Are you sure you want to RETIRE this node? (Y/N)")
if confirm == 'Y':
node.make_fully_inoperative()
node.retire()
