def notifies_hipchat(start_msg, end_msg): """ A decorator to post a notification to hipchat at the start and end of this function. The `FOO_msg` arguments define template strings that can use the following variables as context: * `deployer` The deploying user * `deployment_name` The deploying environment. Eg. "beta" * `generation` The generational target. Eg. "live", "pending" * `git_branch` The current git branch name. * `duration` The number of wall-clock seconds taken to complete the decorated method. Note: Only available to `end_msg`. """ # Ensure we have the required configs hipchat_conf = {} for key in ['api_token', 'room']: hipchat_conf[key] = env.get('hipchat_%s' % key, None) for key, value in hipchat_conf.items(): if value is None: logger.warning( "No hipchat_%s found. Not notifying.", key, ) yield logger.warning( "No hipchat_%s found. Not notifying.", key, ) return hipchat_conf['color'] = env.get('hipchat_color', 'green') hipchat_conf['from'] = env.get('hipchat_from', 'Neckbeard') # Get our git branchname. Fallback to a SHA if detached head r = git.Repo('.') branch_name = r.commit().hexsha[:7] if not r.head.is_detached: branch_name = r.active_branch.name # Build the message context = { 'deployer': get_deployer(), 'deployment_name': env.get('_deployment_name', 'unknown'), 'generation': _get_gen_target().lower(), 'git_branch': branch_name, } message = start_msg % context _send_hipchat_msg(message, hipchat_conf) method_start = datetime.now() yield duration = datetime.now() - method_start context['duration'] = duration.seconds message = end_msg % context _send_hipchat_msg(message, hipchat_conf)
def _send_deployment_end_newrelic(): ''' API: https://rpm.newrelic.com/accounts/87516/applications/402046/deployments/instructions # noqa ''' GIT_MOST_RECENT_TWO_TAGS = 'git tag | tail -2' GIT_MOST_RECENT_COMMIT_MESSAGE = 'git log -1 --format="%s"' GIT_CURRENT_TAG = 'git describe --tags' GITHUB_COMPARE_OPERATOR = '...' NEWRELIC_API_HTTP_METHOD = 'POST' def generate_github_changelog_url(tags): return GITHUB_COMPARE_URL % GITHUB_COMPARE_OPERATOR.join(tags) if env.get('newrelic_api_token', False): logger.info('Announcing deployment to newrelic') headers = { 'x-api-key': env.newrelic_api_token, } # Description is the generation target, e.g. active, pending description = '%s ' % (_get_gen_target().lower(), ) params = { 'deployment[application_id]': env.newrelic_application_id, 'deployment[description]': description, } # Add user information to deployment user = get_deployer() if user: params['deployment[user]'] = user # Set the changelog to a github comparison URL result = local(GIT_MOST_RECENT_TWO_TAGS, capture=True) if result.return_code == 0: tags = result.split() url = generate_github_changelog_url(tags) params['deployment[changelog]'] = url # Append the most recent commit message to the description result = local(GIT_MOST_RECENT_COMMIT_MESSAGE, capture=True) if result.return_code == 0: params['deployment[description]'] += result.strip() # Set the revision to the current tag result = local(GIT_CURRENT_TAG, capture=True) if result.return_code == 0: params['deployment[revision]'] = result.strip() # Attempt to post the deployment to newrelic conn = httplib.HTTPSConnection(NEWRELIC_API_HTTP_HOST) conn.request( NEWRELIC_API_HTTP_METHOD, NEWRELIC_API_HTTP_URL, urllib.urlencode(params), headers, ) response = conn.getresponse() if response.status != 201: logger.warn('Failed to post deployment to newrelic')
def notifies_hipchat(start_msg, end_msg): """ A decorator to post a notification to hipchat at the start and end of this function. The `FOO_msg` arguments define template strings that can use the following variables as context: * `deployer` The deploying user * `deployment_name` The deploying environment. Eg. "beta" * `generation` The generational target. Eg. "live", "pending" * `git_branch` The current git branch name. * `duration` The number of wall-clock seconds taken to complete the decorated method. Note: Only available to `end_msg`. """ # Ensure we have the required configs hipchat_conf = {} for key in ["api_token", "room"]: hipchat_conf[key] = env.get("hipchat_%s" % key, None) for key, value in hipchat_conf.items(): if value is None: logger.warning("No hipchat_%s found. Not notifying.", key) yield logger.warning("No hipchat_%s found. Not notifying.", key) return hipchat_conf["color"] = env.get("hipchat_color", "green") hipchat_conf["from"] = env.get("hipchat_from", "Neckbeard") # Get our git branchname. Fallback to a SHA if detached head r = git.Repo(".") branch_name = r.commit().hexsha[:7] if not r.head.is_detached: branch_name = r.active_branch.name # Build the message context = { "deployer": get_deployer(), "deployment_name": env.get("_deployment_name", "unknown"), "generation": _get_gen_target().lower(), "git_branch": branch_name, } message = start_msg % context _send_hipchat_msg(message, hipchat_conf) method_start = datetime.now() yield duration = datetime.now() - method_start context["duration"] = duration.seconds message = end_msg % context _send_hipchat_msg(message, hipchat_conf)
def _send_deployment_done_desktop_notification(pre_deploy_time, deployer): if not DT_NOTIFY: return title = "%(deployment)s %(target)s %(node_name)s deployed" content = "Took %(seconds)s seconds" time_diff = datetime.now() - pre_deploy_time context = { "deployment": env._deployment_name, "target": _get_gen_target().lower(), "node_name": deployer.node_name, "seconds": time_diff.seconds, } notification = pynotify.Notification(title % context, content % context) notification.show()
def _send_deployment_done_desktop_notification(pre_deploy_time, deployer): if not DT_NOTIFY: return title = "%(deployment)s %(target)s %(node_name)s deployed" content = "Took %(seconds)s seconds" time_diff = datetime.now() - pre_deploy_time context = { 'deployment': env._deployment_name, 'target': _get_gen_target().lower(), 'node_name': deployer.node_name, 'seconds': time_diff.seconds, } notification = pynotify.Notification(title % context, content % context) notification.show()
def _send_deployment_end_newrelic(): """ API: https://rpm.newrelic.com/accounts/87516/applications/402046/deployments/instructions # noqa """ GIT_MOST_RECENT_TWO_TAGS = "git tag | tail -2" GIT_MOST_RECENT_COMMIT_MESSAGE = 'git log -1 --format="%s"' GIT_CURRENT_TAG = "git describe --tags" GITHUB_COMPARE_OPERATOR = "..." NEWRELIC_API_HTTP_METHOD = "POST" def generate_github_changelog_url(tags): return GITHUB_COMPARE_URL % GITHUB_COMPARE_OPERATOR.join(tags) if env.get("newrelic_api_token", False): logger.info("Announcing deployment to newrelic") headers = {"x-api-key": env.newrelic_api_token} # Description is the generation target, e.g. active, pending description = "%s " % (_get_gen_target().lower(),) params = {"deployment[application_id]": env.newrelic_application_id, "deployment[description]": description} # Add user information to deployment user = get_deployer() if user: params["deployment[user]"] = user # Set the changelog to a github comparison URL result = local(GIT_MOST_RECENT_TWO_TAGS, capture=True) if result.return_code == 0: tags = result.split() url = generate_github_changelog_url(tags) params["deployment[changelog]"] = url # Append the most recent commit message to the description result = local(GIT_MOST_RECENT_COMMIT_MESSAGE, capture=True) if result.return_code == 0: params["deployment[description]"] += result.strip() # Set the revision to the current tag result = local(GIT_CURRENT_TAG, capture=True) if result.return_code == 0: params["deployment[revision]"] = result.strip() # Attempt to post the deployment to newrelic conn = httplib.HTTPSConnection(NEWRELIC_API_HTTP_HOST) conn.request(NEWRELIC_API_HTTP_METHOD, NEWRELIC_API_HTTP_URL, urllib.urlencode(params), headers) response = conn.getresponse() if response.status != 201: logger.warn("Failed to post deployment to newrelic")
def view( environment_name, configuration_manager, resource_tracker, generation=ACTIVE, ): """ The view task output status information about all of the cloud resources associated with a specific generation of a specific deployment. """ # Hard-coding everything to work on active for now env._active_gen = True generation_target = _get_gen_target() logger.info("Gathering deployment status") environment_config = configuration_manager.get_environment_config( environment_name, ) deployment = Deployment( environment_name, environment_config.get('ec2', {}), environment_config.get('rds', {}), environment_config.get('elb', {}), ) deployment.verify_deployment_state() logger.info("Gathering nodes") if generation_target == 'ACTIVE': nodes = deployment.get_all_active_nodes() elif generation_target == 'PENDING': nodes = deployment.get_all_pending_nodes() else: nodes = deployment.get_all_old_nodes(is_running=1) ec2_nodes = [] rds_nodes = [] for node in nodes: if node.aws_type == 'ec2': ec2_nodes.append(node) elif node.aws_type == 'rds': rds_nodes.append(node) # Generation output gen_id = None if generation_target == 'PENDING': gen_id = deployment.pending_gen_id else: gen_id = deployment.active_gen_id if gen_id: if generation_target == 'OLD': print "%s generation: older than %s\n" % ( generation_target, gen_id, ) else: print "%s generation: %s\n" % (generation_target, gen_id) else: print "No %s generation found\n" % generation_target # Ec2 output ec2_nodes.sort(key=lambda x: x.is_running) print "===%s EC2 Node(s)===" % len(ec2_nodes) if len(ec2_nodes) == 0: print "No configured nodes" else: for node in ec2_nodes: print "%s" % node.get_status_output() print "" # RDS output rds_nodes.sort(key=lambda x: x.is_running) print "===%s RDS Node(s)===" % len(rds_nodes) if len(rds_nodes) == 0: print "No configured nodes" else: for node in rds_nodes: print "%s" % node.get_status_output()
def terminate(soft=None): require('_deployment_name') require('_deployment_confs') while soft not in ['H', 'S']: soft = prompt("Hard (permanent) or soft termination? (H/S)") soft_terminate = bool(soft == 'S') generation_target = _get_gen_target() if generation_target == 'ACTIVE': logger.critical("Can't terminate active generation") exit(1) if soft_terminate: logger.info( "SOFT terminating %s nodes." % generation_target) logger.info("They will be removed from operation, but not terminated") else: logger.info( "HARD terminating %s nodes." % generation_target) logger.info("They will be TERMINATED. This is not reversible") deployment = Deployment( env._deployment_name, env._deployment_confs['ec2'], env._deployment_confs['rds'], env._deployment_confs['elb'], ) if generation_target == 'PENDING': possible_nodes = deployment.get_all_pending_nodes(is_running=1) else: # OLD possible_nodes = deployment.get_all_old_nodes(is_running=1) # Filter out the nodes whose termination isn't yet known # This is an optimization versus just calling `verify_deployment_state` # directly, since we need the nodes afterwards anyway. logger.info("Verifying run statuses") for node in possible_nodes: node.verify_running_state() running_nodes = [node for node in possible_nodes if node.is_running] if not running_nodes: logger.info( "No running nodes exist for generation: %s", generation_target, ) return # Print the nodes we're going to terminate for node in running_nodes: logger.info("Terminating: %s", node) confirm = '' while not confirm in ['Y', 'N']: confirm = prompt( "Are you sure you want to TERMINATE these nodes? (Y/N)") if confirm == 'N': exit(1) for node in running_nodes: node.make_fully_inoperative() if soft_terminate: _disable_newrelic_monitoring(node) else: node.terminate()
def terminate(soft=None): require('_deployment_name') require('_deployment_confs') while soft not in ['H', 'S']: soft = prompt("Hard (permanent) or soft termination? (H/S)") soft_terminate = bool(soft == 'S') generation_target = _get_gen_target() if generation_target == 'ACTIVE': logger.critical("Can't terminate active generation") exit(1) if soft_terminate: logger.info("SOFT terminating %s nodes." % generation_target) logger.info("They will be removed from operation, but not terminated") else: logger.info("HARD terminating %s nodes." % generation_target) logger.info("They will be TERMINATED. This is not reversible") deployment = Deployment( env._deployment_name, env._deployment_confs['ec2'], env._deployment_confs['rds'], env._deployment_confs['elb'], ) if generation_target == 'PENDING': possible_nodes = deployment.get_all_pending_nodes(is_running=1) else: # OLD possible_nodes = deployment.get_all_old_nodes(is_running=1) # Filter out the nodes whose termination isn't yet known # This is an optimization versus just calling `verify_deployment_state` # directly, since we need the nodes afterwards anyway. logger.info("Verifying run statuses") for node in possible_nodes: node.verify_running_state() running_nodes = [node for node in possible_nodes if node.is_running] if not running_nodes: logger.info( "No running nodes exist for generation: %s", generation_target, ) return # Print the nodes we're going to terminate for node in running_nodes: logger.info("Terminating: %s", node) confirm = '' while not confirm in ['Y', 'N']: confirm = prompt( "Are you sure you want to TERMINATE these nodes? (Y/N)") if confirm == 'N': exit(1) for node in running_nodes: node.make_fully_inoperative() if soft_terminate: _disable_newrelic_monitoring(node) else: node.terminate()
def override(): """ Manually fix the generational config for the given generation. This is required for initial setup of the generational system. We are only modifying the simpledb records of the instances, not the instances themselves. """ require('_deployment_name') require('_deployment_confs') generation_target = _get_gen_target() deployment = Deployment( env._deployment_name, env._deployment_confs['ec2'], env._deployment_confs['rds'], env._deployment_confs['elb'], ) deployment.verify_deployment_state() if generation_target not in ['ACTIVE', 'PENDING']: exit(1) opts = ['Y', 'N'] for aws_type, confs in env._deployment_confs.items(): for node_name, node_confs in confs.items(): if generation_target == 'ACTIVE': node = deployment.get_active_node(aws_type, node_name) else: node = deployment.get_pending_node(aws_type, node_name) if node: print "Existing node found for %s: %s\n" % (node_name, node) replace_node = '' while replace_node not in opts: replace_node = prompt("Change this node? (Y/N)") if replace_node == 'N': continue else: print "No node for %s: %s\n" % (aws_type, node_name) retire_alter_opts = ['Retire', 'Alter'] retire_alter_response = '' should_alter_node = False should_retire_node = False while retire_alter_response not in retire_alter_opts: retire_alter_response = prompt( "Retire or Alter node? (Retire/Alter)") if retire_alter_response == 'Retire': should_retire_node = True else: should_alter_node = True if should_alter_node: # Prompt if the node doesn't already exist if not node: add_node = '' while add_node not in opts: add_node = prompt( 'No node record found for <%s>-%s. Add one? ' '(Y/N)' % (aws_type, node_name) ) if add_node == 'N': should_alter_node = False if should_retire_node and not node: logger.critical( "No node record found. Can't retire a non-existent node.") continue if should_alter_node: _override_node( node, deployment, aws_type, node_name) elif should_retire_node: logger.info("Retiring: %s", node) confirm = '' while confirm not in opts: confirm = prompt( "Are you sure you want to RETIRE this node? (Y/N)") if confirm == 'Y': node.make_fully_inoperative() node.retire()
def override(): """ Manually fix the generational config for the given generation. This is required for initial setup of the generational system. We are only modifying the simpledb records of the instances, not the instances themselves. """ require('_deployment_name') require('_deployment_confs') generation_target = _get_gen_target() deployment = Deployment( env._deployment_name, env._deployment_confs['ec2'], env._deployment_confs['rds'], env._deployment_confs['elb'], ) deployment.verify_deployment_state() if generation_target not in ['ACTIVE', 'PENDING']: exit(1) opts = ['Y', 'N'] for aws_type, confs in env._deployment_confs.items(): for node_name, node_confs in confs.items(): if generation_target == 'ACTIVE': node = deployment.get_active_node(aws_type, node_name) else: node = deployment.get_pending_node(aws_type, node_name) if node: print "Existing node found for %s: %s\n" % (node_name, node) replace_node = '' while replace_node not in opts: replace_node = prompt("Change this node? (Y/N)") if replace_node == 'N': continue else: print "No node for %s: %s\n" % (aws_type, node_name) retire_alter_opts = ['Retire', 'Alter'] retire_alter_response = '' should_alter_node = False should_retire_node = False while retire_alter_response not in retire_alter_opts: retire_alter_response = prompt( "Retire or Alter node? (Retire/Alter)") if retire_alter_response == 'Retire': should_retire_node = True else: should_alter_node = True if should_alter_node: # Prompt if the node doesn't already exist if not node: add_node = '' while add_node not in opts: add_node = prompt( 'No node record found for <%s>-%s. Add one? ' '(Y/N)' % (aws_type, node_name)) if add_node == 'N': should_alter_node = False if should_retire_node and not node: logger.critical( "No node record found. Can't retire a non-existent node.") continue if should_alter_node: _override_node(node, deployment, aws_type, node_name) elif should_retire_node: logger.info("Retiring: %s", node) confirm = '' while confirm not in opts: confirm = prompt( "Are you sure you want to RETIRE this node? (Y/N)") if confirm == 'Y': node.make_fully_inoperative() node.retire()