def repair(force='n'): """ Ensure that all healthy active-generation nodes are operational. """ require('_deployment_name') require('_deployment_confs') require('_active_gen') force = force == 'y' assert env._active_gen deployment = Deployment( env._deployment_name, env._deployment_confs['ec2'], env._deployment_confs['rds'], env._deployment_confs['elb'], ) deployment.verify_deployment_state() activated_nodes = deployment.repair_active_generation( force_operational=force) if len(activated_nodes): logger.info("Succesfully made %s node(s) operational", len(activated_nodes)) else: logger.info("No nodes modified")
def increment(): """ Move the fully-healthy pending generation to the ACTIVE state. """ require('_deployment_name') require('_deployment_confs') deployment = Deployment( env._deployment_name, env._deployment_confs['ec2'], env._deployment_confs['rds'], env._deployment_confs['elb'], ) deployment.verify_deployment_state() try: deployment.increment_generation() except: logger.critical("Error incrementing generation") raise pagerduty_conf = env._deployment_confs['conf'].get('pagerduty', {}) if pagerduty_conf.get('temporarily_become_oncall', False): _take_temporary_pagerduty( duration=pagerduty_conf.get('temporary_oncall_duration'), api_key=pagerduty_conf.get('api_key'), user_id=pagerduty_conf.get('user_id'), project_subdomain=pagerduty_conf.get('project_subdomain'), schedule_key=pagerduty_conf.get('schedule_key'), ) logger.warning("Previously-active generation still exists in the " "loadbalancer. Use `terminate` to remove it")
def test_differing_aws_credentials(self): ec2_configs = { 'web0-0': { "name": "web0", "unique_id": "web0-0", "aws": { "access_key_id": "FOO", "secret_access_key": "FOO", }, }, } rds_configs = { 'web1-0': { "name": "web1", "unique_id": "web1-0", "aws": { "access_key_id": "BAR", "secret_access_key": "BAR", }, }, } self.assertRaises( NonUniformAWSCredentials, lambda: Deployment('test', ec2_configs, rds_configs, {}), )
def test_boto_connections_created(self): # If the AWS credentials are valid, those credentials are used to # create connections ec2_configs = { 'web0-0': { "name": "web0", "unique_id": "web0-0", "aws": { "access_key_id": "FOO", "secret_access_key": "FOO", }, }, } rds_configs = { 'web1-0': { "name": "web1", "unique_id": "web1-0", "aws": { "access_key_id": "FOO", "secret_access_key": "FOO", }, }, } with mock.patch('boto.auth.get_auth_handler', autospec=True): deployment = Deployment('test', ec2_configs, rds_configs, {}) self.assertNotEqual(deployment.ec2conn, None) self.assertNotEqual(deployment.rdsconn, None) for conn in [deployment.ec2conn, deployment.rdsconn]: self.assertEqual(conn.access_key, 'FOO') self.assertEqual(conn.secret_key, 'FOO')
def run(): """ Sets the env.hosts variable to contain all of the app servers in the appropriate generation and deployment. """ require('_deployment_name') require('_deployment_confs') require('_active_gen') deployment = Deployment( env._deployment_name, env._deployment_confs['ec2'], env._deployment_confs['rds'], env._deployment_confs['elb'], ) deployment.verify_deployment_state() # All rds and ec2 nodes, rds nodes first dep_confs = [] dep_confs.append(('rds', sorted(env._deployment_confs['rds'].items()))) dep_confs.append(('ec2', sorted(env._deployment_confs['ec2'].items()))) hosts = [] for aws_type, node_confs in dep_confs: for node_name, conf_ in node_confs: if aws_type != 'ec2': continue if env._active_gen: node = deployment.get_active_node('ec2', node_name) else: node = deployment.get_pending_node('ec2', node_name) if (not node or not node.boto_instance or not node.boto_instance.public_dns_name): continue # Set the user value, only the last value holds conf_key = env._deployment_confs[aws_type][node_name]['conf_key'] if 'user' in env.INSTANCES[conf_key]: env.user = env.INSTANCES[conf_key]['user'] hosts.append(node.boto_instance.public_dns_name) env.hosts = hosts
def test_no_aws_level(self): missing_aws_configs = { 'web0-0': { "name": "web0", "unique_id": "web0-0", }, } self.assertRaises( MissingAWSCredentials, lambda: Deployment('test', missing_aws_configs, {}, {}), )
def test_no_aws_secret_key(self): no_secret_key = { 'web0-0': { "name": "web0", "unique_id": "web0-0", "aws": { "access_key_id": "FOO", }, }, } self.assertRaises( MissingAWSCredentials, lambda: Deployment('test', no_secret_key, {}, {}), )
def test_aws_credentials_none(self): none_configs = { 'web0-0': { "name": "web0", "unique_id": "web0-0", "aws": { "access_key_id": None, "secret_access_key": None, }, }, } self.assertRaises( MissingAWSCredentials, lambda: Deployment('test', none_configs, {}, {}), )
def view( environment_name, configuration_manager, resource_tracker, generation=ACTIVE, ): """ The view task output status information about all of the cloud resources associated with a specific generation of a specific deployment. """ # Hard-coding everything to work on active for now env._active_gen = True generation_target = _get_gen_target() logger.info("Gathering deployment status") environment_config = configuration_manager.get_environment_config( environment_name, ) deployment = Deployment( environment_name, environment_config.get('ec2', {}), environment_config.get('rds', {}), environment_config.get('elb', {}), ) deployment.verify_deployment_state() logger.info("Gathering nodes") if generation_target == 'ACTIVE': nodes = deployment.get_all_active_nodes() elif generation_target == 'PENDING': nodes = deployment.get_all_pending_nodes() else: nodes = deployment.get_all_old_nodes(is_running=1) ec2_nodes = [] rds_nodes = [] for node in nodes: if node.aws_type == 'ec2': ec2_nodes.append(node) elif node.aws_type == 'rds': rds_nodes.append(node) # Generation output gen_id = None if generation_target == 'PENDING': gen_id = deployment.pending_gen_id else: gen_id = deployment.active_gen_id if gen_id: if generation_target == 'OLD': print "%s generation: older than %s\n" % ( generation_target, gen_id, ) else: print "%s generation: %s\n" % (generation_target, gen_id) else: print "No %s generation found\n" % generation_target # Ec2 output ec2_nodes.sort(key=lambda x: x.is_running) print "===%s EC2 Node(s)===" % len(ec2_nodes) if len(ec2_nodes) == 0: print "No configured nodes" else: for node in ec2_nodes: print "%s" % node.get_status_output() print "" # RDS output rds_nodes.sort(key=lambda x: x.is_running) print "===%s RDS Node(s)===" % len(rds_nodes) if len(rds_nodes) == 0: print "No configured nodes" else: for node in rds_nodes: print "%s" % node.get_status_output()
def override(): """ Manually fix the generational config for the given generation. This is required for initial setup of the generational system. We are only modifying the simpledb records of the instances, not the instances themselves. """ require('_deployment_name') require('_deployment_confs') generation_target = _get_gen_target() deployment = Deployment( env._deployment_name, env._deployment_confs['ec2'], env._deployment_confs['rds'], env._deployment_confs['elb'], ) deployment.verify_deployment_state() if generation_target not in ['ACTIVE', 'PENDING']: exit(1) opts = ['Y', 'N'] for aws_type, confs in env._deployment_confs.items(): for node_name, node_confs in confs.items(): if generation_target == 'ACTIVE': node = deployment.get_active_node(aws_type, node_name) else: node = deployment.get_pending_node(aws_type, node_name) if node: print "Existing node found for %s: %s\n" % (node_name, node) replace_node = '' while replace_node not in opts: replace_node = prompt("Change this node? (Y/N)") if replace_node == 'N': continue else: print "No node for %s: %s\n" % (aws_type, node_name) retire_alter_opts = ['Retire', 'Alter'] retire_alter_response = '' should_alter_node = False should_retire_node = False while retire_alter_response not in retire_alter_opts: retire_alter_response = prompt( "Retire or Alter node? (Retire/Alter)") if retire_alter_response == 'Retire': should_retire_node = True else: should_alter_node = True if should_alter_node: # Prompt if the node doesn't already exist if not node: add_node = '' while add_node not in opts: add_node = prompt( 'No node record found for <%s>-%s. Add one? ' '(Y/N)' % (aws_type, node_name)) if add_node == 'N': should_alter_node = False if should_retire_node and not node: logger.critical( "No node record found. Can't retire a non-existent node.") continue if should_alter_node: _override_node(node, deployment, aws_type, node_name) elif should_retire_node: logger.info("Retiring: %s", node) confirm = '' while confirm not in opts: confirm = prompt( "Are you sure you want to RETIRE this node? (Y/N)") if confirm == 'Y': node.make_fully_inoperative() node.retire()
def terminate(soft=None): require('_deployment_name') require('_deployment_confs') while soft not in ['H', 'S']: soft = prompt("Hard (permanent) or soft termination? (H/S)") soft_terminate = bool(soft == 'S') generation_target = _get_gen_target() if generation_target == 'ACTIVE': logger.critical("Can't terminate active generation") exit(1) if soft_terminate: logger.info( "SOFT terminating %s nodes." % generation_target) logger.info("They will be removed from operation, but not terminated") else: logger.info( "HARD terminating %s nodes." % generation_target) logger.info("They will be TERMINATED. This is not reversible") deployment = Deployment( env._deployment_name, env._deployment_confs['ec2'], env._deployment_confs['rds'], env._deployment_confs['elb'], ) if generation_target == 'PENDING': possible_nodes = deployment.get_all_pending_nodes(is_running=1) else: # OLD possible_nodes = deployment.get_all_old_nodes(is_running=1) # Filter out the nodes whose termination isn't yet known # This is an optimization versus just calling `verify_deployment_state` # directly, since we need the nodes afterwards anyway. logger.info("Verifying run statuses") for node in possible_nodes: node.verify_running_state() running_nodes = [node for node in possible_nodes if node.is_running] if not running_nodes: logger.info( "No running nodes exist for generation: %s", generation_target, ) return # Print the nodes we're going to terminate for node in running_nodes: logger.info("Terminating: %s", node) confirm = '' while not confirm in ['Y', 'N']: confirm = prompt( "Are you sure you want to TERMINATE these nodes? (Y/N)") if confirm == 'N': exit(1) for node in running_nodes: node.make_fully_inoperative() if soft_terminate: _disable_newrelic_monitoring(node) else: node.terminate()
def up( environment_name, configuration_manager, resource_tracker, generation=ACTIVE, ): """ Make sure that the instances for the specified generation are running and have current code. Will update code and deploy new EC2 and RDS instances as needed. """ env._active_gen = True if generation == ACTIVE: # Always force the active generation in operation if possible make_operational = True with logs_duration(timer, timer_name='pre_deploy_validation'): # TODO: Make this an optional hook that can be registered git_conf = {} if git_conf.get('enable'): repo = _get_git_repo() # Force submodules to be updated # TODO: Make this an optional hook that can be registered with prompt_on_exception("Git submodule update failed"): repo.submodule_update(init=True, recursive=True) # Optionally require that we deploy from a tagged commit. if git_conf.get('require_tag', False): logger.info("Enforcing git tag requirement") if not _is_unchanged_from_head(repo): logger.critical( "Refusing to deploy, uncommitted changes exist.") exit(1) if not _is_tagged_version(repo): logger.critical( "Refusing to deploy from an untagged commit.", ) exit(1) _push_tags(repo) # TODO: Make this an optional hook that can be registered pagerduty_conf = {} if pagerduty_conf.get('temporarily_become_oncall', False): logger.info("Taking Pagerduty, temporarily") _take_temporary_pagerduty( duration=pagerduty_conf.get('temporary_oncall_duration'), api_key=pagerduty_conf.get('api_key'), user_id=pagerduty_conf.get('user_id'), project_subdomain=pagerduty_conf.get('project_subdomain'), schedule_key=pagerduty_conf.get('schedule_key'), ) logger.info("Gathering deployment state") with logs_duration(timer, timer_name='gather deployment state'): environment_config = configuration_manager.get_environment_config( environment_name, ) deployment = Deployment( environment_name, environment_config.get('ec2', {}), environment_config.get('rds', {}), environment_config.get('elb', {}), ) # up never deals with old nodes, so just verify pending and active to # save HTTP round trips deployment.verify_deployment_state(verify_old=False) # Gather all of the configurations for each node, including their # seed deployment information logger.info("Gathering seed deployment state") with logs_duration(timer, timer_name='seed_deployment_state'): # If this environment has a seed environment, build that environment # manager seed_deployment = None seed_deployment_name = configuration_manager.get_seed_environment_name( environment_name, ) if seed_deployment_name: seed_config = configuration_manager.get_environment_config( seed_deployment_name, ) seed_deployment = Deployment( seed_deployment_name, seed_config.get('ec2', {}), seed_config.get('rds', {}), seed_config.get('elb', {}), ) logger.info("Verifying seed deployment state") seed_deployment.verify_deployment_state(verify_old=False) # Build all of the deployment objects logger.info("Building Node deployers") with logs_duration(timer, timer_name='build deployers'): ec2_deployers = [] rds_deployers = [] # All rds and ec2 nodes, rds nodes first dep_confs = [ ( 'rds', environment_config.get('ec2', {}), ), ( 'ec2', environment_config.get('rds', {}), ), ] for aws_type, node_confs in dep_confs: for node_name, conf in node_confs.items(): # Get the seed deployment new instances will be copied from seed_node_name = None if seed_deployment and 'seed' in conf: seed_node_name = conf['seed']['unique_id'] verify_seed_data = conf['seed_node'].get('verify', False) else: logger.info("No seed node configured") seed_node_name = None verify_seed_data = False if aws_type == 'ec2': klass = Ec2NodeDeployment elif aws_type == 'rds': klass = RdsNodeDeployment deployer = klass( deployment=deployment, seed_deployment=seed_deployment, is_active=env._active_gen, aws_type=aws_type, node_name=node_name, seed_node_name=seed_node_name, seed_verification=verify_seed_data, brain_wrinkles=conf.get('brain_wrinkles', {}), conf=conf, ) if aws_type == 'ec2': ec2_deployers.append(deployer) elif aws_type == 'rds': rds_deployers.append(deployer) # We don't actually want to do deployments until we have tests assert False # Provision the RDS nodes with logs_duration(timer, timer_name='initial provision'): logger.info("Provisioning RDS nodes") for deployer in rds_deployers: if deployer.seed_verification and deployer.get_node() is None: _prompt_for_seed_verification(deployer) deployer.ensure_node_created() # Provision the EC2 nodes logger.info("Provisioning EC2 nodes") for deployer in ec2_deployers: if deployer.seed_verification and deployer.get_node() is None: _prompt_for_seed_verification(deployer) deployer.ensure_node_created() # Configure the RDS nodes logger.info("Configuring RDS nodes") with logs_duration(timer, timer_name='deploy rds'): for deployer in rds_deployers: deployer.run() logger.info("Determining EC2 node deploy priority") ec2_deployers = _order_ec2_deployers_by_priority(ec2_deployers) # Configure the EC2 nodes logger.info("Deploying to EC2 nodes") for deployer in ec2_deployers: timer_name = '%s deploy' % deployer.node_name with logs_duration(timer, timer_name='full %s' % timer_name): node = deployer.get_node() with seamless_modification( node, deployer.deployment, force_seamless=env._active_gen, make_operational_if_not_already=make_operational, ): pre_deploy_time = datetime.now() with logs_duration( timer, timer_name=timer_name, output_result=True, ): deployer.run() if DT_NOTIFY: _send_deployment_done_desktop_notification( pre_deploy_time, deployer, ) _announce_deployment() time_logger.info("Timing Breakdown:") sorted_timers = sorted( timer.items(), key=lambda x: x[1], reverse=True, ) for timer_name, duration in sorted_timers: time_logger.info("%02ds- %s", duration, timer_name)
def override(): """ Manually fix the generational config for the given generation. This is required for initial setup of the generational system. We are only modifying the simpledb records of the instances, not the instances themselves. """ require('_deployment_name') require('_deployment_confs') generation_target = _get_gen_target() deployment = Deployment( env._deployment_name, env._deployment_confs['ec2'], env._deployment_confs['rds'], env._deployment_confs['elb'], ) deployment.verify_deployment_state() if generation_target not in ['ACTIVE', 'PENDING']: exit(1) opts = ['Y', 'N'] for aws_type, confs in env._deployment_confs.items(): for node_name, node_confs in confs.items(): if generation_target == 'ACTIVE': node = deployment.get_active_node(aws_type, node_name) else: node = deployment.get_pending_node(aws_type, node_name) if node: print "Existing node found for %s: %s\n" % (node_name, node) replace_node = '' while replace_node not in opts: replace_node = prompt("Change this node? (Y/N)") if replace_node == 'N': continue else: print "No node for %s: %s\n" % (aws_type, node_name) retire_alter_opts = ['Retire', 'Alter'] retire_alter_response = '' should_alter_node = False should_retire_node = False while retire_alter_response not in retire_alter_opts: retire_alter_response = prompt( "Retire or Alter node? (Retire/Alter)") if retire_alter_response == 'Retire': should_retire_node = True else: should_alter_node = True if should_alter_node: # Prompt if the node doesn't already exist if not node: add_node = '' while add_node not in opts: add_node = prompt( 'No node record found for <%s>-%s. Add one? ' '(Y/N)' % (aws_type, node_name) ) if add_node == 'N': should_alter_node = False if should_retire_node and not node: logger.critical( "No node record found. Can't retire a non-existent node.") continue if should_alter_node: _override_node( node, deployment, aws_type, node_name) elif should_retire_node: logger.info("Retiring: %s", node) confirm = '' while confirm not in opts: confirm = prompt( "Are you sure you want to RETIRE this node? (Y/N)") if confirm == 'Y': node.make_fully_inoperative() node.retire()
def terminate(soft=None): require('_deployment_name') require('_deployment_confs') while soft not in ['H', 'S']: soft = prompt("Hard (permanent) or soft termination? (H/S)") soft_terminate = bool(soft == 'S') generation_target = _get_gen_target() if generation_target == 'ACTIVE': logger.critical("Can't terminate active generation") exit(1) if soft_terminate: logger.info("SOFT terminating %s nodes." % generation_target) logger.info("They will be removed from operation, but not terminated") else: logger.info("HARD terminating %s nodes." % generation_target) logger.info("They will be TERMINATED. This is not reversible") deployment = Deployment( env._deployment_name, env._deployment_confs['ec2'], env._deployment_confs['rds'], env._deployment_confs['elb'], ) if generation_target == 'PENDING': possible_nodes = deployment.get_all_pending_nodes(is_running=1) else: # OLD possible_nodes = deployment.get_all_old_nodes(is_running=1) # Filter out the nodes whose termination isn't yet known # This is an optimization versus just calling `verify_deployment_state` # directly, since we need the nodes afterwards anyway. logger.info("Verifying run statuses") for node in possible_nodes: node.verify_running_state() running_nodes = [node for node in possible_nodes if node.is_running] if not running_nodes: logger.info( "No running nodes exist for generation: %s", generation_target, ) return # Print the nodes we're going to terminate for node in running_nodes: logger.info("Terminating: %s", node) confirm = '' while not confirm in ['Y', 'N']: confirm = prompt( "Are you sure you want to TERMINATE these nodes? (Y/N)") if confirm == 'N': exit(1) for node in running_nodes: node.make_fully_inoperative() if soft_terminate: _disable_newrelic_monitoring(node) else: node.terminate()