def repair(force='n'):
"""
Ensure that all healthy active-generation nodes are operational.
"""
require('_deployment_name')
require('_deployment_confs')
require('_active_gen')
force = force == 'y'
assert env._active_gen
deployment = Deployment(
env._deployment_name,
env._deployment_confs['ec2'],
env._deployment_confs['rds'],
env._deployment_confs['elb'],
)
deployment.verify_deployment_state()
activated_nodes = deployment.repair_active_generation(
force_operational=force)
if len(activated_nodes):
logger.info("Succesfully made %s node(s) operational",
len(activated_nodes))
else:
logger.info("No nodes modified")
def increment():
"""
Move the fully-healthy pending generation to the ACTIVE state.
"""
require('_deployment_name')
require('_deployment_confs')
deployment = Deployment(
env._deployment_name,
env._deployment_confs['ec2'],
env._deployment_confs['rds'],
env._deployment_confs['elb'],
)
deployment.verify_deployment_state()
try:
deployment.increment_generation()
except:
logger.critical("Error incrementing generation")
raise
pagerduty_conf = env._deployment_confs['conf'].get('pagerduty', {})
if pagerduty_conf.get('temporarily_become_oncall', False):
_take_temporary_pagerduty(
duration=pagerduty_conf.get('temporary_oncall_duration'),
api_key=pagerduty_conf.get('api_key'),
user_id=pagerduty_conf.get('user_id'),
project_subdomain=pagerduty_conf.get('project_subdomain'),
schedule_key=pagerduty_conf.get('schedule_key'),
)
logger.warning("Previously-active generation still exists in the "
"loadbalancer. Use `terminate` to remove it")
def repair(force='n'):
"""
Ensure that all healthy active-generation nodes are operational.
"""
require('_deployment_name')
require('_deployment_confs')
require('_active_gen')
force = force == 'y'
assert env._active_gen
deployment = Deployment(
env._deployment_name,
env._deployment_confs['ec2'],
env._deployment_confs['rds'],
env._deployment_confs['elb'],
)
deployment.verify_deployment_state()
activated_nodes = deployment.repair_active_generation(
force_operational=force)
if len(activated_nodes):
logger.info("Succesfully made %s node(s) operational",
len(activated_nodes))
else:
logger.info("No nodes modified")
def test_differing_aws_credentials(self):
ec2_configs = {
'web0-0': {
"name": "web0",
"unique_id": "web0-0",
"aws": {
"access_key_id": "FOO",
"secret_access_key": "FOO",
},
},
}
rds_configs = {
'web1-0': {
"name": "web1",
"unique_id": "web1-0",
"aws": {
"access_key_id": "BAR",
"secret_access_key": "BAR",
},
},
}
self.assertRaises(
NonUniformAWSCredentials,
lambda: Deployment('test', ec2_configs, rds_configs, {}),
)
def test_boto_connections_created(self):
# If the AWS credentials are valid, those credentials are used to
# create connections
ec2_configs = {
'web0-0': {
"name": "web0",
"unique_id": "web0-0",
"aws": {
"access_key_id": "FOO",
"secret_access_key": "FOO",
},
},
}
rds_configs = {
'web1-0': {
"name": "web1",
"unique_id": "web1-0",
"aws": {
"access_key_id": "FOO",
"secret_access_key": "FOO",
},
},
}
with mock.patch('boto.auth.get_auth_handler', autospec=True):
deployment = Deployment('test', ec2_configs, rds_configs, {})
self.assertNotEqual(deployment.ec2conn, None)
self.assertNotEqual(deployment.rdsconn, None)
for conn in [deployment.ec2conn, deployment.rdsconn]:
self.assertEqual(conn.access_key, 'FOO')
self.assertEqual(conn.secret_key, 'FOO')
def run():
"""
Sets the env.hosts variable to contain all of the app servers in the
appropriate generation and deployment.
"""
require('_deployment_name')
require('_deployment_confs')
require('_active_gen')
deployment = Deployment(
env._deployment_name,
env._deployment_confs['ec2'],
env._deployment_confs['rds'],
env._deployment_confs['elb'],
)
deployment.verify_deployment_state()
# All rds and ec2 nodes, rds nodes first
dep_confs = []
dep_confs.append(('rds', sorted(env._deployment_confs['rds'].items())))
dep_confs.append(('ec2', sorted(env._deployment_confs['ec2'].items())))
hosts = []
for aws_type, node_confs in dep_confs:
for node_name, conf_ in node_confs:
if aws_type != 'ec2':
continue
if env._active_gen:
node = deployment.get_active_node('ec2', node_name)
else:
node = deployment.get_pending_node('ec2', node_name)
if (not node
or not node.boto_instance
or not node.boto_instance.public_dns_name):
continue
# Set the user value, only the last value holds
conf_key = env._deployment_confs[aws_type][node_name]['conf_key']
if 'user' in env.INSTANCES[conf_key]:
env.user = env.INSTANCES[conf_key]['user']
hosts.append(node.boto_instance.public_dns_name)
env.hosts = hosts
def test_no_aws_level(self):
missing_aws_configs = {
'web0-0': {
"name": "web0",
"unique_id": "web0-0",
},
}
self.assertRaises(
MissingAWSCredentials,
lambda: Deployment('test', missing_aws_configs, {}, {}),
)
def increment():
"""
Move the fully-healthy pending generation to the ACTIVE state.
"""
require('_deployment_name')
require('_deployment_confs')
deployment = Deployment(
env._deployment_name,
env._deployment_confs['ec2'],
env._deployment_confs['rds'],
env._deployment_confs['elb'],
)
deployment.verify_deployment_state()
try:
deployment.increment_generation()
except:
logger.critical("Error incrementing generation")
raise
pagerduty_conf = env._deployment_confs['conf'].get('pagerduty', {})
if pagerduty_conf.get('temporarily_become_oncall', False):
_take_temporary_pagerduty(
duration=pagerduty_conf.get('temporary_oncall_duration'),
api_key=pagerduty_conf.get('api_key'),
user_id=pagerduty_conf.get('user_id'),
project_subdomain=pagerduty_conf.get('project_subdomain'),
schedule_key=pagerduty_conf.get('schedule_key'),
)
logger.warning("Previously-active generation still exists in the "
"loadbalancer. Use `terminate` to remove it")
def test_no_aws_secret_key(self):
no_secret_key = {
'web0-0': {
"name": "web0",
"unique_id": "web0-0",
"aws": {
"access_key_id": "FOO",
},
},
}
self.assertRaises(
MissingAWSCredentials,
lambda: Deployment('test', no_secret_key, {}, {}),
)
def test_aws_credentials_none(self):
none_configs = {
'web0-0': {
"name": "web0",
"unique_id": "web0-0",
"aws": {
"access_key_id": None,
"secret_access_key": None,
},
},
}
self.assertRaises(
MissingAWSCredentials,
lambda: Deployment('test', none_configs, {}, {}),
)
def run():
"""
Sets the env.hosts variable to contain all of the app servers in the
appropriate generation and deployment.
"""
require('_deployment_name')
require('_deployment_confs')
require('_active_gen')
deployment = Deployment(
env._deployment_name,
env._deployment_confs['ec2'],
env._deployment_confs['rds'],
env._deployment_confs['elb'],
)
deployment.verify_deployment_state()
# All rds and ec2 nodes, rds nodes first
dep_confs = []
dep_confs.append(('rds', sorted(env._deployment_confs['rds'].items())))
dep_confs.append(('ec2', sorted(env._deployment_confs['ec2'].items())))
hosts = []
for aws_type, node_confs in dep_confs:
for node_name, conf_ in node_confs:
if aws_type != 'ec2':
continue
if env._active_gen:
node = deployment.get_active_node('ec2', node_name)
else:
node = deployment.get_pending_node('ec2', node_name)
if (not node or not node.boto_instance
or not node.boto_instance.public_dns_name):
continue
# Set the user value, only the last value holds
conf_key = env._deployment_confs[aws_type][node_name]['conf_key']
if 'user' in env.INSTANCES[conf_key]:
env.user = env.INSTANCES[conf_key]['user']
hosts.append(node.boto_instance.public_dns_name)
env.hosts = hosts
def view(
environment_name,
configuration_manager,
resource_tracker,
generation=ACTIVE,
):
"""
The view task output status information about all of the cloud resources
associated with a specific generation of a specific deployment.
"""
# Hard-coding everything to work on active for now
env._active_gen = True
generation_target = _get_gen_target()
logger.info("Gathering deployment status")
environment_config = configuration_manager.get_environment_config(
environment_name,
)
deployment = Deployment(
environment_name,
environment_config.get('ec2', {}),
environment_config.get('rds', {}),
environment_config.get('elb', {}),
)
deployment.verify_deployment_state()
logger.info("Gathering nodes")
if generation_target == 'ACTIVE':
nodes = deployment.get_all_active_nodes()
elif generation_target == 'PENDING':
nodes = deployment.get_all_pending_nodes()
else:
nodes = deployment.get_all_old_nodes(is_running=1)
ec2_nodes = []
rds_nodes = []
for node in nodes:
if node.aws_type == 'ec2':
ec2_nodes.append(node)
elif node.aws_type == 'rds':
rds_nodes.append(node)
# Generation output
gen_id = None
if generation_target == 'PENDING':
gen_id = deployment.pending_gen_id
else:
gen_id = deployment.active_gen_id
if gen_id:
if generation_target == 'OLD':
print "%s generation: older than %s\n" % (
generation_target,
gen_id,
)
else:
print "%s generation: %s\n" % (generation_target, gen_id)
else:
print "No %s generation found\n" % generation_target
# Ec2 output
ec2_nodes.sort(key=lambda x: x.is_running)
print "===%s EC2 Node(s)===" % len(ec2_nodes)
if len(ec2_nodes) == 0:
print "No configured nodes"
else:
for node in ec2_nodes:
print "%s" % node.get_status_output()
print ""
# RDS output
rds_nodes.sort(key=lambda x: x.is_running)
print "===%s RDS Node(s)===" % len(rds_nodes)
if len(rds_nodes) == 0:
print "No configured nodes"
else:
for node in rds_nodes:
print "%s" % node.get_status_output()
def override():
"""
Manually fix the generational config for the given generation.
This is required for initial setup of the generational system. We are only
modifying the simpledb records of the instances, not the instances
themselves.
"""
require('_deployment_name')
require('_deployment_confs')
generation_target = _get_gen_target()
deployment = Deployment(
env._deployment_name,
env._deployment_confs['ec2'],
env._deployment_confs['rds'],
env._deployment_confs['elb'],
)
deployment.verify_deployment_state()
if generation_target not in ['ACTIVE', 'PENDING']:
exit(1)
opts = ['Y', 'N']
for aws_type, confs in env._deployment_confs.items():
for node_name, node_confs in confs.items():
if generation_target == 'ACTIVE':
node = deployment.get_active_node(aws_type, node_name)
else:
node = deployment.get_pending_node(aws_type, node_name)
if node:
print "Existing node found for %s: %s\n" % (node_name, node)
replace_node = ''
while replace_node not in opts:
replace_node = prompt("Change this node? (Y/N)")
if replace_node == 'N':
continue
else:
print "No node for %s: %s\n" % (aws_type, node_name)
retire_alter_opts = ['Retire', 'Alter']
retire_alter_response = ''
should_alter_node = False
should_retire_node = False
while retire_alter_response not in retire_alter_opts:
retire_alter_response = prompt(
"Retire or Alter node? (Retire/Alter)")
if retire_alter_response == 'Retire':
should_retire_node = True
else:
should_alter_node = True
if should_alter_node:
# Prompt if the node doesn't already exist
if not node:
add_node = ''
while add_node not in opts:
add_node = prompt(
'No node record found for <%s>-%s. Add one? '
'(Y/N)' % (aws_type, node_name))
if add_node == 'N':
should_alter_node = False
if should_retire_node and not node:
logger.critical(
"No node record found. Can't retire a non-existent node.")
continue
if should_alter_node:
_override_node(node, deployment, aws_type, node_name)
elif should_retire_node:
logger.info("Retiring: %s", node)
confirm = ''
while confirm not in opts:
confirm = prompt(
"Are you sure you want to RETIRE this node? (Y/N)")
if confirm == 'Y':
node.make_fully_inoperative()
node.retire()
def terminate(soft=None):
require('_deployment_name')
require('_deployment_confs')
while soft not in ['H', 'S']:
soft = prompt("Hard (permanent) or soft termination? (H/S)")
soft_terminate = bool(soft == 'S')
generation_target = _get_gen_target()
if generation_target == 'ACTIVE':
logger.critical("Can't terminate active generation")
exit(1)
if soft_terminate:
logger.info(
"SOFT terminating %s nodes." % generation_target)
logger.info("They will be removed from operation, but not terminated")
else:
logger.info(
"HARD terminating %s nodes." % generation_target)
logger.info("They will be TERMINATED. This is not reversible")
deployment = Deployment(
env._deployment_name,
env._deployment_confs['ec2'],
env._deployment_confs['rds'],
env._deployment_confs['elb'],
)
if generation_target == 'PENDING':
possible_nodes = deployment.get_all_pending_nodes(is_running=1)
else:
# OLD
possible_nodes = deployment.get_all_old_nodes(is_running=1)
# Filter out the nodes whose termination isn't yet known
# This is an optimization versus just calling `verify_deployment_state`
# directly, since we need the nodes afterwards anyway.
logger.info("Verifying run statuses")
for node in possible_nodes:
node.verify_running_state()
running_nodes = [node for node in possible_nodes if node.is_running]
if not running_nodes:
logger.info(
"No running nodes exist for generation: %s",
generation_target,
)
return
# Print the nodes we're going to terminate
for node in running_nodes:
logger.info("Terminating: %s", node)
confirm = ''
while not confirm in ['Y', 'N']:
confirm = prompt(
"Are you sure you want to TERMINATE these nodes? (Y/N)")
if confirm == 'N':
exit(1)
for node in running_nodes:
node.make_fully_inoperative()
if soft_terminate:
_disable_newrelic_monitoring(node)
else:
node.terminate()
def up(
environment_name,
configuration_manager,
resource_tracker,
generation=ACTIVE,
):
"""
Make sure that the instances for the specified generation are running and
have current code. Will update code and deploy new EC2 and RDS instances as
needed.
"""
env._active_gen = True
if generation == ACTIVE:
# Always force the active generation in operation if possible
make_operational = True
with logs_duration(timer, timer_name='pre_deploy_validation'):
# TODO: Make this an optional hook that can be registered
git_conf = {}
if git_conf.get('enable'):
repo = _get_git_repo()
# Force submodules to be updated
# TODO: Make this an optional hook that can be registered
with prompt_on_exception("Git submodule update failed"):
repo.submodule_update(init=True, recursive=True)
# Optionally require that we deploy from a tagged commit.
if git_conf.get('require_tag', False):
logger.info("Enforcing git tag requirement")
if not _is_unchanged_from_head(repo):
logger.critical(
"Refusing to deploy, uncommitted changes exist.")
exit(1)
if not _is_tagged_version(repo):
logger.critical(
"Refusing to deploy from an untagged commit.", )
exit(1)
_push_tags(repo)
# TODO: Make this an optional hook that can be registered
pagerduty_conf = {}
if pagerduty_conf.get('temporarily_become_oncall', False):
logger.info("Taking Pagerduty, temporarily")
_take_temporary_pagerduty(
duration=pagerduty_conf.get('temporary_oncall_duration'),
api_key=pagerduty_conf.get('api_key'),
user_id=pagerduty_conf.get('user_id'),
project_subdomain=pagerduty_conf.get('project_subdomain'),
schedule_key=pagerduty_conf.get('schedule_key'),
)
logger.info("Gathering deployment state")
with logs_duration(timer, timer_name='gather deployment state'):
environment_config = configuration_manager.get_environment_config(
environment_name, )
deployment = Deployment(
environment_name,
environment_config.get('ec2', {}),
environment_config.get('rds', {}),
environment_config.get('elb', {}),
)
# up never deals with old nodes, so just verify pending and active to
# save HTTP round trips
deployment.verify_deployment_state(verify_old=False)
# Gather all of the configurations for each node, including their
# seed deployment information
logger.info("Gathering seed deployment state")
with logs_duration(timer, timer_name='seed_deployment_state'):
# If this environment has a seed environment, build that environment
# manager
seed_deployment = None
seed_deployment_name = configuration_manager.get_seed_environment_name(
environment_name, )
if seed_deployment_name:
seed_config = configuration_manager.get_environment_config(
seed_deployment_name, )
seed_deployment = Deployment(
seed_deployment_name,
seed_config.get('ec2', {}),
seed_config.get('rds', {}),
seed_config.get('elb', {}),
)
logger.info("Verifying seed deployment state")
seed_deployment.verify_deployment_state(verify_old=False)
# Build all of the deployment objects
logger.info("Building Node deployers")
with logs_duration(timer, timer_name='build deployers'):
ec2_deployers = []
rds_deployers = []
# All rds and ec2 nodes, rds nodes first
dep_confs = [
(
'rds',
environment_config.get('ec2', {}),
),
(
'ec2',
environment_config.get('rds', {}),
),
]
for aws_type, node_confs in dep_confs:
for node_name, conf in node_confs.items():
# Get the seed deployment new instances will be copied from
seed_node_name = None
if seed_deployment and 'seed' in conf:
seed_node_name = conf['seed']['unique_id']
verify_seed_data = conf['seed_node'].get('verify', False)
else:
logger.info("No seed node configured")
seed_node_name = None
verify_seed_data = False
if aws_type == 'ec2':
klass = Ec2NodeDeployment
elif aws_type == 'rds':
klass = RdsNodeDeployment
deployer = klass(
deployment=deployment,
seed_deployment=seed_deployment,
is_active=env._active_gen,
aws_type=aws_type,
node_name=node_name,
seed_node_name=seed_node_name,
seed_verification=verify_seed_data,
brain_wrinkles=conf.get('brain_wrinkles', {}),
conf=conf,
)
if aws_type == 'ec2':
ec2_deployers.append(deployer)
elif aws_type == 'rds':
rds_deployers.append(deployer)
# We don't actually want to do deployments until we have tests
assert False
# Provision the RDS nodes
with logs_duration(timer, timer_name='initial provision'):
logger.info("Provisioning RDS nodes")
for deployer in rds_deployers:
if deployer.seed_verification and deployer.get_node() is None:
_prompt_for_seed_verification(deployer)
deployer.ensure_node_created()
# Provision the EC2 nodes
logger.info("Provisioning EC2 nodes")
for deployer in ec2_deployers:
if deployer.seed_verification and deployer.get_node() is None:
_prompt_for_seed_verification(deployer)
deployer.ensure_node_created()
# Configure the RDS nodes
logger.info("Configuring RDS nodes")
with logs_duration(timer, timer_name='deploy rds'):
for deployer in rds_deployers:
deployer.run()
logger.info("Determining EC2 node deploy priority")
ec2_deployers = _order_ec2_deployers_by_priority(ec2_deployers)
# Configure the EC2 nodes
logger.info("Deploying to EC2 nodes")
for deployer in ec2_deployers:
timer_name = '%s deploy' % deployer.node_name
with logs_duration(timer, timer_name='full %s' % timer_name):
node = deployer.get_node()
with seamless_modification(
node,
deployer.deployment,
force_seamless=env._active_gen,
make_operational_if_not_already=make_operational,
):
pre_deploy_time = datetime.now()
with logs_duration(
timer,
timer_name=timer_name,
output_result=True,
):
deployer.run()
if DT_NOTIFY:
_send_deployment_done_desktop_notification(
pre_deploy_time,
deployer,
)
_announce_deployment()
time_logger.info("Timing Breakdown:")
sorted_timers = sorted(
timer.items(),
key=lambda x: x[1],
reverse=True,
)
for timer_name, duration in sorted_timers:
time_logger.info("%02ds- %s", duration, timer_name)
def up(
environment_name,
configuration_manager,
resource_tracker,
generation=ACTIVE,
):
"""
Make sure that the instances for the specified generation are running and
have current code. Will update code and deploy new EC2 and RDS instances as
needed.
"""
env._active_gen = True
if generation == ACTIVE:
# Always force the active generation in operation if possible
make_operational = True
with logs_duration(timer, timer_name='pre_deploy_validation'):
# TODO: Make this an optional hook that can be registered
git_conf = {}
if git_conf.get('enable'):
repo = _get_git_repo()
# Force submodules to be updated
# TODO: Make this an optional hook that can be registered
with prompt_on_exception("Git submodule update failed"):
repo.submodule_update(init=True, recursive=True)
# Optionally require that we deploy from a tagged commit.
if git_conf.get('require_tag', False):
logger.info("Enforcing git tag requirement")
if not _is_unchanged_from_head(repo):
logger.critical(
"Refusing to deploy, uncommitted changes exist.")
exit(1)
if not _is_tagged_version(repo):
logger.critical(
"Refusing to deploy from an untagged commit.",
)
exit(1)
_push_tags(repo)
# TODO: Make this an optional hook that can be registered
pagerduty_conf = {}
if pagerduty_conf.get('temporarily_become_oncall', False):
logger.info("Taking Pagerduty, temporarily")
_take_temporary_pagerduty(
duration=pagerduty_conf.get('temporary_oncall_duration'),
api_key=pagerduty_conf.get('api_key'),
user_id=pagerduty_conf.get('user_id'),
project_subdomain=pagerduty_conf.get('project_subdomain'),
schedule_key=pagerduty_conf.get('schedule_key'),
)
logger.info("Gathering deployment state")
with logs_duration(timer, timer_name='gather deployment state'):
environment_config = configuration_manager.get_environment_config(
environment_name,
)
deployment = Deployment(
environment_name,
environment_config.get('ec2', {}),
environment_config.get('rds', {}),
environment_config.get('elb', {}),
)
# up never deals with old nodes, so just verify pending and active to
# save HTTP round trips
deployment.verify_deployment_state(verify_old=False)
# Gather all of the configurations for each node, including their
# seed deployment information
logger.info("Gathering seed deployment state")
with logs_duration(timer, timer_name='seed_deployment_state'):
# If this environment has a seed environment, build that environment
# manager
seed_deployment = None
seed_deployment_name = configuration_manager.get_seed_environment_name(
environment_name,
)
if seed_deployment_name:
seed_config = configuration_manager.get_environment_config(
seed_deployment_name,
)
seed_deployment = Deployment(
seed_deployment_name,
seed_config.get('ec2', {}),
seed_config.get('rds', {}),
seed_config.get('elb', {}),
)
logger.info("Verifying seed deployment state")
seed_deployment.verify_deployment_state(verify_old=False)
# Build all of the deployment objects
logger.info("Building Node deployers")
with logs_duration(timer, timer_name='build deployers'):
ec2_deployers = []
rds_deployers = []
# All rds and ec2 nodes, rds nodes first
dep_confs = [
(
'rds',
environment_config.get('ec2', {}),
),
(
'ec2',
environment_config.get('rds', {}),
),
]
for aws_type, node_confs in dep_confs:
for node_name, conf in node_confs.items():
# Get the seed deployment new instances will be copied from
seed_node_name = None
if seed_deployment and 'seed' in conf:
seed_node_name = conf['seed']['unique_id']
verify_seed_data = conf['seed_node'].get('verify', False)
else:
logger.info("No seed node configured")
seed_node_name = None
verify_seed_data = False
if aws_type == 'ec2':
klass = Ec2NodeDeployment
elif aws_type == 'rds':
klass = RdsNodeDeployment
deployer = klass(
deployment=deployment,
seed_deployment=seed_deployment,
is_active=env._active_gen,
aws_type=aws_type,
node_name=node_name,
seed_node_name=seed_node_name,
seed_verification=verify_seed_data,
brain_wrinkles=conf.get('brain_wrinkles', {}),
conf=conf,
)
if aws_type == 'ec2':
ec2_deployers.append(deployer)
elif aws_type == 'rds':
rds_deployers.append(deployer)
# We don't actually want to do deployments until we have tests
assert False
# Provision the RDS nodes
with logs_duration(timer, timer_name='initial provision'):
logger.info("Provisioning RDS nodes")
for deployer in rds_deployers:
if deployer.seed_verification and deployer.get_node() is None:
_prompt_for_seed_verification(deployer)
deployer.ensure_node_created()
# Provision the EC2 nodes
logger.info("Provisioning EC2 nodes")
for deployer in ec2_deployers:
if deployer.seed_verification and deployer.get_node() is None:
_prompt_for_seed_verification(deployer)
deployer.ensure_node_created()
# Configure the RDS nodes
logger.info("Configuring RDS nodes")
with logs_duration(timer, timer_name='deploy rds'):
for deployer in rds_deployers:
deployer.run()
logger.info("Determining EC2 node deploy priority")
ec2_deployers = _order_ec2_deployers_by_priority(ec2_deployers)
# Configure the EC2 nodes
logger.info("Deploying to EC2 nodes")
for deployer in ec2_deployers:
timer_name = '%s deploy' % deployer.node_name
with logs_duration(timer, timer_name='full %s' % timer_name):
node = deployer.get_node()
with seamless_modification(
node,
deployer.deployment,
force_seamless=env._active_gen,
make_operational_if_not_already=make_operational,
):
pre_deploy_time = datetime.now()
with logs_duration(
timer,
timer_name=timer_name,
output_result=True,
):
deployer.run()
if DT_NOTIFY:
_send_deployment_done_desktop_notification(
pre_deploy_time,
deployer,
)
_announce_deployment()
time_logger.info("Timing Breakdown:")
sorted_timers = sorted(
timer.items(),
key=lambda x: x[1],
reverse=True,
)
for timer_name, duration in sorted_timers:
time_logger.info("%02ds- %s", duration, timer_name)
def override():
"""
Manually fix the generational config for the given generation.
This is required for initial setup of the generational system. We are only
modifying the simpledb records of the instances, not the instances
themselves.
"""
require('_deployment_name')
require('_deployment_confs')
generation_target = _get_gen_target()
deployment = Deployment(
env._deployment_name,
env._deployment_confs['ec2'],
env._deployment_confs['rds'],
env._deployment_confs['elb'],
)
deployment.verify_deployment_state()
if generation_target not in ['ACTIVE', 'PENDING']:
exit(1)
opts = ['Y', 'N']
for aws_type, confs in env._deployment_confs.items():
for node_name, node_confs in confs.items():
if generation_target == 'ACTIVE':
node = deployment.get_active_node(aws_type, node_name)
else:
node = deployment.get_pending_node(aws_type, node_name)
if node:
print "Existing node found for %s: %s\n" % (node_name, node)
replace_node = ''
while replace_node not in opts:
replace_node = prompt("Change this node? (Y/N)")
if replace_node == 'N':
continue
else:
print "No node for %s: %s\n" % (aws_type, node_name)
retire_alter_opts = ['Retire', 'Alter']
retire_alter_response = ''
should_alter_node = False
should_retire_node = False
while retire_alter_response not in retire_alter_opts:
retire_alter_response = prompt(
"Retire or Alter node? (Retire/Alter)")
if retire_alter_response == 'Retire':
should_retire_node = True
else:
should_alter_node = True
if should_alter_node:
# Prompt if the node doesn't already exist
if not node:
add_node = ''
while add_node not in opts:
add_node = prompt(
'No node record found for <%s>-%s. Add one? '
'(Y/N)' % (aws_type, node_name)
)
if add_node == 'N':
should_alter_node = False
if should_retire_node and not node:
logger.critical(
"No node record found. Can't retire a non-existent node.")
continue
if should_alter_node:
_override_node(
node, deployment, aws_type, node_name)
elif should_retire_node:
logger.info("Retiring: %s", node)
confirm = ''
while confirm not in opts:
confirm = prompt(
"Are you sure you want to RETIRE this node? (Y/N)")
if confirm == 'Y':
node.make_fully_inoperative()
node.retire()
def terminate(soft=None):
require('_deployment_name')
require('_deployment_confs')
while soft not in ['H', 'S']:
soft = prompt("Hard (permanent) or soft termination? (H/S)")
soft_terminate = bool(soft == 'S')
generation_target = _get_gen_target()
if generation_target == 'ACTIVE':
logger.critical("Can't terminate active generation")
exit(1)
if soft_terminate:
logger.info("SOFT terminating %s nodes." % generation_target)
logger.info("They will be removed from operation, but not terminated")
else:
logger.info("HARD terminating %s nodes." % generation_target)
logger.info("They will be TERMINATED. This is not reversible")
deployment = Deployment(
env._deployment_name,
env._deployment_confs['ec2'],
env._deployment_confs['rds'],
env._deployment_confs['elb'],
)
if generation_target == 'PENDING':
possible_nodes = deployment.get_all_pending_nodes(is_running=1)
else:
# OLD
possible_nodes = deployment.get_all_old_nodes(is_running=1)
# Filter out the nodes whose termination isn't yet known
# This is an optimization versus just calling `verify_deployment_state`
# directly, since we need the nodes afterwards anyway.
logger.info("Verifying run statuses")
for node in possible_nodes:
node.verify_running_state()
running_nodes = [node for node in possible_nodes if node.is_running]
if not running_nodes:
logger.info(
"No running nodes exist for generation: %s",
generation_target,
)
return
# Print the nodes we're going to terminate
for node in running_nodes:
logger.info("Terminating: %s", node)
confirm = ''
while not confirm in ['Y', 'N']:
confirm = prompt(
"Are you sure you want to TERMINATE these nodes? (Y/N)")
if confirm == 'N':
exit(1)
for node in running_nodes:
node.make_fully_inoperative()
if soft_terminate:
_disable_newrelic_monitoring(node)
else:
node.terminate()
