def delete_asg():
"""
Method to delete AWS Auto-Scaling Groups via Asgard that are tagged for deletion.
"""
error = False
try:
click.echo("Getting ASGs to delete")
asgs = get_asgs_pending_delete()
click.echo("Got ASGs to delete")
for asg in asgs:
click.echo("Attempting to delete ASG {0}".format(asg.name))
try:
asgard.delete_asg(asg.name, wait_for_deletion=False)
click.secho("Successfully deleted ASG {0}".format(asg.name),
fg='green')
except Exception as e: # pylint: disable=broad-except
click.secho("Unable to delete ASG: {0} - {1}".format(asg, e),
fg='red')
error = True
except Exception as e: # pylint: disable=broad-except
traceback.print_exc()
click.secho("An error occured while cleaning up ASGs: {0}".format(e),
fg='red')
error = True
if error:
sys.exit(1)
else:
sys.exit(0)
def rollback(current_clustered_asgs, rollback_to_clustered_asgs, ami_id=None):
"""
Rollback to a particular list of ASGs for one or more clusters.
If rollback does not succeed, create new ASGs based on the AMI ID and deploy those ASGs.
Arguments:
current_clustered_asgs(dict): ASGs currently enabled, grouped by cluster.
rollback_to_clustered_asgs(dict): ASGs to rollback to, grouped by cluster.
ami_id(str): AWS AMI ID to which to rollback to.
Returns:
dict(str, str, dict): Returns a dictionary with the keys:
'ami_id' - AMI id used to deploy the AMI, None if unspecified
'current_asgs' - Lists of current active ASGs, keyed by cluster.
'disabled_asgs' - Lists of current inactive ASGs, keyed by cluster.
Raises:
TimeoutException: When the task to bring up the new instance times out.
BackendError: When the task to bring up the new instance fails.
ASGDoesNotExistException: If the ASG being queried does not exist.
"""
# First, ensure that the ASGs to which we'll rollback are not tagged for deletion.
# Also, ensure that those same ASGs are not in the process of deletion.
rollback_ready = True
asgs_tagged_for_deletion = [asg.name for asg in ec2.get_asgs_pending_delete()]
for asgs in rollback_to_clustered_asgs.values():
for asg in asgs:
try:
if asg in asgs_tagged_for_deletion:
# ASG is tagged for deletion. Remove the deletion tag.
ec2.remove_asg_deletion_tag(asg)
if is_asg_pending_delete(asg):
# Too late for rollback - this ASG is already pending delete.
LOG.info("Rollback ASG '{}' is pending delete. Aborting rollback to ASGs.".format(asg))
rollback_ready = False
break
except ASGDoesNotExistException:
LOG.info("Rollback ASG '{}' has been removed. Aborting rollback to ASGs.".format(asg))
rollback_ready = False
break
if rollback_ready:
# Perform the rollback.
success, enabled_asgs, disabled_asgs = _red_black_deploy(rollback_to_clustered_asgs, current_clustered_asgs)
if not success:
LOG.info("Rollback failed for cluster(s) {}.".format(current_clustered_asgs.keys()))
else:
LOG.info("Woot! Rollback Done!")
return {'ami_id': ami_id, 'current_asgs': enabled_asgs, 'disabled_asgs': disabled_asgs}
# Rollback failed -or- wasn't attempted. Attempt a deploy.
if ami_id:
LOG.info("Attempting rollback via deploy of AMI {}.".format(ami_id))
return deploy(ami_id)
else:
LOG.info("No AMI id specified - so no deploy occurred during rollback.")
return {'ami_id': None, 'current_asgs': current_clustered_asgs, 'disabled_asgs': rollback_to_clustered_asgs}
def rollback(current_clustered_asgs, rollback_to_clustered_asgs, ami_id=None):
"""
Rollback to a particular list of ASGs for one or more clusters.
If rollback does not succeed, create new ASGs based on the AMI ID and deploy those ASGs.
Arguments:
current_clustered_asgs(dict): ASGs currently enabled, grouped by cluster.
rollback_to_clustered_asgs(dict): ASGs to rollback to, grouped by cluster.
ami_id(str): AWS AMI ID to which to rollback to.
Returns:
dict(str, str, dict): Returns a dictionary with the keys:
'ami_id' - AMI id used to deploy the AMI, None if unspecified
'current_asgs' - Lists of current active ASGs, keyed by cluster.
'disabled_asgs' - Lists of current inactive ASGs, keyed by cluster.
Raises:
TimeoutException: When the task to bring up the new instance times out.
BackendError: When the task to bring up the new instance fails.
ASGDoesNotExistException: If the ASG being queried does not exist.
"""
# First, ensure that the ASGs to which we'll rollback are not tagged for deletion.
# Also, ensure that those same ASGs are not in the process of deletion.
rollback_ready = True
asgs_tagged_for_deletion = [asg.name for asg in ec2.get_asgs_pending_delete()]
for asgs in rollback_to_clustered_asgs.values():
for asg in asgs:
try:
if asg in asgs_tagged_for_deletion:
# ASG is tagged for deletion. Remove the deletion tag.
ec2.remove_asg_deletion_tag(asg)
if is_asg_pending_delete(asg):
# Too late for rollback - this ASG is already pending delete.
LOG.info("Rollback ASG '{}' is pending delete. Aborting rollback to ASGs.".format(asg))
rollback_ready = False
break
except ASGDoesNotExistException:
LOG.info("Rollback ASG '{}' has been removed. Aborting rollback to ASGs.".format(asg))
rollback_ready = False
break
if rollback_ready:
# Perform the rollback.
success, enabled_asgs, disabled_asgs = _red_black_deploy(rollback_to_clustered_asgs, current_clustered_asgs)
if not success:
LOG.info("Rollback failed for cluster(s) {}.".format(current_clustered_asgs.keys()))
else:
LOG.info("Woot! Rollback Done!")
return {'ami_id': ami_id, 'current_asgs': enabled_asgs, 'disabled_asgs': disabled_asgs}
# Rollback failed -or- wasn't attempted. Attempt a deploy.
if ami_id:
LOG.info("Attempting rollback via deploy of AMI {}.".format(ami_id))
return deploy(ami_id)
else:
LOG.info("No AMI id specified - so no deploy occurred during rollback.")
return {'ami_id': None, 'current_asgs': current_clustered_asgs, 'disabled_asgs': rollback_to_clustered_asgs}
def test_get_asgs_pending_delete(self):
asg_name = "test-asg-deletion"
deletion_dttm_str = datetime.datetime.utcnow().isoformat()
create_asg_with_tags(asg_name, {ec2.ASG_DELETE_TAG_KEY: deletion_dttm_str})
asgs = ec2.get_asgs_pending_delete()
self.assertEqual(len(asgs), 1)
asg = asgs.pop()
self.assertEqual(asg.name, asg_name)
self.assertEqual(asg.tags[0].key, ec2.ASG_DELETE_TAG_KEY)
self.assertEqual(asg.tags[0].value, deletion_dttm_str)
def test_get_asgs_pending_delete(self):
asg_name = "test-asg-deletion"
deletion_dttm_str = datetime.datetime.utcnow().isoformat()
create_asg_with_tags(asg_name,
{ec2.ASG_DELETE_TAG_KEY: deletion_dttm_str})
asgs = ec2.get_asgs_pending_delete()
self.assertEqual(len(asgs), 1)
asg = asgs.pop()
self.assertEqual(asg.name, asg_name)
self.assertEqual(asg.tags[0].key, ec2.ASG_DELETE_TAG_KEY)
self.assertEqual(asg.tags[0].value, deletion_dttm_str)
def test_get_asgs_pending_delete_incorrectly_formatted_timestamp(self):
asg_name1 = "test-asg-deletion"
asg_name2 = "test-asg-deletion-bad-timestamp"
deletion_dttm_str1 = datetime.datetime.utcnow().isoformat()
deletion_dttm_str2 = "2016-05-18 18:19:46.144884"
create_asg_with_tags(asg_name1, {ec2.ASG_DELETE_TAG_KEY: deletion_dttm_str1})
create_asg_with_tags(asg_name2, {ec2.ASG_DELETE_TAG_KEY: deletion_dttm_str2})
asgs = ec2.get_asgs_pending_delete()
self.assertEqual(len(asgs), 1)
# boto.ec2.autoscale.group.AutoScalingGroup does not implement __eq__ so we need to iterate the list to see if
# the ASGs we are interested in are members
self.assertEqual(len([asg for asg in asgs if asg.name == asg_name1]), 1)
self.assertEqual(len([asg for asg in asgs if asg.name == asg_name2]), 0)
def test_get_asgs_pending_delete_incorrectly_formatted_timestamp(self):
asg_name1 = "test-asg-deletion"
asg_name2 = "test-asg-deletion-bad-timestamp"
deletion_dttm_str1 = datetime.datetime.utcnow().isoformat()
deletion_dttm_str2 = "2016-05-18 18:19:46.144884"
create_asg_with_tags(asg_name1,
{ec2.ASG_DELETE_TAG_KEY: deletion_dttm_str1})
create_asg_with_tags(asg_name2,
{ec2.ASG_DELETE_TAG_KEY: deletion_dttm_str2})
asgs = ec2.get_asgs_pending_delete()
self.assertEqual(len(asgs), 1)
# boto.ec2.autoscale.group.AutoScalingGroup does not implement __eq__ so we need to iterate the list to see if
# the ASGs we are interested in are members
self.assertEqual(len([asg for asg in asgs if asg.name == asg_name1]),
1)
self.assertEqual(len([asg for asg in asgs if asg.name == asg_name2]),
0)
def delete_asg():
"""
Method to delete AWS Auto-Scaling Groups via Asgard that are tagged for deletion.
"""
error = False
try:
asgs = get_asgs_pending_delete()
for asg in asgs:
try:
asgard.delete_asg(asg.name)
except Exception as e: # pylint: disable=broad-except
click.secho("Unable to delete ASG: {0} - {1}".format(asg, e.message), fg='red')
error = True
except Exception as e: # pylint: disable=broad-except
traceback.print_exc()
click.secho("An error occured while cleaning up ASGs: {0}".format(e.message), fg='red')
error = True
if error:
sys.exit(1)
else:
sys.exit(0)
def delete_asg():
"""
Method to delete AWS Auto-Scaling Groups via Asgard that are tagged for deletion.
"""
error = False
try:
asgs = get_asgs_pending_delete()
for asg in asgs:
try:
asgard.delete_asg(asg.name)
except Exception as e: # pylint: disable=broad-except
click.secho("Unable to delete ASG: {0} - {1}".format(asg, e),
fg='red')
error = True
except Exception as e: # pylint: disable=broad-except
traceback.print_exc()
click.secho("An error occured while cleaning up ASGs: {0}".format(e),
fg='red')
error = True
if error:
sys.exit(1)
else:
sys.exit(0)
