def deployment(self, safe_deployment_strategy):
""" Main entry point for Host Deployment Manager process
:type safe_deployment_strategy: string/enum
:return True if operation succeed otherwise an Exception will be raised.
"""
app_name = self._app['name']
app_env = self._app['env']
app_role = self._app['role']
app_region = self._app['region']
app_blue_green, app_color = get_blue_green_from_app(self._app)
# Retrieve autoscaling infos, if any
as_conn = self._cloud_connection.get_connection(app_region, ['autoscaling'], boto_version='boto3')
as_group, as_group_processes_to_suspend = get_autoscaling_group_and_processes_to_suspend(as_conn, self._app,
self._log_file)
try:
# Suspend autoscaling
suspend_autoscaling_group_processes(as_conn, as_group, as_group_processes_to_suspend, self._log_file)
# Wait for pending instances to become ready
while True:
pending_instances = find_ec2_pending_instances(self._cloud_connection, app_name, app_env, app_role,
app_region, as_group, ghost_color=app_color)
if not pending_instances:
break
log(
"INFO: waiting 10s for {} instance(s) to become running before proceeding with deployment: {}".format(
len(pending_instances), pending_instances), self._log_file)
time.sleep(10)
running_instances = find_ec2_running_instances(self._cloud_connection, app_name, app_env, app_role,
app_region, ghost_color=app_color)
if running_instances:
if safe_deployment_strategy and self._safe_infos:
self._as_name = as_group
self._hosts_list = running_instances
return self.safe_manager(safe_deployment_strategy)
else:
self._hosts_list = [host['private_ip_address'] for host in running_instances]
self.trigger_launch(self._hosts_list)
return True
else:
raise GCallException(
"No instance found in region {region} with tags app:{app}, env:{env}, role:{role}{color}".format(
region=app_region,
app=app_name,
env=app_env,
role=app_role,
color=', color:%s' % app_color if app_color else ''))
finally:
resume_autoscaling_group_processes(as_conn, as_group, as_group_processes_to_suspend, self._log_file)
def do_rolling(self, rolling_strategy):
""" Main entry point for Rolling Update process.
:param rolling_strategy string: The type of rolling strategy(1by1-1/3-25%-50%)
:return True if operation succeed otherwise an Exception will be raised.
"""
hosts = split_hosts_list(
self.hosts_list,
rolling_strategy) if rolling_strategy else [self.hosts_list]
for host_group in hosts:
if self.safe_infos['load_balancer_type'] == 'elb':
self.elb_rolling_update(host_group)
# elif self.safe_infos['load_balancer_type'] == 'alb':
else:
raise GCallException(
'Load balancer type not supported for Rolling update option'
)
log('Waiting 10s before going on next instance group',
self.log_file)
time.sleep(10)
return True
def alb_safe_deployment(self, instances_list):
""" Manage the safe deployment process for the Application Load Balancer.
:param instances_list: list: Instances on which to
(list of dict. ex: [{'id':XXX, 'private_ip_address':XXXX}...]).
:return True if operation successed or raise an Exception.
"""
if not self._as_name:
raise GCallException('Cannot continue because there is no AuoScaling Group configured')
app_region = self._app['region']
alb_mgr = load_balancing.get_lb_manager(self._cloud_connection, app_region, load_balancing.LB_TYPE_AWS_ALB)
alb_targets = alb_mgr.get_instances_status_from_autoscale(self._as_name, self._log_file)
if not len(alb_targets):
raise GCallException('Cannot continue because there is no ALB configured in the AutoScaling Group')
elif len([i for i in alb_targets.values() if 'unhealthy' in i.values()]):
raise GCallException('Cannot continue because one or more instances are in the unhealthy state')
else:
alb_mgr.deregister_instances_from_lbs(self._as_name,
[host['id'] for host in instances_list],
self._log_file)
wait_before_deploy = int(alb_mgr.get_lbs_max_connection_draining_value(self._as_name)) + int(
self._safe_infos['wait_before_deploy'])
log('Waiting {0}s: The deregistation delay time plus the custom value set for wait_before_deploy'.format(
wait_before_deploy), self._log_file)
time.sleep(wait_before_deploy)
host_list = [host['private_ip_address'] for host in instances_list]
self.trigger_launch(host_list)
log('Waiting {0}s: The value set for wait_after_deploy'.format(self._safe_infos['wait_after_deploy']),
self._log_file)
time.sleep(int(self._safe_infos['wait_after_deploy']))
alb_mgr.register_instances_from_lbs(self._as_name,
[host['id'] for host in instances_list],
self._log_file)
while len([i for i in alb_mgr.get_instances_status_from_autoscale(self._as_name, self._log_file).values() if
'unhealthy' in i.values()]):
log('Waiting 10s because the instance is unhealthy in the ALB', self._log_file)
time.sleep(10)
log('Instances: {0} have been deployed and are registered in their ALB'.format(
str([host['private_ip_address'] for host in instances_list])), self._log_file)
return True
def haproxy_safe_deployment(self, instances_list):
""" Manage the safe deployment process for the Haproxy.
:param instances_list: list: Instances on which to deploy
(list of dict. ex: [{'id':XXX, 'private_ip_address':XXXX}...]).
:return True if operation successed or raise an Exception.
"""
lb_infos = [host['private_ip_address'] for host in find_ec2_running_instances(self._cloud_connection,
self._safe_infos['app_tag_value'],
self._app['env'], 'loadbalancer',
self._app['region'])]
if lb_infos:
hapi = haproxy.Haproxyapi(lb_infos, self._log_file, self._safe_infos['api_port'])
ha_urls = hapi.get_haproxy_urls()
if not self.haproxy_configuration_validation(hapi, ha_urls, self._safe_infos['ha_backend']):
raise GCallException('Cannot initialize the safe deployment process because there are differences in the Haproxy \
configuration files between the instances: {0}'.format(lb_infos))
if not hapi.change_instance_state('disableserver', self._safe_infos['ha_backend'],
[host['private_ip_address'] for host in instances_list]):
raise GCallException(
'Cannot disable some instances: {0} in {1}. Deployment aborted'.format(instances_list, lb_infos))
log('Waiting {0}s: The value set for wait_before_deploy'.format(self._safe_infos['wait_before_deploy']),
self._log_file)
time.sleep(int(self._safe_infos['wait_before_deploy']))
host_list = [host['private_ip_address'] for host in instances_list]
self.trigger_launch(host_list)
log('Waiting {0}s: The value set for wait_after_deploy'.format(self._safe_infos['wait_after_deploy']),
self._log_file)
time.sleep(int(self._safe_infos['wait_after_deploy']))
if not hapi.change_instance_state('enableserver', self._safe_infos['ha_backend'],
[host['private_ip_address'] for host in instances_list]):
raise GCallException(
'Cannot enabled some instances: {0} in {1}. Deployment aborted'.format(instances_list, lb_infos))
# Add a sleep to let the time to pass the health check process
time.sleep(5)
if not self.haproxy_configuration_validation(hapi, ha_urls, self._safe_infos['ha_backend']):
raise GCallException('Error in the post safe deployment process because there are differences in the Haproxy \
configuration files between the instances: {0}. Instances: {1} have been deployed but not well enabled'.format(
lb_infos, instances_list))
if not hapi.check_all_instances_up(self._safe_infos['ha_backend'], hapi.get_haproxy_conf(ha_urls[0], True)):
raise GCallException(
'Error in the post safe deployment process because some instances are disable or down in the Haproxy: {0}.'.format(
lb_infos, instances_list))
log('Instances: {0} have been deployed and are registered in their Haproxy'.format(str(instances_list)),
self._log_file)
return True
else:
raise GCallException('Cannot continue because no Haproxy found with the parameters: app_tag_value: {0}, app_env: {1}, app_role: loadbalancer,\
app_region: {2}'.format(self._safe_infos['app_tag_value'], self._app['env'],
self._app['region']))
def elb_rolling_update(self, instances_list):
""" Manage the safe destroy process for the ELB.
:param instances_list list: Instances on which to destroy (list of dict. ex: [{'id':XXX, 'private_ip_address':XXXX}...]).
:return True if operation successed or raise an Exception.
"""
if not self.as_name:
raise GCallException(
'Cannot continue because there is no AutoScaling Group configured'
)
app_region = self.app['region']
as_conn = self.cloud_connection.get_connection(app_region,
['autoscaling'],
boto_version='boto3')
lb_mgr = load_balancing.get_lb_manager(self.cloud_connection,
app_region,
load_balancing.LB_TYPE_AWS_CLB)
destroy_asg_policy = ['OldestLaunchConfiguration']
try:
elb_instances = lb_mgr.get_instances_status_from_autoscale(
self.as_name, self.log_file)
asg_infos = get_autoscaling_group_object(as_conn, self.as_name)
if not len(elb_instances):
raise GCallException(
'Cannot continue because there is no ELB configured in the AutoScaling Group'
)
elif len([
i for i in elb_instances.values()
if 'outofservice' in i.values()
]):
raise GCallException(
'Cannot continue because one or more instances are in the out of service state'
)
elif not check_autoscale_instances_lifecycle_state(
asg_infos['Instances']):
raise GCallException(
'Cannot continue because one or more instances are not in InService Lifecycle state'
)
else:
group_size = len(instances_list)
original_termination_policies = asg_infos[
'TerminationPolicies']
log(
_green(
'Suspending "Terminate" process in the AutoScale and provisioning %s instance(s)'
% group_size), self.log_file)
suspend_autoscaling_group_processes(as_conn, self.as_name,
['Terminate'],
self.log_file)
update_auto_scaling_group_attributes(
as_conn, self.as_name, asg_infos['MinSize'],
asg_infos['MaxSize'] + group_size,
asg_infos['DesiredCapacity'] + group_size)
log(
_green(
'Deregister old instances from the Load Balancer (%s)'
% str([host['id'] for host in instances_list])),
self.log_file)
lb_mgr.deregister_instances_from_lbs(
elb_instances.keys(),
[host['id'] for host in instances_list], self.log_file)
wait_con_draining = int(
lb_mgr.get_lbs_max_connection_draining_value(
elb_instances.keys()))
log(
'Waiting {0}s: The connection draining time'.format(
wait_con_draining), self.log_file)
time.sleep(wait_con_draining)
asg_updated_infos = get_autoscaling_group_object(
as_conn, self.as_name)
while len(asg_updated_infos['Instances']
) < asg_updated_infos['DesiredCapacity']:
log(
'Waiting 30s because the instance(s) are not provisioned in the AutoScale',
self.log_file)
time.sleep(30)
asg_updated_infos = get_autoscaling_group_object(
as_conn, self.as_name)
while not check_autoscale_instances_lifecycle_state(
asg_updated_infos['Instances']):
log(
'Waiting 30s because the instance(s) are not in InService state in the AutoScale',
self.log_file)
time.sleep(30)
asg_updated_infos = get_autoscaling_group_object(
as_conn, self.as_name)
while len([
i for i in lb_mgr.get_instances_status_from_autoscale(
self.as_name, self.log_file).values()
if 'outofservice' in i.values()
]):
log(
'Waiting 10s because the instance(s) are not in service in the ELB',
self.log_file)
time.sleep(10)
suspend_autoscaling_group_processes(as_conn, self.as_name,
['Launch', 'Terminate'],
self.log_file)
log(
_green(
'Restore initial AutoScale attributes and destroy old instances for this group (%s)'
% str([host['id'] for host in instances_list])),
self.log_file)
update_auto_scaling_group_attributes(
as_conn, self.as_name, asg_infos['MinSize'],
asg_infos['MaxSize'], asg_infos['DesiredCapacity'],
destroy_asg_policy)
destroy_specific_ec2_instances(self.cloud_connection, self.app,
instances_list, self.log_file)
resume_autoscaling_group_processes(as_conn, self.as_name,
['Terminate'],
self.log_file)
asg_updated_infos = get_autoscaling_group_object(
as_conn, self.as_name)
while len(asg_updated_infos['Instances']
) > asg_updated_infos['DesiredCapacity']:
log(
'Waiting 20s because the old instance(s) are not removed from the AutoScale',
self.log_file)
time.sleep(20)
asg_updated_infos = get_autoscaling_group_object(
as_conn, self.as_name)
update_auto_scaling_group_attributes(
as_conn, self.as_name, asg_infos['MinSize'],
asg_infos['MaxSize'], asg_infos['DesiredCapacity'],
original_termination_policies)
log(
_green(
'%s instance(s) have been re-generated and are registered in their ELB'
% group_size), self.log_file)
return True
except Exception as e:
raise
finally:
resume_autoscaling_group_processes(as_conn, self.as_name,
['Launch', 'Terminate'],
self.log_file)