def alb_safe_deployment(self, instances_list): """ Manage the safe deployment process for the Application Load Balancer. :param instances_list: list: Instances on which to (list of dict. ex: [{'id':XXX, 'private_ip_address':XXXX}...]). :return True if operation successed or raise an Exception. """ if not self._as_name: raise GCallException('Cannot continue because there is no AuoScaling Group configured') app_region = self._app['region'] alb_mgr = load_balancing.get_lb_manager(self._cloud_connection, app_region, load_balancing.LB_TYPE_AWS_ALB) alb_targets = alb_mgr.get_instances_status_from_autoscale(self._as_name, self._log_file) if not len(alb_targets): raise GCallException('Cannot continue because there is no ALB configured in the AutoScaling Group') elif len([i for i in alb_targets.values() if 'unhealthy' in i.values()]): raise GCallException('Cannot continue because one or more instances are in the unhealthy state') else: alb_mgr.deregister_instances_from_lbs(self._as_name, [host['id'] for host in instances_list], self._log_file) wait_before_deploy = int(alb_mgr.get_lbs_max_connection_draining_value(self._as_name)) + int( self._safe_infos['wait_before_deploy']) log('Waiting {0}s: The deregistation delay time plus the custom value set for wait_before_deploy'.format( wait_before_deploy), self._log_file) time.sleep(wait_before_deploy) host_list = [host['private_ip_address'] for host in instances_list] self.trigger_launch(host_list) log('Waiting {0}s: The value set for wait_after_deploy'.format(self._safe_infos['wait_after_deploy']), self._log_file) time.sleep(int(self._safe_infos['wait_after_deploy'])) alb_mgr.register_instances_from_lbs(self._as_name, [host['id'] for host in instances_list], self._log_file) while len([i for i in alb_mgr.get_instances_status_from_autoscale(self._as_name, self._log_file).values() if 'unhealthy' in i.values()]): log('Waiting 10s because the instance is unhealthy in the ALB', self._log_file) time.sleep(10) log('Instances: {0} have been deployed and are registered in their ALB'.format( str([host['private_ip_address'] for host in instances_list])), self._log_file) return True
def execute(self): log(_green("STATE: Started"), self._log_file) online_app, offline_app = get_blue_green_apps(self._app, self._worker._db.apps, self._log_file) if not offline_app: self._worker.update_status( "aborted", message=self._get_notification_message_aborted( self._app, "Blue/green is not enabled on this app or not well configured" )) return running_jobs = get_running_jobs(self._db, online_app['_id'], offline_app['_id'], self._job['_id']) if abort_if_other_bluegreen_job( running_jobs, self._worker, self._get_notification_message_aborted( self._app, "Please wait until the end of the current jobs before triggering a Blue/green operation" ), self._log_file): return # Check ASG if offline_app['autoscale']['name'] and online_app['autoscale']['name']: if not (check_autoscale_exists(self._cloud_connection, offline_app['autoscale']['name'], offline_app['region']) and check_autoscale_exists(self._cloud_connection, online_app['autoscale']['name'], online_app['region'])): self._worker.update_status( "aborted", message=self._get_notification_message_aborted( offline_app, "Not AutoScale group found on the offline app to purge." )) return else: self._worker.update_status( "aborted", message=self._get_notification_message_aborted( offline_app, "Not AutoScale group found on the offline app to purge.")) return # Check if we have two different AS ! if offline_app['autoscale']['name'] == online_app['autoscale']['name']: self._worker.update_status( "aborted", message=self._get_notification_message_aborted( offline_app, "Please set a different AutoScale on green and blue app.")) return # Retrieve autoscaling infos, if any app_region = offline_app['region'] as_conn3 = self._cloud_connection.get_connection(app_region, ['autoscaling'], boto_version='boto3') as_group, as_group_processes_to_suspend = get_autoscaling_group_and_processes_to_suspend( as_conn3, offline_app, self._log_file) suspend_autoscaling_group_processes(as_conn3, as_group, as_group_processes_to_suspend, self._log_file) try: lb_mgr = load_balancing.get_lb_manager( self._cloud_connection, self._app['region'], online_app["safe-deployment"]["load_balancer_type"]) # Check if instances are running if not get_instances_from_autoscaling( offline_app['autoscale']['name'], as_conn3): log( _yellow( " WARNING: Autoscaling Group [{0}] of offline app is empty. " "No running instances to clean detected.".format( offline_app['autoscale']['name'])), self._log_file) temp_elb_names = lb_mgr.list_lbs_from_autoscale( offline_app['autoscale']['name'], self._log_file, {'bluegreen-temporary': 'true'}) if len(temp_elb_names) > 1: self._worker.update_status( "aborted", message=self._get_notification_message_aborted( offline_app, "There are more than one temporary ELB associated to the ASG '{0}' \n" "ELB found: {1}".format( offline_app['autoscale']['name'], str(temp_elb_names)))) return # Detach temp ELB from ASG log( _green( "Detach the current temporary ELB [{0}] from the AutoScale [{1}]" .format(temp_elb_names, offline_app['autoscale']['name'])), self._log_file) lb_mgr.register_lbs_into_autoscale( offline_app['autoscale']['name'], temp_elb_names, None, self._log_file) # Update ASG and kill instances log("Update AutoScale with `0` on mix, max, desired values.", self._log_file) log( _yellow( "Destroy all instances in the AutoScale and all instances matching the `app_id` [{0}]" .format(offline_app['_id'])), self._log_file) flush_instances_update_autoscale(as_conn3, self._cloud_connection, offline_app, self._log_file) # Destroy temp ELB if temp_elb_names: lb_mgr.destroy_lb(temp_elb_names[0], self._log_file) else: log(" INFO: No ELB to destroy", self._log_file) # Update App Autoscale values, next buildimage or updateautoscaling should not set values different from 0 self._update_app_autoscale_options(offline_app, self._log_file) # All good self._worker.update_status( "done", message=self._get_notification_message_done(offline_app)) except GCallException as e: self._worker.update_status( "failed", message=self._get_notification_message_failed( offline_app, str(e))) finally: # Resume autoscaling groups in any case resume_autoscaling_group_processes(as_conn3, as_group, as_group_processes_to_suspend, self._log_file)
def elb_rolling_update(self, instances_list): """ Manage the safe destroy process for the ELB. :param instances_list list: Instances on which to destroy (list of dict. ex: [{'id':XXX, 'private_ip_address':XXXX}...]). :return True if operation successed or raise an Exception. """ if not self.as_name: raise GCallException( 'Cannot continue because there is no AutoScaling Group configured' ) app_region = self.app['region'] as_conn = self.cloud_connection.get_connection(app_region, ['autoscaling'], boto_version='boto3') lb_mgr = load_balancing.get_lb_manager(self.cloud_connection, app_region, load_balancing.LB_TYPE_AWS_CLB) destroy_asg_policy = ['OldestLaunchConfiguration'] try: elb_instances = lb_mgr.get_instances_status_from_autoscale( self.as_name, self.log_file) asg_infos = get_autoscaling_group_object(as_conn, self.as_name) if not len(elb_instances): raise GCallException( 'Cannot continue because there is no ELB configured in the AutoScaling Group' ) elif len([ i for i in elb_instances.values() if 'outofservice' in i.values() ]): raise GCallException( 'Cannot continue because one or more instances are in the out of service state' ) elif not check_autoscale_instances_lifecycle_state( asg_infos['Instances']): raise GCallException( 'Cannot continue because one or more instances are not in InService Lifecycle state' ) else: group_size = len(instances_list) original_termination_policies = asg_infos[ 'TerminationPolicies'] log( _green( 'Suspending "Terminate" process in the AutoScale and provisioning %s instance(s)' % group_size), self.log_file) suspend_autoscaling_group_processes(as_conn, self.as_name, ['Terminate'], self.log_file) update_auto_scaling_group_attributes( as_conn, self.as_name, asg_infos['MinSize'], asg_infos['MaxSize'] + group_size, asg_infos['DesiredCapacity'] + group_size) log( _green( 'Deregister old instances from the Load Balancer (%s)' % str([host['id'] for host in instances_list])), self.log_file) lb_mgr.deregister_instances_from_lbs( elb_instances.keys(), [host['id'] for host in instances_list], self.log_file) wait_con_draining = int( lb_mgr.get_lbs_max_connection_draining_value( elb_instances.keys())) log( 'Waiting {0}s: The connection draining time'.format( wait_con_draining), self.log_file) time.sleep(wait_con_draining) asg_updated_infos = get_autoscaling_group_object( as_conn, self.as_name) while len(asg_updated_infos['Instances'] ) < asg_updated_infos['DesiredCapacity']: log( 'Waiting 30s because the instance(s) are not provisioned in the AutoScale', self.log_file) time.sleep(30) asg_updated_infos = get_autoscaling_group_object( as_conn, self.as_name) while not check_autoscale_instances_lifecycle_state( asg_updated_infos['Instances']): log( 'Waiting 30s because the instance(s) are not in InService state in the AutoScale', self.log_file) time.sleep(30) asg_updated_infos = get_autoscaling_group_object( as_conn, self.as_name) while len([ i for i in lb_mgr.get_instances_status_from_autoscale( self.as_name, self.log_file).values() if 'outofservice' in i.values() ]): log( 'Waiting 10s because the instance(s) are not in service in the ELB', self.log_file) time.sleep(10) suspend_autoscaling_group_processes(as_conn, self.as_name, ['Launch', 'Terminate'], self.log_file) log( _green( 'Restore initial AutoScale attributes and destroy old instances for this group (%s)' % str([host['id'] for host in instances_list])), self.log_file) update_auto_scaling_group_attributes( as_conn, self.as_name, asg_infos['MinSize'], asg_infos['MaxSize'], asg_infos['DesiredCapacity'], destroy_asg_policy) destroy_specific_ec2_instances(self.cloud_connection, self.app, instances_list, self.log_file) resume_autoscaling_group_processes(as_conn, self.as_name, ['Terminate'], self.log_file) asg_updated_infos = get_autoscaling_group_object( as_conn, self.as_name) while len(asg_updated_infos['Instances'] ) > asg_updated_infos['DesiredCapacity']: log( 'Waiting 20s because the old instance(s) are not removed from the AutoScale', self.log_file) time.sleep(20) asg_updated_infos = get_autoscaling_group_object( as_conn, self.as_name) update_auto_scaling_group_attributes( as_conn, self.as_name, asg_infos['MinSize'], asg_infos['MaxSize'], asg_infos['DesiredCapacity'], original_termination_policies) log( _green( '%s instance(s) have been re-generated and are registered in their ELB' % group_size), self.log_file) return True except Exception as e: raise finally: resume_autoscaling_group_processes(as_conn, self.as_name, ['Launch', 'Terminate'], self.log_file)
def execute(self): """Execute all checks and preparations.""" log(_green("STATE: Started"), self._log_file) online_app, offline_app = get_blue_green_apps(self._app, self._db.apps, self._log_file) if not online_app: self._worker.update_status( "aborted", message=self._get_notification_message_aborted( self._app, "Blue/green is not enabled on this app or not well configured" )) return copy_ami_option = (self._job['options'][0] if 'options' in self._job and len(self._job['options']) > 0 else get_blue_green_copy_ami_config(self._config)) copy_ami_option = boolify(copy_ami_option) app_region = self._app['region'] as_conn3 = self._cloud_connection.get_connection(app_region, ['autoscaling'], boto_version='boto3') as_group, as_group_processes_to_suspend = get_autoscaling_group_and_processes_to_suspend( as_conn3, offline_app, self._log_file) suspend_autoscaling_group_processes(as_conn3, as_group, as_group_processes_to_suspend, self._log_file) try: lb_mgr = load_balancing.get_lb_manager( self._cloud_connection, self._app['region'], online_app["safe-deployment"]["load_balancer_type"]) # check if app is online if not online_app: self._worker.update_status( "aborted", message=self._get_notification_message_aborted( self._app, "Blue/green is not enabled on this app or not well configured" )) return running_jobs = get_running_jobs(self._db, online_app['_id'], offline_app['_id'], self._job['_id']) if abort_if_other_bluegreen_job( running_jobs, self._worker, self._get_notification_message_aborted( self._app, "Please wait until the end of the current jobs before triggering a Blue/green operation" ), self._log_file): return # Check if app has up to date AMI if ((not copy_ami_option and 'ami' not in offline_app) or (copy_ami_option and 'ami' not in online_app)): self._worker.update_status( "aborted", message=self._get_notification_message_aborted( offline_app, "Please run `Buildimage` first or use the `copy_ami` option" )) return # Check if app has AS if offline_app['autoscale']['name'] and online_app['autoscale'][ 'name']: if not (check_autoscale_exists( self._cloud_connection, offline_app['autoscale']['name'], offline_app['region']) and check_autoscale_exists( self._cloud_connection, online_app['autoscale']['name'], online_app['region'])): self._worker.update_status( "aborted", message=self._get_notification_message_aborted( offline_app, "Please check that the configured AutoScale on both green and blue app exists." )) return else: self._worker.update_status( "aborted", message=self._get_notification_message_aborted( offline_app, "Please set an AutoScale on both green and blue app.")) return # Check if we have two different AS ! if offline_app['autoscale']['name'] == online_app['autoscale'][ 'name']: self._worker.update_status( "aborted", message=self._get_notification_message_aborted( offline_app, "Please set a different AutoScale on green and blue app." )) return if copy_ami_option: log( "Copy AMI option activated. AMI used by [{0}] will be reused by [{1}]" .format(online_app['autoscale']['name'], offline_app['autoscale']['name']), self._log_file) # Check if modules have been deployed if get_blue_green_config(self._config, 'preparebluegreen', 'module_deploy_required', False): if not check_app_manifest( offline_app, self._config, self._log_file, get_path_from_app_with_color(offline_app)): self._worker.update_status( "aborted", message=self._get_notification_message_aborted( offline_app, "Please deploy your app's modules")) return # Check if instances are already running if get_instances_from_autoscaling(offline_app['autoscale']['name'], as_conn3): self._worker.update_status( "aborted", message=self._get_notification_message_aborted( offline_app, "Autoscaling Group of offline app should be empty.")) return # Get the online ELB online_elbs = lb_mgr.list_lbs_from_autoscale( online_app['autoscale']['name'], self._log_file) if len(online_elbs) == 0: self._worker.update_status( "aborted", message=self._get_notification_message_aborted( offline_app, "Online app AutoScale is not attached to a valid Elastic Load Balancer" )) return # Create the temporary ELB: ghost-bluegreentemp-{original ELB name}, duplicated from the online ELB temp_elb_name, new_elb_dns = (None, None) create_temporary_elb_option = ( self._job['options'][1] if 'options' in self._job and len(self._job['options']) > 1 else get_blue_green_create_temporary_elb_config(self._config)) if boolify(create_temporary_elb_option): online_elb = online_elbs[0] temp_elb_name = "bgtmp-{0}".format( offline_app['_id'])[:31] # ELB name is 32 char long max log( _green( "Creating the temporary ELB [{0}] by copying parameters from [{1}]" .format(temp_elb_name, online_elb)), self._log_file) new_elb_dns = lb_mgr.copy_lb( temp_elb_name, online_elb, { 'app_id': str(offline_app['_id']), 'bluegreen-temporary': 'true' }, self._log_file) # Register the temporary ELB into the AutoScale log( _green("Attaching ELB [{0}] to the AutoScale [{1}]".format( temp_elb_name, offline_app['autoscale']['name'])), self._log_file) lb_mgr.register_lbs_into_autoscale( offline_app['autoscale']['name'], [], [temp_elb_name], self._log_file) offline_app['autoscale']['min'] = online_app['autoscale']['min'] offline_app['autoscale']['max'] = online_app['autoscale']['max'] if copy_ami_option: offline_app['ami'] = online_app['ami'] offline_app['build_infos']['ami_name'] = online_app[ 'build_infos']['ami_name'] log( "Copying AMI [{0}]({1}) into offline app [{2}]".format( offline_app['ami'], offline_app['build_infos']['ami_name'], str(offline_app['_id'])), self._log_file) self._update_app_ami(offline_app) # Update AutoScale properties in DB App self._update_app_autoscale_options(offline_app, online_app, self._log_file) # Update AutoScale properties and starts instances if copy_ami_option: try: if not create_userdata_launchconfig_update_asg( offline_app['ami'], self._cloud_connection, offline_app, self._config, self._log_file, update_as_params=True): self._worker.update_status( "failed", message=self._get_notification_message_failed( online_app, offline_app, "")) return except: traceback.print_exc(self._log_file) self._worker.update_status( "failed", message=self._get_notification_message_failed( online_app, offline_app, "")) return else: update_auto_scale(self._cloud_connection, offline_app, None, self._log_file, update_as_params=True) log( _green( "Starting at least [{0}] instance(s) into the AutoScale [{1}]" .format(offline_app['autoscale']['min'], offline_app['autoscale']['name'])), self._log_file) self._worker.update_status( "done", message=self._get_notification_message_done( offline_app, temp_elb_name, new_elb_dns)) except GCallException as e: self._worker.update_status( "failed", message=self._get_notification_message_failed( online_app, offline_app, e)) finally: resume_autoscaling_group_processes(as_conn3, as_group, as_group_processes_to_suspend, self._log_file)
def execute(self): log(_green("STATE: Started"), self._log_file) swap_execution_strategy = ( self._job['options'][0] if 'options' in self._job and len(self._job['options']) > 0 else "isolated") online_app, to_deploy_app = get_blue_green_apps( self._app, self._worker._db.apps, self._log_file) if not online_app: self._worker.update_status( "aborted", message=self._get_notification_message_aborted( self._app, "Blue/green is not enabled on this app or not well configured" )) return running_jobs = get_running_jobs(self._db, online_app['_id'], to_deploy_app['_id'], self._job['_id']) if abort_if_other_bluegreen_job( running_jobs, self._worker, self._get_notification_message_aborted( self._app, "Please wait until the end of the current jobs before triggering a Blue/green operation" ), self._log_file): return try: lb_mgr = load_balancing.get_lb_manager( self._cloud_connection, self._app['region'], online_app["safe-deployment"]["load_balancer_type"]) # Check AMI if 'ami' not in to_deploy_app: self._worker.update_status( "aborted", message=self._get_notification_message_aborted( to_deploy_app, "Please run `Buildimage` first")) return # Check if modules have been deployed if not check_app_manifest( to_deploy_app, self._config, self._log_file, get_path_from_app_with_color(to_deploy_app)): self._worker.update_status( "aborted", message=self._get_notification_message_aborted( to_deploy_app, "Please deploy your app's modules")) return # Check ASG if to_deploy_app['autoscale']['name'] and online_app['autoscale'][ 'name']: if not (check_autoscale_exists( self._cloud_connection, to_deploy_app['autoscale']['name'], to_deploy_app['region']) and check_autoscale_exists( self._cloud_connection, online_app['autoscale']['name'], online_app['region'])): self._worker.update_status( "aborted", message=self._get_notification_message_aborted( to_deploy_app, "Please set an AutoScale on both green and blue app" )) return else: self._worker.update_status( "aborted", message=self._get_notification_message_aborted( to_deploy_app, "Please set an AutoScale on both green and blue app.")) return # Check if we have two different AS ! if to_deploy_app['autoscale']['name'] == online_app['autoscale'][ 'name']: self._worker.update_status( "aborted", message=self._get_notification_message_aborted( to_deploy_app, "Please set a different AutoScale on green and blue app." )) return # Check if we're ready to swap. If an instance is out of service # into the ELB pool raise an exception elb_instances = lb_mgr.get_instances_status_from_autoscale( to_deploy_app['autoscale']['name'], self._log_file) if len(elb_instances) == 0: self._worker.update_status( "aborted", message=self._get_notification_message_aborted( to_deploy_app, "The offline application [{0}] doesn't have a valid Load Balancer associated.'" .format(to_deploy_app['_id']))) return for e in elb_instances.values(): if len(e.values()) == 0: self._worker.update_status( "aborted", message=self._get_notification_message_aborted( to_deploy_app, "An ELB of the offline application [{0}] has no instances associated.'" .format(to_deploy_app['_id']))) return if len([ i for i in elb_instances.values() if 'outofservice' in i.values() ]): raise GCallException( 'Cannot continue because one or more instances are in the out of service state in the temp ELB' ) else: log( _green( "AutoScale blue [{0}] and green [{1}] ready for swap". format(online_app['autoscale']['name'], to_deploy_app['autoscale']['name'])), self._log_file) self._execute_swap_hook( online_app, to_deploy_app, 'pre_swap', 'Pre swap script for current {status} application', self._log_file) # Swap ! elb_name, elb_dns = self._swap_asg(lb_mgr, swap_execution_strategy, online_app, to_deploy_app, self._log_file) if not elb_name: self._worker.update_status( "failed", message=self._get_notification_message_failed( online_app, to_deploy_app, 'Unable to make blue-green swap')) return self._execute_swap_hook( online_app, to_deploy_app, 'post_swap', 'Post swap script for previously {status} application', self._log_file) # All good done_notif = self._get_notification_message_done( online_app, online_app['autoscale']['name'], to_deploy_app['autoscale']['name'], elb_name, elb_dns) self._worker.update_status("done", message=done_notif) except GCallException as e: self._worker.update_status( "failed", message=self._get_notification_message_failed( online_app, to_deploy_app, str(e)))