def update_tyr_step(time_dict=None, only_bina=True, up_confs=True, check_bina=False): # TODO only_bina is highly error prone """ deploy an upgrade of tyr """ if not time_dict: time_dict = TimeCollector() execute(tyr.stop_tyr_beat) execute(upgrade_tyr, up_confs=up_confs, pilot_tyr_beat=False) time_dict.register_start('bina') instances_failed = execute(tyr.launch_rebinarization_upgrade, pilot_tyr_beat=False).values()[0] if check_bina and instances_failed: if float(len(instances_failed)) / len(env.instances) <= env.acceptable_bina_fail_rate: print(yellow(" WARNING: {} binarisation(s) have failed, process again".format(len(instances_failed)))) instances_failed = execute(tyr.launch_rebinarization_upgrade, pilot_supervision=False, pilot_tyr_beat=False, instances=instances_failed).values()[0] else: print(yellow(" WARNING: Too many ({}) binarisations have failed, do not process again". format(len(instances_failed)))) time_dict.register_end('bina') if only_bina: print show_time_deploy(time_dict) return if check_bina and instances_failed: abort(red("\n ERROR: {} binarisation(s) have failed.".format(len(instances_failed)))) return time_dict
def upgrade_all(up_tyr=True, up_confs=True, upgrade_db_tyr=True, check_version=True, send_mail='no', manual_lb=False, check_dead=True, check_bina=True, skip_bina=False): """Upgrade all navitia packages, databases and launch rebinarisation of all instances """ up_tyr = get_bool_from_cli(up_tyr) up_confs = get_bool_from_cli(up_confs) check_version = get_bool_from_cli(check_version) check_dead = get_bool_from_cli(check_dead) check_bina = get_bool_from_cli(check_bina) upgrade_db_tyr = get_bool_from_cli(upgrade_db_tyr) skip_bina = get_bool_from_cli(skip_bina) # check if all krakens are running with data not_loaded_instances = kraken.get_not_loaded_instances_per_host() # check one instance on each WS #TODO: Check all instance not only random one. #pylint: disable=fixme for server in env.roledefs['ws']: instance = random.choice(env.instances.values()) execute(jormungandr.test_jormungandr, get_host_addr(server), instance=instance.name) if check_version: execute(compare_version_candidate_installed, host_name='tyr') if env.use_load_balancer: if manual_lb: print( yellow( "WARNING : you are in MANUAL mode :\n" "Check frequently for message asking you to switch nodes manually" )) else: # check credential NOW _adc_connection(check=True) execute(check_last_dataset) if send_mail in ('start', 'all'): broadcast_email('start') time_dict = TimeCollector() time_dict.register_start('total_deploy') if up_tyr: execute(update_tyr_step, time_dict, only_bina=False, check_bina=check_bina, upgrade_db_tyr=upgrade_db_tyr, skip_bina=skip_bina) if check_version: execute(compare_version_candidate_installed) if not skip_bina: execute(kraken.swap_all_data_nav) # Upgrade kraken/jormun on first hosts set if env.eng_hosts_1 and env.ws_hosts_1: env.roledefs['eng'] = env.eng_hosts_1 env.roledefs['ws'] = env.ws_hosts_1 if manual_lb: raw_input( yellow( "Please disable ENG1,3/WS7-9 and enable ENG2,4/WS10-12")) else: execute(switch_to_first_phase, env.eng_haproxy1, env.ws_hosts_1, env.ws_hosts_2) execute(upgrade_kraken, wait=env.KRAKEN_RESTART_SCHEME, up_confs=up_confs, supervision=True) if check_dead: execute(kraken.check_dead_instances, not_loaded_instances) execute(upgrade_jormungandr, reload=False, up_confs=up_confs) # need restart apache without using upgrade_jormungandr task previously # because that causes a problem in prod execute(jormungandr.reload_jormun_safe_all, safe=False) # check one instance on each WS #TODO: Check all instance not only random one. for server in env.roledefs['ws']: instance = random.choice(env.instances.values()) execute(jormungandr.test_jormungandr, get_host_addr(server), instance=instance.name) if env.eng_hosts_2 and env.ws_hosts_2: # Upgrade kraken/jormun on remaining hosts env.roledefs['eng'] = env.eng_hosts_2 env.roledefs['ws'] = env.ws_hosts_2 if manual_lb: raw_input( yellow( "Please enable ENG1,3/WS7-9 and disable ENG2,4/WS10-12")) else: execute(switch_to_second_phase, env.eng_haproxy1, env.eng_haproxy2, env.ws_hosts_1, env.ws_hosts_2) execute(upgrade_jormungandr, reload=False, up_confs=up_confs) # need restart apache without using upgrade_jormungandr task previously # because that causes a problem in prod execute(jormungandr.reload_jormun_safe_all, safe=False) if manual_lb: raw_input(yellow("Please enable WS7-12")) else: execute(switch_to_third_phase, env.ws_hosts_2) env.roledefs['ws'] = env.ws_hosts execute(upgrade_kraken, wait=env.KRAKEN_RESTART_SCHEME, up_confs=up_confs) if check_dead: execute(kraken.check_dead_instances, not_loaded_instances) # check second hosts set for server in env.roledefs['ws']: instance = random.choice(env.instances.values()) execute(jormungandr.test_jormungandr, get_host_addr(server), instance=instance.name) env.roledefs['eng'] = env.eng_hosts if not manual_lb: #execute(enable_nodes, env.eng_hosts) execute(enable_kraken_haproxy, env.eng_names) # start tyr_beat even if up_tyr is False execute(tyr.start_tyr_beat) time_dict.register_end('total_deploy') if send_mail in ('end', 'all'): warn_dict = jormungandr.check_kraken_jormun_after_deploy() status = show_dead_kraken_status(warn_dict, show=True) status += show_time_deploy(time_dict) broadcast_email('end', status) if env.use_load_balancer and manual_lb: print(yellow("Please enable ENG1-4/WS7-12"))
def upgrade_all(up_tyr=True, up_confs=True, check_version=True, send_mail='no', manual_lb=False, check_dead=True, check_bina=True): """Upgrade all navitia packages, databases and launch rebinarisation of all instances """ up_tyr = get_bool_from_cli(up_tyr) up_confs = get_bool_from_cli(up_confs) check_version = get_bool_from_cli(check_version) check_dead = get_bool_from_cli(check_dead) check_bina = get_bool_from_cli(check_bina) if check_version: execute(compare_version_candidate_installed, host_name='tyr') if env.use_load_balancer: if manual_lb: print(yellow("WARNING : you are in MANUAL mode :\n" "Check frequently for message asking you to switch nodes manually")) else: # check credential NOW _adc_connection(check=True) execute(check_last_dataset) if send_mail in ('start', 'all'): broadcast_email('start') time_dict = TimeCollector() time_dict.register_start('total_deploy') if up_tyr: execute(update_tyr_step, time_dict, only_bina=False, check_bina=check_bina) if check_version: execute(compare_version_candidate_installed) execute(kraken.swap_all_data_nav) # Upgrade kraken/jormun on first hosts set if not env.roledefs['eng'] and not env.roledefs['ws']: env.roledefs['eng'] = env.eng_hosts_1 env.roledefs['ws'] = env.ws_hosts_1 if manual_lb: raw_input(yellow("Please disable ENG1,3/WS7-9 and enable ENG2,4/WS10-12")) else: execute(switch_to_first_phase, env.eng_hosts_1, env.ws_hosts_1, env.ws_hosts_2) execute(upgrade_kraken, wait=env.KRAKEN_RESTART_SCHEME, up_confs=up_confs, supervision=True) if check_dead: execute(kraken.check_dead_instances) execute(upgrade_jormungandr, reload=False, up_confs=up_confs) # check first hosts set for server in env.roledefs['ws']: instance = random.choice(env.instances.values()) execute(jormungandr.test_jormungandr, get_host_addr(server), instance=instance.name) if not env.roledefs['eng'] and not env.roledefs['ws']: # Upgrade kraken/jormun on remaining hosts env.roledefs['eng'] = env.eng_hosts_2 env.roledefs['ws'] = env.ws_hosts_2 if manual_lb: raw_input(yellow("Please enable ENG1,3/WS7-9 and disable ENG2,4/WS10-12")) else: execute(switch_to_second_phase, env.eng_hosts_1, env.eng_hosts_2, env.ws_hosts_1, env.ws_hosts_2) execute(upgrade_jormungandr, reload=False, up_confs=up_confs) if manual_lb: raw_input(yellow("Please enable WS7-12")) else: execute(switch_to_third_phase, env.ws_hosts_2) env.roledefs['ws'] = env.ws_hosts execute(upgrade_kraken, wait=env.KRAKEN_RESTART_SCHEME, up_confs=up_confs) # check second hosts set for server in env.roledefs['ws']: instance = random.choice(env.instances.values()) execute(jormungandr.test_jormungandr, get_host_addr(server), instance=instance.name) if not manual_lb: execute(enable_all_nodes, env.eng_hosts, env.ws_hosts_1, env.ws_hosts_2) env.roledefs['eng'] = env.eng_hosts # start tyr_beat even if up_tyr is False execute(tyr.start_tyr_beat) time_dict.register_end('total_deploy') if send_mail in ('end', 'all'): warn_dict = jormungandr.check_kraken_jormun_after_deploy() status = show_dead_kraken_status(warn_dict, show=True) status += show_time_deploy(time_dict) broadcast_email('end', status) if env.use_load_balancer and manual_lb: print(yellow("Please enable ENG1-4/WS7-12"))