def update_tyr_step(time_dict=None, only_bina=True, up_confs=True, check_bina=False):
# TODO only_bina is highly error prone
""" deploy an upgrade of tyr
"""
if not time_dict:
time_dict = TimeCollector()
execute(tyr.stop_tyr_beat)
execute(upgrade_tyr, up_confs=up_confs, pilot_tyr_beat=False)
time_dict.register_start('bina')
instances_failed = execute(tyr.launch_rebinarization_upgrade, pilot_tyr_beat=False).values()[0]
if check_bina and instances_failed:
if float(len(instances_failed)) / len(env.instances) <= env.acceptable_bina_fail_rate:
print(yellow(" WARNING: {} binarisation(s) have failed, process again".format(len(instances_failed))))
instances_failed = execute(tyr.launch_rebinarization_upgrade,
pilot_supervision=False,
pilot_tyr_beat=False,
instances=instances_failed).values()[0]
else:
print(yellow(" WARNING: Too many ({}) binarisations have failed, do not process again".
format(len(instances_failed))))
time_dict.register_end('bina')
if only_bina:
print show_time_deploy(time_dict)
return
if check_bina and instances_failed:
abort(red("\n ERROR: {} binarisation(s) have failed.".format(len(instances_failed))))
return time_dict
def update_tyr_step(time_dict=None, only_bina=True, up_confs=True, check_bina=False):
# TODO only_bina is highly error prone
""" deploy an upgrade of tyr
"""
if not time_dict:
time_dict = TimeCollector()
execute(tyr.stop_tyr_beat)
execute(upgrade_tyr, up_confs=up_confs, pilot_tyr_beat=False)
time_dict.register_start('bina')
instances_failed = execute(tyr.launch_rebinarization_upgrade, pilot_tyr_beat=False).values()[0]
if check_bina and instances_failed:
if float(len(instances_failed)) / len(env.instances) <= env.acceptable_bina_fail_rate:
print(yellow(" WARNING: {} binarisation(s) have failed, process again".format(len(instances_failed))))
instances_failed = execute(tyr.launch_rebinarization_upgrade,
pilot_supervision=False,
pilot_tyr_beat=False,
instances=instances_failed).values()[0]
else:
print(yellow(" WARNING: Too many ({}) binarisations have failed, do not process again".
format(len(instances_failed))))
time_dict.register_end('bina')
if only_bina:
print show_time_deploy(time_dict)
return
if check_bina and instances_failed:
abort(red("\n ERROR: {} binarisation(s) have failed.".format(len(instances_failed))))
return time_dict
def upgrade_all(up_tyr=True,
up_confs=True,
upgrade_db_tyr=True,
check_version=True,
send_mail='no',
manual_lb=False,
check_dead=True,
check_bina=True,
skip_bina=False):
"""Upgrade all navitia packages, databases and launch rebinarisation of all instances """
up_tyr = get_bool_from_cli(up_tyr)
up_confs = get_bool_from_cli(up_confs)
check_version = get_bool_from_cli(check_version)
check_dead = get_bool_from_cli(check_dead)
check_bina = get_bool_from_cli(check_bina)
upgrade_db_tyr = get_bool_from_cli(upgrade_db_tyr)
skip_bina = get_bool_from_cli(skip_bina)
# check if all krakens are running with data
not_loaded_instances = kraken.get_not_loaded_instances_per_host()
# check one instance on each WS
#TODO: Check all instance not only random one. #pylint: disable=fixme
for server in env.roledefs['ws']:
instance = random.choice(env.instances.values())
execute(jormungandr.test_jormungandr,
get_host_addr(server),
instance=instance.name)
if check_version:
execute(compare_version_candidate_installed, host_name='tyr')
if env.use_load_balancer:
if manual_lb:
print(
yellow(
"WARNING : you are in MANUAL mode :\n"
"Check frequently for message asking you to switch nodes manually"
))
else:
# check credential NOW
_adc_connection(check=True)
execute(check_last_dataset)
if send_mail in ('start', 'all'):
broadcast_email('start')
time_dict = TimeCollector()
time_dict.register_start('total_deploy')
if up_tyr:
execute(update_tyr_step,
time_dict,
only_bina=False,
check_bina=check_bina,
upgrade_db_tyr=upgrade_db_tyr,
skip_bina=skip_bina)
if check_version:
execute(compare_version_candidate_installed)
if not skip_bina:
execute(kraken.swap_all_data_nav)
# Upgrade kraken/jormun on first hosts set
if env.eng_hosts_1 and env.ws_hosts_1:
env.roledefs['eng'] = env.eng_hosts_1
env.roledefs['ws'] = env.ws_hosts_1
if manual_lb:
raw_input(
yellow(
"Please disable ENG1,3/WS7-9 and enable ENG2,4/WS10-12"))
else:
execute(switch_to_first_phase, env.eng_haproxy1, env.ws_hosts_1,
env.ws_hosts_2)
execute(upgrade_kraken,
wait=env.KRAKEN_RESTART_SCHEME,
up_confs=up_confs,
supervision=True)
if check_dead:
execute(kraken.check_dead_instances, not_loaded_instances)
execute(upgrade_jormungandr, reload=False, up_confs=up_confs)
# need restart apache without using upgrade_jormungandr task previously
# because that causes a problem in prod
execute(jormungandr.reload_jormun_safe_all, safe=False)
# check one instance on each WS
#TODO: Check all instance not only random one.
for server in env.roledefs['ws']:
instance = random.choice(env.instances.values())
execute(jormungandr.test_jormungandr,
get_host_addr(server),
instance=instance.name)
if env.eng_hosts_2 and env.ws_hosts_2:
# Upgrade kraken/jormun on remaining hosts
env.roledefs['eng'] = env.eng_hosts_2
env.roledefs['ws'] = env.ws_hosts_2
if manual_lb:
raw_input(
yellow(
"Please enable ENG1,3/WS7-9 and disable ENG2,4/WS10-12"))
else:
execute(switch_to_second_phase, env.eng_haproxy1, env.eng_haproxy2,
env.ws_hosts_1, env.ws_hosts_2)
execute(upgrade_jormungandr, reload=False, up_confs=up_confs)
# need restart apache without using upgrade_jormungandr task previously
# because that causes a problem in prod
execute(jormungandr.reload_jormun_safe_all, safe=False)
if manual_lb:
raw_input(yellow("Please enable WS7-12"))
else:
execute(switch_to_third_phase, env.ws_hosts_2)
env.roledefs['ws'] = env.ws_hosts
execute(upgrade_kraken,
wait=env.KRAKEN_RESTART_SCHEME,
up_confs=up_confs)
if check_dead:
execute(kraken.check_dead_instances, not_loaded_instances)
# check second hosts set
for server in env.roledefs['ws']:
instance = random.choice(env.instances.values())
execute(jormungandr.test_jormungandr,
get_host_addr(server),
instance=instance.name)
env.roledefs['eng'] = env.eng_hosts
if not manual_lb:
#execute(enable_nodes, env.eng_hosts)
execute(enable_kraken_haproxy, env.eng_names)
# start tyr_beat even if up_tyr is False
execute(tyr.start_tyr_beat)
time_dict.register_end('total_deploy')
if send_mail in ('end', 'all'):
warn_dict = jormungandr.check_kraken_jormun_after_deploy()
status = show_dead_kraken_status(warn_dict, show=True)
status += show_time_deploy(time_dict)
broadcast_email('end', status)
if env.use_load_balancer and manual_lb:
print(yellow("Please enable ENG1-4/WS7-12"))
def upgrade_all(up_tyr=True, up_confs=True, check_version=True, send_mail='no',
manual_lb=False, check_dead=True, check_bina=True):
"""Upgrade all navitia packages, databases and launch rebinarisation of all instances """
up_tyr = get_bool_from_cli(up_tyr)
up_confs = get_bool_from_cli(up_confs)
check_version = get_bool_from_cli(check_version)
check_dead = get_bool_from_cli(check_dead)
check_bina = get_bool_from_cli(check_bina)
if check_version:
execute(compare_version_candidate_installed, host_name='tyr')
if env.use_load_balancer:
if manual_lb:
print(yellow("WARNING : you are in MANUAL mode :\n"
"Check frequently for message asking you to switch nodes manually"))
else:
# check credential NOW
_adc_connection(check=True)
execute(check_last_dataset)
if send_mail in ('start', 'all'):
broadcast_email('start')
time_dict = TimeCollector()
time_dict.register_start('total_deploy')
if up_tyr:
execute(update_tyr_step, time_dict, only_bina=False, check_bina=check_bina)
if check_version:
execute(compare_version_candidate_installed)
execute(kraken.swap_all_data_nav)
# Upgrade kraken/jormun on first hosts set
if not env.roledefs['eng'] and not env.roledefs['ws']:
env.roledefs['eng'] = env.eng_hosts_1
env.roledefs['ws'] = env.ws_hosts_1
if manual_lb:
raw_input(yellow("Please disable ENG1,3/WS7-9 and enable ENG2,4/WS10-12"))
else:
execute(switch_to_first_phase, env.eng_hosts_1, env.ws_hosts_1, env.ws_hosts_2)
execute(upgrade_kraken, wait=env.KRAKEN_RESTART_SCHEME, up_confs=up_confs, supervision=True)
if check_dead:
execute(kraken.check_dead_instances)
execute(upgrade_jormungandr, reload=False, up_confs=up_confs)
# check first hosts set
for server in env.roledefs['ws']:
instance = random.choice(env.instances.values())
execute(jormungandr.test_jormungandr, get_host_addr(server), instance=instance.name)
if not env.roledefs['eng'] and not env.roledefs['ws']:
# Upgrade kraken/jormun on remaining hosts
env.roledefs['eng'] = env.eng_hosts_2
env.roledefs['ws'] = env.ws_hosts_2
if manual_lb:
raw_input(yellow("Please enable ENG1,3/WS7-9 and disable ENG2,4/WS10-12"))
else:
execute(switch_to_second_phase, env.eng_hosts_1, env.eng_hosts_2,
env.ws_hosts_1, env.ws_hosts_2)
execute(upgrade_jormungandr, reload=False, up_confs=up_confs)
if manual_lb:
raw_input(yellow("Please enable WS7-12"))
else:
execute(switch_to_third_phase, env.ws_hosts_2)
env.roledefs['ws'] = env.ws_hosts
execute(upgrade_kraken, wait=env.KRAKEN_RESTART_SCHEME, up_confs=up_confs)
# check second hosts set
for server in env.roledefs['ws']:
instance = random.choice(env.instances.values())
execute(jormungandr.test_jormungandr, get_host_addr(server), instance=instance.name)
if not manual_lb:
execute(enable_all_nodes, env.eng_hosts, env.ws_hosts_1, env.ws_hosts_2)
env.roledefs['eng'] = env.eng_hosts
# start tyr_beat even if up_tyr is False
execute(tyr.start_tyr_beat)
time_dict.register_end('total_deploy')
if send_mail in ('end', 'all'):
warn_dict = jormungandr.check_kraken_jormun_after_deploy()
status = show_dead_kraken_status(warn_dict, show=True)
status += show_time_deploy(time_dict)
broadcast_email('end', status)
if env.use_load_balancer and manual_lb:
print(yellow("Please enable ENG1-4/WS7-12"))
