def upgrade_kraken(kraken_wait=True, up_confs=True, supervision=False):
"""Upgrade and restart all kraken instances"""
if supervision:
supervision_downtime(step='kraken')
kraken_wait = get_bool_from_cli(kraken_wait)
execute(kraken.upgrade_engine_packages)
execute(kraken.upgrade_monitor_kraken_packages)
if up_confs:
execute(kraken.update_monitor_configuration)
for instance in env.instances.values():
execute(kraken.update_eng_instance_conf, instance)
execute(kraken.restart_all_krakens, wait=kraken_wait)
def upgrade_kraken(wait='serial', up_confs=True, supervision=False):
"""Upgrade and restart all kraken instances"""
if supervision:
supervision_downtime(step='kraken')
execute(kraken.upgrade_engine_packages)
execute(kraken.upgrade_monitor_kraken_packages)
for instance in env.instances.values():
execute(kraken.set_kraken_binary, instance)
if up_confs:
execute(kraken.update_monitor_configuration)
for instance in env.instances.values():
execute(kraken.update_eng_instance_conf, instance)
execute(kraken.restart_all_krakens, wait=wait)
def launch_rebinarization_upgrade(pilot_supervision=True, pilot_tyr_beat=True, instances=None):
"""launch binarization on all instances for the upgrade"""
if pilot_supervision:
supervision_downtime(step='tyr_beat')
supervision_downtime(step='bina')
# avoid any other normal binarization during upgrade
if pilot_tyr_beat:
stop_tyr_beat()
# for each instance:
# - upgrade the ed database
# - binarize last processed data, blocking step (sync, which is good to
# ensure the binarization of this instance is done)
# - if kraken farm, disable the first engine kraken instance in lb; if
# standalone server nothing to do
# - restart corresponding kraken instance even if no data (to have the new
# binary)
# - test kraken instance via monitor-kraken
# - only continue to next instance if instance is ok, to avoid breaking all
# instances
try:
if instances is None:
execute(get_no_data_instances)
instances2process = set(env.instances)
else:
instances2process = set(instances)
def binarize_instance(i_name):
with time_that(blue("data loaded for " + i_name + " in {elapsed}")):
print(blue("loading data for {}".format(i_name)))
update_ed_db(i_name)
if i_name in env.excluded_instances:
print(blue("NOTICE: i_name {} has been excluded, skipping it".format(i_name)))
instances2process.remove(i_name)
else:
if launch_rebinarization(i_name, True):
# do not remove if bina failed
instances2process.remove(i_name)
# print instances not yet binarized, this allows to easily resume the binarization
# process in case of crash or freeze (use include:x,y,z,....)
# see http://jira.canaltp.fr/browse/DEVOP-408
print(blue("Instances left: {}".format(','.join(instances2process))))
# run the bina in parallel (if you want sequential, set env.nb_thread_for_bina = 1)
with Parallel(env.nb_thread_for_bina) as pool:
pool.map(binarize_instance, instances2process)
return tuple(instances2process)
finally:
if pilot_tyr_beat:
start_tyr_beat()
def launch_rebinarization_upgrade(pilot_tyr_beat=True):
"""launch binarization on all instances for the upgrade"""
supervision_downtime(step='tyr_beat')
supervision_downtime(step='bina')
# avoid any other normal binarization during upgrade
if pilot_tyr_beat:
stop_tyr_beat()
# for each instance:
# - upgrade the ed database
# - binarize last processed data, blocking step (sync, which is good to
# ensure the binarization of this instance is done)
# - if kraken farm, disable the first engine kraken instance in lb; if
# standalone server nothing to do
# - restart corresponding kraken instance even if no data (to have the new
# binary)
# - test kraken instance via monitor-kraken
# - only continue to next instance if instance is ok, to avoid breaking all
# instances
try:
execute(get_no_data_instances)
def binarize_instance(i_name):
with time_that(blue("data loaded for " + i_name +
" in {elapsed}")):
print(blue("loading data for {}".format(i_name)))
update_ed_db(i_name)
if i_name in env.excluded_instances:
print(
blue("NOTICE: i_name {} has been excluded, skiping it".
format(i_name)))
else:
launch_rebinarization(i_name, True)
# we run the bina in parallele (if you want sequenciel run, set env.nb_thread_for_bina = 1)
with Parallel(env.nb_thread_for_bina) as pool:
pool.map(binarize_instance, env.instances.keys())
finally:
if pilot_tyr_beat:
start_tyr_beat()
def launch_rebinarization_upgrade(pilot_tyr_beat=True):
"""launch binarization on all instances for the upgrade"""
supervision_downtime(step='tyr_beat')
supervision_downtime(step='bina')
# avoid any other normal binarization during upgrade
if pilot_tyr_beat:
stop_tyr_beat()
# for each instance:
# - upgrade the ed database
# - binarize last processed data, blocking step (sync, which is good to
# ensure the binarization of this instance is done)
# - if kraken farm, disable the first engine kraken instance in lb; if
# standalone server nothing to do
# - restart corresponding kraken instance even if no data (to have the new
# binary)
# - test kraken instance via monitor-kraken
# - only continue to next instance if instance is ok, to avoid breaking all
# instances
try:
execute(get_no_data_instances)
def binarize_instance(i_name):
with utils.time_that(blue("data loaded for " + i_name + " in {elapsed}")):
print(blue("loading data for {}".format(i_name)))
update_ed_db(i_name)
if i_name in env.excluded_instances:
print(blue("NOTICE: i_name {} has been excluded, skiping it".format(i_name)))
else:
launch_rebinarization(i_name, True)
# we run the bina in parallele (if you want sequenciel run, set env.nb_thread_for_bina = 1)
with utils.Parallel(env.nb_thread_for_bina) as pool:
pool.map(binarize_instance, env.instances.keys())
finally:
if pilot_tyr_beat:
start_tyr_beat()
def launch_rebinarization_upgrade(pilot_supervision=True,
pilot_tyr_beat=True,
instances=None):
"""launch binarization on all instances for the upgrade"""
if pilot_supervision:
supervision_downtime(step='tyr_beat')
supervision_downtime(step='bina')
# avoid any other normal binarization during upgrade
if pilot_tyr_beat:
stop_tyr_beat()
# for each instance:
# - upgrade the ed database
# - binarize last processed data, blocking step (sync, which is good to
# ensure the binarization of this instance is done)
# - if kraken farm, disable the first engine kraken instance in lb; if
# standalone server nothing to do
# - restart corresponding kraken instance even if no data (to have the new
# binary)
# - test kraken instance via monitor-kraken
# - only continue to next instance if instance is ok, to avoid breaking all
# instances
try:
if instances is None:
execute(component.kraken.get_no_data_instances)
instances2process = set(env.instances)
else:
instances2process = set(instances)
def binarize_instance(i_name):
with time_that(
blue("Binarization of " + i_name +
" completed in {elapsed}")):
if i_name in env.excluded_instances:
print(
blue(
"NOTICE: instance {} has been excluded, skipping it"
.format(i_name)))
instances2process.remove(i_name)
elif launch_rebinarization(i_name, True):
# remove instance only if bina succeeds
instances2process.remove(i_name)
# print instances not yet binarized, this allows to easily resume the binarization
# process in case of crash or freeze (use include:x,y,z,....)
# see http://jira.canaltp.fr/browse/DEVOP-408
print(
blue("Instances left: {}".format(','.join(instances2process))))
def bina_watchdog():
frozen_bina_count = 0
step_count = 0
while run_watchdog:
time.sleep(10)
if step_count >= 6: # every minute
step_count = 0
else:
step_count += 1
continue
if get_role_processes():
frozen_bina_count = 0
else:
frozen_bina_count += 1
if frozen_bina_count >= 3:
print(
red("\n ERROR: binarisation process is frozen."))
if raw_input(
yellow("Aborting ? (y/N)")).lower()[0] == 'y':
abort(red("aborted"))
break
for instance in instances2process:
update_ed_db(instance)
run_watchdog = True
with watchdog_manager(bina_watchdog):
# run the bina in parallel (if you want sequential, set env.nb_thread_for_bina = 1)
with Parallel(env.nb_thread_for_bina) as pool:
pool.map(binarize_instance, instances2process)
run_watchdog = False
return tuple(instances2process)
finally:
if pilot_tyr_beat:
start_tyr_beat()
def launch_rebinarization_upgrade(pilot_supervision=True, pilot_tyr_beat=True, instances=None):
"""launch binarization on all instances for the upgrade"""
if pilot_supervision:
supervision_downtime(step='tyr_beat')
supervision_downtime(step='bina')
# avoid any other normal binarization during upgrade
if pilot_tyr_beat:
stop_tyr_beat()
# for each instance:
# - upgrade the ed database
# - binarize last processed data, blocking step (sync, which is good to
# ensure the binarization of this instance is done)
# - if kraken farm, disable the first engine kraken instance in lb; if
# standalone server nothing to do
# - restart corresponding kraken instance even if no data (to have the new
# binary)
# - test kraken instance via monitor-kraken
# - only continue to next instance if instance is ok, to avoid breaking all
# instances
try:
if instances is None:
execute(component.kraken.get_no_data_instances)
instances2process = set(env.instances)
else:
instances2process = set(instances)
def binarize_instance(i_name):
with time_that(blue("Binarization of " + i_name + " completed in {elapsed}")):
if i_name in env.excluded_instances:
print(blue("NOTICE: instance {} has been excluded, skipping it".format(i_name)))
instances2process.remove(i_name)
elif launch_rebinarization(i_name, True):
# remove instance only if bina succeeds
instances2process.remove(i_name)
# print instances not yet binarized, this allows to easily resume the binarization
# process in case of crash or freeze (use include:x,y,z,....)
# see http://jira.canaltp.fr/browse/DEVOP-408
print(blue("Instances left: {}".format(','.join(instances2process))))
def bina_watchdog():
frozen_bina_count = 0
step_count = 0
while run_watchdog:
time.sleep(10)
if step_count >= 6: # every minute
step_count = 0
else:
step_count += 1
continue
if get_role_processes():
frozen_bina_count = 0
else:
frozen_bina_count += 1
if frozen_bina_count >= 3:
print(red("\n ERROR: binarisation process is frozen."))
if raw_input(yellow("Aborting ? (y/N)")).lower()[0] == 'y':
abort(red("aborted"))
break
for instance in instances2process:
update_ed_db(instance)
run_watchdog = True
with watchdog_manager(bina_watchdog):
# run the bina in parallel (if you want sequential, set env.nb_thread_for_bina = 1)
with Parallel(env.nb_thread_for_bina) as pool:
pool.map(binarize_instance, instances2process)
run_watchdog = False
return tuple(instances2process)
finally:
if pilot_tyr_beat:
start_tyr_beat()