def restart_kraken_on_host(instance, host):
""" Restart a kraken of an instance on a given server
"""
instance = get_real_instance(instance)
with settings(host_string=host):
kraken = 'kraken_' + instance.name
start_or_stop_with_delay(kraken, 4000, 500, start=False, only_once=True)
start_or_stop_with_delay(kraken, 4000, 500, only_once=env.KRAKEN_START_ONLY_ONCE)
def require_kraken_started(instance):
"""start a kraken instance on all servers if it is not already started
"""
instance = get_real_instance(instance)
kraken = 'kraken_' + instance.name
for host in instance.kraken_engines:
with settings(host_string=host):
start_or_stop_with_delay(kraken, 4000, 500, only_once=True)
def stop_kraken(instance):
"""Stop a kraken instance on all servers
"""
instance = get_real_instance(instance)
kraken = 'kraken_' + instance.name
for host in instance.kraken_engines:
with settings(host_string=host):
start_or_stop_with_delay(kraken, 4000, 500, start=False, only_once=True)
def restart_kraken_on_host(instance, host):
""" Restart a kraken of an instance on a given server
"""
instance = get_real_instance(instance)
with settings(host_string=host):
kraken = 'kraken_' + instance.name
start_or_stop_with_delay(kraken, 4000, 500, start=False, only_once=True)
start_or_stop_with_delay(kraken, 4000, 500, only_once=env.KRAKEN_START_ONLY_ONCE)
def stop_kraken(instance):
"""Stop a kraken instance on all servers
"""
instance = get_real_instance(instance)
kraken = 'kraken_' + instance.name
for host in instance.kraken_engines:
with settings(host_string=host):
start_or_stop_with_delay(kraken, 4000, 500, start=False, only_once=True)
def require_kraken_started(instance):
"""start a kraken instance on all servers if it is not already started
"""
instance = get_real_instance(instance)
kraken = 'kraken_' + instance.name
for host in instance.kraken_engines:
with settings(host_string=host):
start_or_stop_with_delay(kraken, 4000, 500, only_once=True)
def restart_all_krakens(wait=True):
"""restart and test all kraken instances"""
wait = get_bool_from_cli(wait)
start_or_stop_with_delay('apache2',
env.APACHE_START_DELAY * 1000,
500,
only_once=env.APACHE_START_ONLY_ONCE)
for instance in env.instances.values():
restart_kraken(instance.name, wait=wait)
def test_and_launch(ps_out, required, host=None, service=None, delay=30, retry=2):
# TODO refactor to overcome the SSH problem with respect to "service start"
# see: https://github.com/fabric/fabric/issues/395
if check_contains(ps_out, required):
return
if service:
while retry:
with context_managers.settings(host_string=host):
utils.start_or_stop_with_delay(service, delay * 1000, 1000, only_once=True)
retry -= 1
else:
assert False, str(required)
def restart_kraken(instance, test=True, wait=True):
"""Restart a kraken instance on a given server
To let us not restart all kraken servers in the farm
"""
instance = get_real_instance(instance)
wait = get_bool_from_cli(wait)
if instance.name not in env.excluded_instances:
kraken = 'kraken_' + instance.name
start_or_stop_with_delay(kraken, 4000, 500, start=False, only_once=True)
start_or_stop_with_delay(kraken, 4000, 500, only_once=env.KRAKEN_START_ONLY_ONCE)
if test:
test_kraken(instance.name, fail_if_error=False, wait=wait)
else:
print(yellow("{} has no data, not testing it".format(instance.name)))
def stop_tyr_worker():
if not start_or_stop_with_delay(
'tyr_worker', delay=8000, wait=500, start=False, exc_raise=False):
print(red("there are still tyr_worker alive, something is wrong"))
if env.kill_ghost_tyr_worker:
print(red('killing all workers'))
def get_workers():
with warn_only():
return run('ps -eo pid,command | grep [t]yr_worker')
pids_to_kill = [
s.split(' ', 1)[0] for s in get_workers().split('\n')
]
sudo('kill -9 {pid}'.format(pid=" ".join(pids_to_kill)))
try:
Retrying(stop_max_delay=4000,
wait_fixed=1000,
retry_on_result=get_workers).call(lambda: None)
except RetryError:
print red('Some workers are still alive: {}'.format(
get_workers()))
print red("Aborting")
exit(1)
def start_tyr_worker():
if not start_or_stop_with_delay('tyr_worker',
env.TYR_WORKER_START_DELAY * 1000,
500,
exc_raise=False,
only_once=env.TYR_START_ONLY_ONCE):
print(red('Service tyr refuses to start!'))
exit(1)
def test_and_launch(ps_out,
required,
host=None,
service=None,
delay=30,
retry=2):
# TODO refactor to overcome the SSH problem with respect to "service start"
# see: https://github.com/fabric/fabric/issues/395
if check_contains(ps_out, required):
return
if service:
while retry:
with context_managers.settings(host_string=host):
utils.start_or_stop_with_delay(service,
delay * 1000,
1000,
only_once=True)
retry -= 1
else:
assert False, str(required)
def restart_kraken(instance, test=True, wait=True):
"""Restart a kraken instance on a given server
To let us not restart all kraken servers in the farm
"""
instance = get_real_instance(instance)
wait = get_bool_from_cli(wait)
if instance.name not in env.excluded_instances:
kraken = 'kraken_' + instance.name
start_or_stop_with_delay(kraken,
4000,
500,
start=False,
only_once=True)
start_or_stop_with_delay(kraken,
4000,
500,
only_once=env.KRAKEN_START_ONLY_ONCE)
if test:
test_kraken(instance.name, fail_if_error=False, wait=wait)
else:
print(yellow("{} has no data, not testing it".format(instance.name)))
def stop_tyr_worker():
if not start_or_stop_with_delay('tyr_worker', delay=8000, wait=500, start=False, exc_raise=False):
print(red("there are still tyr_worker alive, something is wrong"))
if env.kill_ghost_tyr_worker:
print(red('killing all workers'))
def get_workers():
with warn_only():
return run('ps -eo pid,command | grep [t]yr_worker')
pids_to_kill = [s.split(' ', 1)[0] for s in get_workers().split('\n')]
sudo('kill -9 {pid}'.format(pid=" ".join(pids_to_kill)))
try:
Retrying(stop_max_delay=4000, wait_fixed=1000,
retry_on_result=get_workers).call(lambda: None)
except RetryError:
print red('Some workers are still alive: {}'.format(get_workers()))
print red("Aborting")
exit(1)
def start_tyr_worker():
if not start_or_stop_with_delay('tyr_worker', env.TYR_WORKER_START_DELAY * 1000, 500,
exc_raise=False, only_once=env.TYR_START_ONLY_ONCE):
print(red('Service tyr refuses to start!'))
exit(1)
def require_monitor_kraken_started():
start_or_stop_with_delay('apache2', env.APACHE_START_DELAY * 1000, 500, only_once=env.APACHE_START_ONLY_ONCE)
def stop_kraken(instance):
"""Stop a kraken instance on all servers
"""
instance = get_real_instance(instance)
kraken = 'kraken_' + instance.name
start_or_stop_with_delay(kraken, 4000, 500, start=False, only_once=True)
def require_monitor_kraken_started():
start_or_stop_with_delay('apache2',
env.APACHE_START_DELAY * 1000,
500,
only_once=env.APACHE_START_ONLY_ONCE)
def start_services():
start_or_stop_with_delay('apache2',
env.APACHE_START_DELAY * 1000,
500,
only_once=env.APACHE_START_ONLY_ONCE)
def stop_tyr_beat():
start_or_stop_with_delay('tyr_beat', 4000, 500, start=False, exc_raise=True)
def start_tyr_beat():
start_or_stop_with_delay('tyr_beat',
4000,
500,
exc_raise=True,
only_once=env.TYR_START_ONLY_ONCE)
def start_tyr_beat():
start_or_stop_with_delay('tyr_beat', 4000, 500, exc_raise=True, only_once=env.TYR_START_ONLY_ONCE)
def restart_all_krakens(wait=True):
"""restart and test all kraken instances"""
wait = get_bool_from_cli(wait)
start_or_stop_with_delay('apache2', env.APACHE_START_DELAY * 1000, 500, only_once=env.APACHE_START_ONLY_ONCE)
for instance in env.instances.values():
restart_kraken(instance.name, wait=wait)
def start_services():
start_or_stop_with_delay('apache2', env.APACHE_START_DELAY * 1000, 500, only_once=env.APACHE_START_ONLY_ONCE)
def stop_kraken(instance):
"""Stop a kraken instance on all servers
"""
instance = get_real_instance(instance)
kraken = 'kraken_' + instance.name
start_or_stop_with_delay(kraken, 4000, 500, start=False, only_once=True)
def stop_tyr_beat():
start_or_stop_with_delay('tyr_beat',
4000,
500,
start=False,
exc_raise=True)
