def restart_kraken_on_host(instance, host): """ Restart a kraken of an instance on a given server """ instance = get_real_instance(instance) with settings(host_string=host): kraken = 'kraken_' + instance.name start_or_stop_with_delay(kraken, 4000, 500, start=False, only_once=True) start_or_stop_with_delay(kraken, 4000, 500, only_once=env.KRAKEN_START_ONLY_ONCE)
def require_kraken_started(instance): """start a kraken instance on all servers if it is not already started """ instance = get_real_instance(instance) kraken = 'kraken_' + instance.name for host in instance.kraken_engines: with settings(host_string=host): start_or_stop_with_delay(kraken, 4000, 500, only_once=True)
def stop_kraken(instance): """Stop a kraken instance on all servers """ instance = get_real_instance(instance) kraken = 'kraken_' + instance.name for host in instance.kraken_engines: with settings(host_string=host): start_or_stop_with_delay(kraken, 4000, 500, start=False, only_once=True)
def restart_all_krakens(wait=True): """restart and test all kraken instances""" wait = get_bool_from_cli(wait) start_or_stop_with_delay('apache2', env.APACHE_START_DELAY * 1000, 500, only_once=env.APACHE_START_ONLY_ONCE) for instance in env.instances.values(): restart_kraken(instance.name, wait=wait)
def test_and_launch(ps_out, required, host=None, service=None, delay=30, retry=2): # TODO refactor to overcome the SSH problem with respect to "service start" # see: https://github.com/fabric/fabric/issues/395 if check_contains(ps_out, required): return if service: while retry: with context_managers.settings(host_string=host): utils.start_or_stop_with_delay(service, delay * 1000, 1000, only_once=True) retry -= 1 else: assert False, str(required)
def restart_kraken(instance, test=True, wait=True): """Restart a kraken instance on a given server To let us not restart all kraken servers in the farm """ instance = get_real_instance(instance) wait = get_bool_from_cli(wait) if instance.name not in env.excluded_instances: kraken = 'kraken_' + instance.name start_or_stop_with_delay(kraken, 4000, 500, start=False, only_once=True) start_or_stop_with_delay(kraken, 4000, 500, only_once=env.KRAKEN_START_ONLY_ONCE) if test: test_kraken(instance.name, fail_if_error=False, wait=wait) else: print(yellow("{} has no data, not testing it".format(instance.name)))
def stop_tyr_worker(): if not start_or_stop_with_delay( 'tyr_worker', delay=8000, wait=500, start=False, exc_raise=False): print(red("there are still tyr_worker alive, something is wrong")) if env.kill_ghost_tyr_worker: print(red('killing all workers')) def get_workers(): with warn_only(): return run('ps -eo pid,command | grep [t]yr_worker') pids_to_kill = [ s.split(' ', 1)[0] for s in get_workers().split('\n') ] sudo('kill -9 {pid}'.format(pid=" ".join(pids_to_kill))) try: Retrying(stop_max_delay=4000, wait_fixed=1000, retry_on_result=get_workers).call(lambda: None) except RetryError: print red('Some workers are still alive: {}'.format( get_workers())) print red("Aborting") exit(1)
def start_tyr_worker(): if not start_or_stop_with_delay('tyr_worker', env.TYR_WORKER_START_DELAY * 1000, 500, exc_raise=False, only_once=env.TYR_START_ONLY_ONCE): print(red('Service tyr refuses to start!')) exit(1)
def stop_tyr_worker(): if not start_or_stop_with_delay('tyr_worker', delay=8000, wait=500, start=False, exc_raise=False): print(red("there are still tyr_worker alive, something is wrong")) if env.kill_ghost_tyr_worker: print(red('killing all workers')) def get_workers(): with warn_only(): return run('ps -eo pid,command | grep [t]yr_worker') pids_to_kill = [s.split(' ', 1)[0] for s in get_workers().split('\n')] sudo('kill -9 {pid}'.format(pid=" ".join(pids_to_kill))) try: Retrying(stop_max_delay=4000, wait_fixed=1000, retry_on_result=get_workers).call(lambda: None) except RetryError: print red('Some workers are still alive: {}'.format(get_workers())) print red("Aborting") exit(1)
def require_monitor_kraken_started(): start_or_stop_with_delay('apache2', env.APACHE_START_DELAY * 1000, 500, only_once=env.APACHE_START_ONLY_ONCE)
def stop_kraken(instance): """Stop a kraken instance on all servers """ instance = get_real_instance(instance) kraken = 'kraken_' + instance.name start_or_stop_with_delay(kraken, 4000, 500, start=False, only_once=True)
def start_services(): start_or_stop_with_delay('apache2', env.APACHE_START_DELAY * 1000, 500, only_once=env.APACHE_START_ONLY_ONCE)
def stop_tyr_beat(): start_or_stop_with_delay('tyr_beat', 4000, 500, start=False, exc_raise=True)
def start_tyr_beat(): start_or_stop_with_delay('tyr_beat', 4000, 500, exc_raise=True, only_once=env.TYR_START_ONLY_ONCE)