示例#1
0
def upgrade_kraken(kraken_wait=True, up_confs=True, supervision=False):
    """Upgrade and restart all kraken instances"""
    if supervision:
        supervision_downtime(step='kraken')
    kraken_wait = get_bool_from_cli(kraken_wait)
    execute(kraken.upgrade_engine_packages)
    execute(kraken.upgrade_monitor_kraken_packages)
    if up_confs:
        execute(kraken.update_monitor_configuration)
        for instance in env.instances.values():
            execute(kraken.update_eng_instance_conf, instance)
    execute(kraken.restart_all_krakens, wait=kraken_wait)
示例#2
0
def upgrade_kraken(wait='serial', up_confs=True, supervision=False):
    """Upgrade and restart all kraken instances"""
    if supervision:
        supervision_downtime(step='kraken')
    execute(kraken.upgrade_engine_packages)
    execute(kraken.upgrade_monitor_kraken_packages)
    for instance in env.instances.values():
        execute(kraken.set_kraken_binary, instance)
    if up_confs:
        execute(kraken.update_monitor_configuration)
        for instance in env.instances.values():
            execute(kraken.update_eng_instance_conf, instance)
    execute(kraken.restart_all_krakens, wait=wait)
示例#3
0
def launch_rebinarization_upgrade(pilot_supervision=True, pilot_tyr_beat=True, instances=None):
    """launch binarization on all instances for the upgrade"""
    if pilot_supervision:
        supervision_downtime(step='tyr_beat')
        supervision_downtime(step='bina')
    # avoid any other normal binarization during upgrade
    if pilot_tyr_beat:
        stop_tyr_beat()
    # for each instance:
    # - upgrade the ed database
    # - binarize last processed data, blocking step (sync, which is good to
    # ensure the binarization of this instance is done)
    # - if kraken farm, disable the first engine kraken instance in lb; if
    # standalone server nothing to do
    # - restart corresponding kraken instance even if no data (to have the new
    # binary)
    # - test kraken instance via monitor-kraken
    # - only continue to next instance if instance is ok, to avoid breaking all
    # instances
    try:
        if instances is None:
            execute(get_no_data_instances)
            instances2process = set(env.instances)
        else:
            instances2process = set(instances)

        def binarize_instance(i_name):
            with time_that(blue("data loaded for " + i_name + " in {elapsed}")):
                print(blue("loading data for {}".format(i_name)))
                update_ed_db(i_name)

                if i_name in env.excluded_instances:
                    print(blue("NOTICE: i_name {} has been excluded, skipping it".format(i_name)))
                    instances2process.remove(i_name)
                else:
                    if launch_rebinarization(i_name, True):
                        # do not remove if bina failed
                        instances2process.remove(i_name)
            # print instances not yet binarized, this allows to easily resume the binarization
            # process in case of crash or freeze (use include:x,y,z,....)
            # see http://jira.canaltp.fr/browse/DEVOP-408
            print(blue("Instances left: {}".format(','.join(instances2process))))

        # run the bina in parallel (if you want sequential, set env.nb_thread_for_bina = 1)
        with Parallel(env.nb_thread_for_bina) as pool:
            pool.map(binarize_instance, instances2process)
        return tuple(instances2process)
    finally:
        if pilot_tyr_beat:
            start_tyr_beat()
示例#4
0
def launch_rebinarization_upgrade(pilot_tyr_beat=True):
    """launch binarization on all instances for the upgrade"""
    supervision_downtime(step='tyr_beat')
    supervision_downtime(step='bina')
    # avoid any other normal binarization during upgrade
    if pilot_tyr_beat:
        stop_tyr_beat()
    # for each instance:
    # - upgrade the ed database
    # - binarize last processed data, blocking step (sync, which is good to
    # ensure the binarization of this instance is done)
    # - if kraken farm, disable the first engine kraken instance in lb; if
    # standalone server nothing to do
    # - restart corresponding kraken instance even if no data (to have the new
    # binary)
    # - test kraken instance via monitor-kraken
    # - only continue to next instance if instance is ok, to avoid breaking all
    # instances
    try:
        execute(get_no_data_instances)

        def binarize_instance(i_name):
            with time_that(blue("data loaded for " + i_name +
                                " in {elapsed}")):
                print(blue("loading data for {}".format(i_name)))
                update_ed_db(i_name)

                if i_name in env.excluded_instances:
                    print(
                        blue("NOTICE: i_name {} has been excluded, skiping it".
                             format(i_name)))
                else:
                    launch_rebinarization(i_name, True)

            # we run the bina in parallele (if you want sequenciel run, set env.nb_thread_for_bina = 1)

        with Parallel(env.nb_thread_for_bina) as pool:
            pool.map(binarize_instance, env.instances.keys())
    finally:
        if pilot_tyr_beat:
            start_tyr_beat()
示例#5
0
def launch_rebinarization_upgrade(pilot_tyr_beat=True):
    """launch binarization on all instances for the upgrade"""
    supervision_downtime(step='tyr_beat')
    supervision_downtime(step='bina')
    # avoid any other normal binarization during upgrade
    if pilot_tyr_beat:
        stop_tyr_beat()
    # for each instance:
    # - upgrade the ed database
    # - binarize last processed data, blocking step (sync, which is good to
    # ensure the binarization of this instance is done)
    # - if kraken farm, disable the first engine kraken instance in lb; if
    # standalone server nothing to do
    # - restart corresponding kraken instance even if no data (to have the new
    # binary)
    # - test kraken instance via monitor-kraken
    # - only continue to next instance if instance is ok, to avoid breaking all
    # instances
    try:
        execute(get_no_data_instances)

        def binarize_instance(i_name):
            with utils.time_that(blue("data loaded for " + i_name + " in {elapsed}")):
                print(blue("loading data for {}".format(i_name)))
                update_ed_db(i_name)

                if i_name in env.excluded_instances:
                    print(blue("NOTICE: i_name {} has been excluded, skiping it".format(i_name)))
                else:
                    launch_rebinarization(i_name, True)

            # we run the bina in parallele (if you want sequenciel run, set env.nb_thread_for_bina = 1)
        with utils.Parallel(env.nb_thread_for_bina) as pool:
            pool.map(binarize_instance, env.instances.keys())
    finally:
        if pilot_tyr_beat:
            start_tyr_beat()
示例#6
0
def launch_rebinarization_upgrade(pilot_supervision=True,
                                  pilot_tyr_beat=True,
                                  instances=None):
    """launch binarization on all instances for the upgrade"""
    if pilot_supervision:
        supervision_downtime(step='tyr_beat')
        supervision_downtime(step='bina')
    # avoid any other normal binarization during upgrade
    if pilot_tyr_beat:
        stop_tyr_beat()
    # for each instance:
    # - upgrade the ed database
    # - binarize last processed data, blocking step (sync, which is good to
    # ensure the binarization of this instance is done)
    # - if kraken farm, disable the first engine kraken instance in lb; if
    # standalone server nothing to do
    # - restart corresponding kraken instance even if no data (to have the new
    # binary)
    # - test kraken instance via monitor-kraken
    # - only continue to next instance if instance is ok, to avoid breaking all
    # instances
    try:
        if instances is None:
            execute(component.kraken.get_no_data_instances)
            instances2process = set(env.instances)
        else:
            instances2process = set(instances)

        def binarize_instance(i_name):
            with time_that(
                    blue("Binarization of " + i_name +
                         " completed in {elapsed}")):
                if i_name in env.excluded_instances:
                    print(
                        blue(
                            "NOTICE: instance {} has been excluded, skipping it"
                            .format(i_name)))
                    instances2process.remove(i_name)
                elif launch_rebinarization(i_name, True):
                    # remove instance only if bina succeeds
                    instances2process.remove(i_name)
            # print instances not yet binarized, this allows to easily resume the binarization
            # process in case of crash or freeze (use include:x,y,z,....)
            # see http://jira.canaltp.fr/browse/DEVOP-408
            print(
                blue("Instances left: {}".format(','.join(instances2process))))

        def bina_watchdog():
            frozen_bina_count = 0
            step_count = 0
            while run_watchdog:
                time.sleep(10)
                if step_count >= 6:  # every minute
                    step_count = 0
                else:
                    step_count += 1
                    continue
                if get_role_processes():
                    frozen_bina_count = 0
                else:
                    frozen_bina_count += 1
                    if frozen_bina_count >= 3:
                        print(
                            red("\n  ERROR: binarisation process is frozen."))
                        if raw_input(
                                yellow("Aborting ? (y/N)")).lower()[0] == 'y':
                            abort(red("aborted"))
                        break

        for instance in instances2process:
            update_ed_db(instance)
        run_watchdog = True
        with watchdog_manager(bina_watchdog):
            # run the bina in parallel (if you want sequential, set env.nb_thread_for_bina = 1)
            with Parallel(env.nb_thread_for_bina) as pool:
                pool.map(binarize_instance, instances2process)
            run_watchdog = False
        return tuple(instances2process)
    finally:
        if pilot_tyr_beat:
            start_tyr_beat()
示例#7
0
def launch_rebinarization_upgrade(pilot_supervision=True, pilot_tyr_beat=True, instances=None):
    """launch binarization on all instances for the upgrade"""
    if pilot_supervision:
        supervision_downtime(step='tyr_beat')
        supervision_downtime(step='bina')
    # avoid any other normal binarization during upgrade
    if pilot_tyr_beat:
        stop_tyr_beat()
    # for each instance:
    # - upgrade the ed database
    # - binarize last processed data, blocking step (sync, which is good to
    # ensure the binarization of this instance is done)
    # - if kraken farm, disable the first engine kraken instance in lb; if
    # standalone server nothing to do
    # - restart corresponding kraken instance even if no data (to have the new
    # binary)
    # - test kraken instance via monitor-kraken
    # - only continue to next instance if instance is ok, to avoid breaking all
    # instances
    try:
        if instances is None:
            execute(component.kraken.get_no_data_instances)
            instances2process = set(env.instances)
        else:
            instances2process = set(instances)

        def binarize_instance(i_name):
            with time_that(blue("Binarization of " + i_name + " completed in {elapsed}")):
                if i_name in env.excluded_instances:
                    print(blue("NOTICE: instance {} has been excluded, skipping it".format(i_name)))
                    instances2process.remove(i_name)
                elif launch_rebinarization(i_name, True):
                    # remove instance only if bina succeeds
                    instances2process.remove(i_name)
            # print instances not yet binarized, this allows to easily resume the binarization
            # process in case of crash or freeze (use include:x,y,z,....)
            # see http://jira.canaltp.fr/browse/DEVOP-408
            print(blue("Instances left: {}".format(','.join(instances2process))))

        def bina_watchdog():
            frozen_bina_count = 0
            step_count = 0
            while run_watchdog:
                time.sleep(10)
                if step_count >= 6:   # every minute
                    step_count = 0
                else:
                    step_count += 1
                    continue
                if get_role_processes():
                    frozen_bina_count = 0
                else:
                    frozen_bina_count += 1
                    if frozen_bina_count >= 3:
                        print(red("\n  ERROR: binarisation process is frozen."))
                        if raw_input(yellow("Aborting ? (y/N)")).lower()[0] == 'y':
                            abort(red("aborted"))
                        break

        for instance in instances2process:
            update_ed_db(instance)
        run_watchdog = True
        with watchdog_manager(bina_watchdog):
            # run the bina in parallel (if you want sequential, set env.nb_thread_for_bina = 1)
            with Parallel(env.nb_thread_for_bina) as pool:
                pool.map(binarize_instance, instances2process)
            run_watchdog = False
        return tuple(instances2process)
    finally:
        if pilot_tyr_beat:
            start_tyr_beat()