def delete_kraken_queue_to_rabbitmq(instance, apply_on='reverse'):
"""
Remove queue for a kraken
"""
instance = get_real_instance(instance)
if apply_on == 'engines':
hosts, exclude_hosts = instance.kraken_engines, ()
elif apply_on == 'reverse':
hosts, exclude_hosts = env.roledefs['eng'], instance.kraken_engines
elif apply_on == 'all':
hosts, exclude_hosts = env.roledefs['eng'], ()
else:
abort("Bad 'apply_on' parameter value: {}".format(apply_on))
if env.rabbitmq_host_api == 'localhost':
host_string = env.roledefs['tyr_master'][0]
else:
host_string = env.rabbitmq_host_api
for host in set(hosts) - set(exclude_hosts):
with settings(host_string=host_string):
run('curl -i -u {}:{} -XDELETE "http://localhost:{}/api/queues/%2F/kraken_{}_{}_rt"'
.format(env.rabbitmq_user, env.rabbitmq_pass, env.rabbitmq_port_api,
get_host_addr(host).split('.')[0], instance))
run('curl -i -u {}:{} -XDELETE "http://localhost:{}/api/queues/%2F/kraken_{}_{}_task"'
.format(env.rabbitmq_user, env.rabbitmq_pass, env.rabbitmq_port_api,
get_host_addr(host).split('.')[0], instance))
def delete_kraken_queue_to_rabbitmq(instance, apply_on='reverse'):
"""
Remove queue for a kraken
"""
instance = get_real_instance(instance)
if apply_on == 'engines':
hosts, exclude_hosts = instance.kraken_engines, ()
elif apply_on == 'reverse':
hosts, exclude_hosts = env.roledefs['eng'], instance.kraken_engines
elif apply_on == 'all':
hosts, exclude_hosts = env.roledefs['eng'], ()
else:
abort("Bad 'apply_on' parameter value: {}".format(apply_on))
host_string = env.rabbitmq_kraken_host
for host in set(hosts) - set(exclude_hosts):
with settings(host_string=host_string):
run('curl -i -u {}:{} -XDELETE "http://localhost:{}/api/queues/{}/kraken_{}_{}_rt"'
.format(env.rabbitmq_kraken_user, env.rabbitmq_kraken_pass,
env.rabbitmq_port_api, rabbitmq_kraken_vhost,
get_host_addr(host).split('.')[0], instance))
run('curl -i -u {}:{} -XDELETE "http://localhost:{}/api/queues/{}/kraken_{}_{}_task"'
.format(env.rabbitmq_kraken_user, env.rabbitmq_kraken_pass,
env.rabbitmq_port_api, rabbitmq_kraken_vhost,
get_host_addr(host).split('.')[0], instance))
def create_eng_instance(instance):
""" Create a new kraken instance (idempotent)
* Install requirements
* Deploy the binary, the templatized ini configuration in a dedicated
directory with rights to www-data and the logdir
* Deploy initscript and add it to startup
* Start the service
"""
instance = get_real_instance(instance)
for host in instance.kraken_engines:
with settings(host_string=host):
# base_conf
require.files.directory(instance.kraken_basedir,
owner=env.KRAKEN_USER,
group=env.KRAKEN_USER,
use_sudo=True)
# logs
require.files.directory(env.kraken_log_basedir,
owner=env.KRAKEN_USER,
group=env.KRAKEN_USER,
use_sudo=True)
update_eng_instance_conf(instance, host)
# kraken.ini, pid and binary symlink
kraken_bin = "{}/{}/kraken".format(env.kraken_basedir,
instance.name)
if not is_link(kraken_bin):
idempotent_symlink("/usr/bin/kraken",
kraken_bin,
use_sudo=True)
sudo('chown -h {user} {bin}'.format(user=env.KRAKEN_USER,
bin=kraken_bin))
kraken = "kraken_{}".format(instance.name)
if not service.is_running(kraken):
# TODO test this on systemd machines
if env.use_systemd:
sudo("systemctl enable kraken_{}.service".format(
instance.name))
else:
sudo("update-rc.d kraken_{} defaults".format(
instance.name))
print(
blue(
"INFO: kraken {instance} instance is starting on {server}, "
"waiting 5 seconds, we will check if processus is running"
.format(instance=instance.name,
server=get_host_addr(env.host_string))))
service.start(kraken)
run("sleep 5") # we wait a bit for the kraken to pop
with settings(warn_only=True):
run("pgrep --list-name --full /srv/kraken/{}/kraken".format(
instance.name))
print(
blue("INFO: kraken {instance} instance is running on {server}".
format(instance=instance.name,
server=get_host_addr(host))))
def create_jormungandr_instance(instance, cheap_journey=False):
#DEPRECATED
"""Create a jormungandr instance
* Deploy /etc/jormungadr.d/<instance>.ini template
* Reload apache
"""
# get the port config from the kraken engine
config = kraken.get_kraken_config(env.roledefs['eng'][0], instance)
zmq_socket = config.get('GENERAL', 'zmq_socket')
instance_port = env.KRAKEN_RE_PORT.match(zmq_socket)
port = instance_port.group('port')
_upload_template("jormungandr/jormungandr.ini.jinja",
instance.jormungandr_config_file,
context={
'env': env,
'instance': instance,
'socket': "tcp://{}:{}".format(env.jormungandr_instance_socket, port),
'cheap_journey': cheap_journey,
},
use_sudo=True
)
# testing if instance appears in JSON return on URI /v1/coverage on each
# Jormungandr
headers = {'Host': env.jormungandr_url }
server = env.host_string
print("→ server: {}".format(server))
execute(reload_jormun_safe, server)
request_str = 'http://{}/v1/coverage/{}/status'.format(get_host_addr(server), instance)
print("request_string: {}".format(request_str))
try:
response = requests.get(request_str, headers=headers)
except (ConnectionError, HTTPError) as e:
print(red("HTTP Error %s: %s" % (e.code, e.readlines()[0])))
exit(1)
except Exception as e:
print(red("Error when connecting to %s: %s" % (env.jormungandr_url, e)))
exit(1)
try:
result = response.json()
except JSONDecodeError:
print(red("cannot read json response : {}".format(response.text)))
exit(1)
if result['status']['kraken_version']:
print(green("OK: Test {} OK".format(instance)))
# really test the instance, warning, maybe no data so 503 returned
test_jormungandr(get_host_addr(env.host_string), instance, fail_if_error=False)
def create_eng_instance(instance):
"""Create a new kraken instance
* Install requirements (idem potem)
* Deploy the binary, the templatized ini configuration in a dedicated
directory with rights to www-data and the logdir
* Deploy initscript and add it to startup
* Start the service
"""
instance = get_real_instance(instance)
for host in instance.kraken_engines:
with settings(host_string=host):
# base_conf
require.files.directory(instance.kraken_basedir,
owner=env.KRAKEN_USER,
group=env.KRAKEN_USER,
use_sudo=True)
# logs
require.files.directory(env.kraken_log_basedir,
owner=env.KRAKEN_USER,
group=env.KRAKEN_USER,
use_sudo=True)
update_eng_instance_conf(instance)
# kraken.ini, pid and binary symlink
kraken_bin = "{}/{}/kraken".format(env.kraken_basedir,
instance.name)
if not is_link(kraken_bin):
files.symlink("/usr/bin/kraken", kraken_bin, use_sudo=True)
sudo('chown -h {user} {bin}'.format(user=env.KRAKEN_USER,
bin=kraken_bin))
#run("chmod 755 /etc/init.d/kraken_{}".format(instance))
# TODO refactor this and test it on systemd and non-systemd machines
if not env.use_systemd:
sudo("update-rc.d kraken_{} defaults".format(instance.name))
print(
blue(
"INFO: Kraken {instance} instance is starting on {server}, "
"waiting 5 seconds, we will check if processus is running".
format(instance=instance.name,
server=get_host_addr(env.host_string))))
service.start("kraken_{}".format(instance.name))
run("sleep 5") # we wait a bit for the kraken to pop
# test it !
# execute(test_kraken, get_host_addr(env.host_string), instance, fail_if_error=False)
with settings(warn_only=True):
run("pgrep --list-name --full {}".format(instance.name))
print(
blue("INFO: kraken {instance} instance is running on {server}".
format(instance=instance.name,
server=get_host_addr(env.host_string))))
def disable_node(server):
""" Disable F5 ADC node """
node = _get_adc_nodename(get_host_addr(server))
connection = _adc_connection()
print("Disable %s node" % node)
if env.dry_run is False:
connection.LocalLB.NodeAddressV2.set_monitor_state(nodes=[node], states=['STATE_DISABLED'])
connection.LocalLB.NodeAddressV2.set_session_enabled_state(nodes=[node], states=['STATE_DISABLED'])
def disable_node(server):
""" Disable F5 ADC node """
node = _get_adc_nodename(get_host_addr(server))
connection = _adc_connection()
print("Disable %s node" % node)
if env.dry_run is False:
connection.LocalLB.NodeAddressV2.set_monitor_state(nodes=[node], states=['STATE_DISABLED'])
connection.LocalLB.NodeAddressV2.set_session_enabled_state(nodes=[node], states=['STATE_DISABLED'])
def create_eng_instance(instance):
"""Create a new kraken instance
* Install requirements (idem potem)
* Deploy the binary, the templatized ini configuration in a dedicated
directory with rights to www-data and the logdir
* Deploy initscript and add it to startup
* Start the service
"""
instance = get_real_instance(instance)
# base_conf
require.files.directory(instance.kraken_basedir,
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
# logs
require.files.directory(env.kraken_log_basedir,
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
require.files.directory(instance.base_destination_dir,
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
update_eng_instance_conf(instance)
# kraken.ini, pid and binary symlink
if not exists("{}/{}/kraken".format(env.kraken_basedir, instance.name)):
kraken_bin = "{}/{}/kraken".format(env.kraken_basedir, instance.name)
files.symlink("/usr/bin/kraken", kraken_bin, use_sudo=True)
sudo('chown {user} {bin}'.format(user=env.KRAKEN_USER, bin=kraken_bin))
#run("chmod 755 /etc/init.d/kraken_{}".format(instance))
sudo("update-rc.d kraken_{} defaults".format(instance.name))
print(blue("INFO: Kraken {instance} instance is starting on {server}, "
"waiting 5 seconds, we will check if processus is running".format(
instance=instance.name, server=get_host_addr(env.host_string))))
service.start("kraken_{} start".format(instance.name))
run("sleep 5") # we wait a bit for the kraken to pop
# test it !
# execute(test_kraken, get_host_addr(env.host_string), instance, fail_if_error=False)
print("server: {}".format(env.host_string))
run("pgrep --list-name --full {}".format(instance.name))
print(blue("INFO: kraken {instance} instance is running on {server}".
format(instance=instance.name, server=get_host_addr(env.host_string))))
def create_eng_instance(instance):
""" Create a new kraken instance (idempotent)
* Install requirements
* Deploy the binary, the templatized ini configuration in a dedicated
directory with rights to www-data and the logdir
* Deploy initscript and add it to startup
* Start the service
"""
instance = get_real_instance(instance)
for host in instance.kraken_engines:
with settings(host_string=host):
# base_conf
require.files.directory(instance.kraken_basedir,
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
# logs
require.files.directory(env.kraken_log_basedir,
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
update_eng_instance_conf(instance, host)
# kraken.ini, pid and binary symlink
kraken_bin = "{}/{}/kraken".format(env.kraken_basedir, instance.name)
if not is_link(kraken_bin):
idempotent_symlink("/usr/bin/kraken", kraken_bin, use_sudo=True)
sudo('chown -h {user} {bin}'.format(user=env.KRAKEN_USER, bin=kraken_bin))
kraken = "kraken_{}".format(instance.name)
if not service.is_running(kraken):
# TODO test this on systemd machines
if env.use_systemd:
sudo("systemctl enable kraken_{}.service".format(instance.name))
else:
sudo("update-rc.d kraken_{} defaults".format(instance.name))
print(blue("INFO: kraken {instance} instance is starting on {server}, "
"waiting 5 seconds, we will check if processus is running"
.format(instance=instance.name, server=get_host_addr(env.host_string))))
service.start(kraken)
run("sleep 5") # we wait a bit for the kraken to pop
with settings(warn_only=True):
run("pgrep --list-name --full /srv/kraken/{}/kraken".format(instance.name))
print(blue("INFO: kraken {instance} instance is running on {server}".
format(instance=instance.name, server=get_host_addr(host))))
def delete_kraken_queue_to_rabbitmq(instance, apply_on='reverse'):
"""
Remove queue for a kraken
"""
instance = get_real_instance(instance)
if apply_on == 'engines':
hosts, exclude_hosts = instance.kraken_engines, ()
elif apply_on == 'reverse':
hosts, exclude_hosts = env.roledefs['eng'], instance.kraken_engines
elif apply_on == 'all':
hosts, exclude_hosts = env.roledefs['eng'], ()
else:
abort("Bad 'apply_on' parameter value: {}".format(apply_on))
for host in set(hosts) - set(exclude_hosts):
with settings(host_string=env.default_ssh_user + '@' + env.rabbitmq_host_api):
run('curl -i -u {}:{} "content-type:application/json" -XDELETE '
'"http://{}:{}/api/queues/%2F/kraken_{}_{}_rt"'
.format(env.rabbitmq_user, env.rabbitmq_pass, env.rabbitmq_host_api, env.rabbitmq_port_api,
get_host_addr(host).split('.')[0], instance))
run('curl -i -u {}:{} "content-type:application/json" -XDELETE '
'"http://{}:{}/api/queues/%2F/kraken_{}_{}_task"'
.format(env.rabbitmq_user, env.rabbitmq_pass, env.rabbitmq_host_api, env.rabbitmq_port_api,
get_host_addr(host).split('.')[0], instance))
def remove_kraken_instance(instance, purge_logs=False, apply_on='engines'):
"""
Remove a kraken instance entirely
* Stop the service
* Remove startup at boot time
* Remove initscript
* Remove configuration and pid directory
apply_on values:
- engines: apply on instance.kraken_engines
- reverse: apply on all engines except instance.kraken_engines
- all: apply on all engines
"""
instance = get_real_instance(instance)
if apply_on == 'engines':
hosts, exclude_hosts = instance.kraken_engines, ()
elif apply_on == 'reverse':
hosts, exclude_hosts = env.roledefs['eng'], instance.kraken_engines
elif apply_on == 'all':
hosts, exclude_hosts = env.roledefs['eng'], ()
else:
abort("Bad 'apply_on' parameter value: {}".format(apply_on))
for host in set(hosts) - set(exclude_hosts):
with settings(host_string=host, warn_only=True):
print("INFO: removing kraken instance {} from {}".format(
instance.name, get_host_addr(host)))
service.stop('kraken_{}'.format(instance.name))
run("sleep 3")
# TODO test this on systemd machines
if env.use_systemd:
run("systemctl disable kraken_{}.service".format(
instance.name))
run('systemctl daemon-reload')
run("rm -f {}/kraken_{}.service".format(
env.service_path(), instance.name))
else:
run("rm -f {}/kraken_{}".format(env.service_path(),
instance.name))
run("rm -rf {}/{}/".format(env.kraken_basedir, instance.name))
if purge_logs:
run("rm -f {}/{}.log".format(env.kraken_log_basedir,
instance.name))
def remove_kraken_instance(instance, purge_logs=False, apply_on='engines'):
"""
Remove a kraken instance entirely
* Stop the service
* Remove startup at boot time
* Remove initscript
* Remove configuration and pid directory
apply_on values:
- engines: apply on instance.kraken_engines
- reverse: apply on all engines except instance.kraken_engines
- all: apply on all engines
"""
instance = get_real_instance(instance)
if apply_on == 'engines':
hosts, exclude_hosts = instance.kraken_engines, ()
elif apply_on == 'reverse':
hosts, exclude_hosts = env.roledefs['eng'], instance.kraken_engines
elif apply_on == 'all':
hosts, exclude_hosts = env.roledefs['eng'], ()
else:
abort("Bad 'apply_on' parameter value: {}".format(apply_on))
for host in set(hosts) - set(exclude_hosts):
with settings(
host_string=host,
warn_only=True
):
print("INFO: removing kraken instance {} from {}".format(instance.name, get_host_addr(host)))
service.stop('kraken_{}'.format(instance.name))
run("sleep 3")
# TODO test this on systemd machines
if env.use_systemd:
run("systemctl disable kraken_{}.service".format(instance.name))
run('systemctl daemon-reload')
run("rm -f {}/kraken_{}".format(env.service_path(), instance.name))
run("rm -rf {}/{}/".format(env.kraken_basedir, instance.name))
if purge_logs:
run("rm -f {}/{}.log".format(env.kraken_log_basedir, instance.name))
def update_all_configurations():
"""
update all configuration and restart all services
does not deploy any packages
"""
execute(kraken.get_no_data_instances)
execute(jormungandr.update_jormungandr_conf)
execute(kraken.update_monitor_configuration)
execute(tyr.update_tyr_conf)
for instance in env.instances.values():
execute(tyr.update_tyr_instance_conf, instance)
execute(jormungandr.deploy_jormungandr_instance_conf, instance)
execute(kraken.update_eng_instance_conf, instance)
#once all has been updated, we restart all services for the conf to be taken into account
execute(tyr.restart_tyr_worker)
execute(tyr.restart_tyr_beat)
execute(jormungandr.reload_jormun_safe_all)
execute(kraken.restart_all_krakens)
# and we test the jormungandr
for server in env.roledefs['ws']:
jormungandr.test_jormungandr(utils.get_host_addr(server))
def update_all_configurations():
"""
update all configuration and restart all services
does not deploy any packages
"""
execute(kraken.get_no_data_instances)
execute(jormungandr.update_jormungandr_conf)
execute(kraken.update_monitor_configuration)
execute(tyr.update_tyr_conf)
for instance in env.instances.values():
execute(tyr.update_tyr_instance_conf, instance)
execute(jormungandr.deploy_jormungandr_instance_conf, instance)
execute(kraken.update_eng_instance_conf, instance)
#once all has been updated, we restart all services for the conf to be taken into account
execute(tyr.restart_tyr_worker)
execute(tyr.restart_tyr_beat)
execute(jormungandr.reload_jormun_safe_all)
execute(kraken.restart_all_krakens)
# and we test the jormungandr
for server in env.roledefs['ws']:
jormungandr.test_jormungandr(utils.get_host_addr(server))
def upgrade_all(up_tyr=True, up_confs=True, check_version=True, send_mail='no',
manual_lb=False, check_dead=True, check_bina=True):
"""Upgrade all navitia packages, databases and launch rebinarisation of all instances """
up_tyr = get_bool_from_cli(up_tyr)
up_confs = get_bool_from_cli(up_confs)
check_version = get_bool_from_cli(check_version)
check_dead = get_bool_from_cli(check_dead)
check_bina = get_bool_from_cli(check_bina)
if check_version:
execute(compare_version_candidate_installed, host_name='tyr')
if env.use_load_balancer:
if manual_lb:
print(yellow("WARNING : you are in MANUAL mode :\n"
"Check frequently for message asking you to switch nodes manually"))
else:
# check credential NOW
_adc_connection(check=True)
execute(check_last_dataset)
if send_mail in ('start', 'all'):
broadcast_email('start')
time_dict = TimeCollector()
time_dict.register_start('total_deploy')
if up_tyr:
execute(update_tyr_step, time_dict, only_bina=False, check_bina=check_bina)
if check_version:
execute(compare_version_candidate_installed)
execute(kraken.swap_all_data_nav)
if env.use_load_balancer:
# Upgrade kraken/jormun on first hosts set
env.roledefs['eng'] = env.eng_hosts_1
env.roledefs['ws'] = env.ws_hosts_1
if manual_lb:
raw_input(yellow("Please disable ENG1,3/WS1,5,6 and enable ENG2,4/WS2-4"))
else:
execute(switch_to_first_phase, env.eng_hosts_1, env.ws_hosts_1, env.ws_hosts_2)
time_dict.register_start('kraken')
execute(upgrade_kraken, wait=env.KRAKEN_RESTART_SCHEME, up_confs=up_confs, supervision=True)
if check_dead:
execute(check_dead_instances)
execute(upgrade_jormungandr, reload=False, up_confs=up_confs)
# check first hosts set before upgrading the second one
for server in env.roledefs['ws']:
instance = random.choice(env.instances.values())
execute(jormungandr.test_jormungandr, get_host_addr(server), instance=instance.name)
# Upgrade kraken/jormun on remaining hosts
env.roledefs['eng'] = env.eng_hosts_2
env.roledefs['ws'] = env.ws_hosts_2
if manual_lb:
raw_input(yellow("Please enable ENG1,3/WS1,5,6 and disable ENG2,4/WS2-4"))
else:
execute(switch_to_second_phase, env.eng_hosts_1, env.eng_hosts_2,
env.ws_hosts_1, env.ws_hosts_2)
execute(upgrade_jormungandr, reload=False, up_confs=up_confs)
if manual_lb:
raw_input(yellow("Please enable WS1-6"))
else:
execute(switch_to_third_phase, env.ws_hosts_2)
env.roledefs['ws'] = env.ws_hosts
execute(upgrade_kraken, wait=env.KRAKEN_RESTART_SCHEME, up_confs=up_confs)
time_dict.register_end('kraken')
if not manual_lb:
execute(enable_all_nodes, env.eng_hosts, env.ws_hosts_1, env.ws_hosts_2)
env.roledefs['eng'] = env.eng_hosts
else:
time_dict.register_start('kraken')
execute(upgrade_kraken, wait=env.KRAKEN_RESTART_SCHEME, up_confs=up_confs, supervision=True)
time_dict.register_end('kraken')
execute(upgrade_jormungandr, up_confs=up_confs)
# check deployment OK
for server in env.roledefs['ws']:
instance = random.choice(env.instances.values())
execute(jormungandr.test_jormungandr, get_host_addr(server), instance=instance.name)
# start tyr_beat even if up_tyr is False
execute(tyr.start_tyr_beat)
time_dict.register_end('total_deploy')
if send_mail in ('end', 'all'):
warn_dict = jormungandr.check_kraken_jormun_after_deploy()
status = show_dead_kraken_status(warn_dict, show=True)
status += show_time_deploy(time_dict)
broadcast_email('end', status)
if env.use_load_balancer and manual_lb:
print(yellow("Please enable ENG1-4/WS1-4"))
def create_jormungandr_instance(instance, cheap_journey=False):
#DEPRECATED
"""Create a jormungandr instance
* Deploy /etc/jormungadr.d/<instance>.ini template
* Reload apache
"""
# get the port config from the kraken engine
config = kraken.get_kraken_config(env.roledefs['eng'][0], instance)
zmq_socket = config.get('GENERAL', 'zmq_socket')
instance_port = env.KRAKEN_RE_PORT.match(zmq_socket)
port = instance_port.group('port')
_upload_template("jormungandr/jormungandr.ini.jinja",
instance.jormungandr_config_file,
context={
'env':
env,
'instance':
instance,
'socket':
"tcp://{}:{}".format(env.jormungandr_instance_socket,
port),
'cheap_journey':
cheap_journey,
},
use_sudo=True)
# testing if instance appears in JSON return on URI /v1/coverage on each
# Jormungandr
headers = {'Host': env.jormungandr_url}
server = env.host_string
print("→ server: {}".format(server))
execute(reload_jormun_safe, server)
request_str = 'http://{}/v1/coverage/{}/status'.format(
get_host_addr(server), instance)
print("request_string: {}".format(request_str))
try:
response = requests.get(request_str, headers=headers)
except (ConnectionError, HTTPError) as e:
print(red("HTTP Error %s: %s" % (e.code, e.readlines()[0])))
exit(1)
except Exception as e:
print(red("Error when connecting to %s: %s" %
(env.jormungandr_url, e)))
exit(1)
try:
result = response.json()
except JSONDecodeError:
print(red("cannot read json response : {}".format(response.text)))
exit(1)
if result['status']['kraken_version']:
print(green("OK: Test {} OK".format(instance)))
# really test the instance, warning, maybe no data so 503 returned
test_jormungandr(get_host_addr(env.host_string),
instance,
fail_if_error=False)
