def purge_data_nav(force=False):
"""
purge temp/data.nav.lz4 files
the whole process will be skipped as soon as a single condition is encountered:
- temp data file is more recent than actual data file
- temp data file exists but actual data file is missing
"""
if not force:
print("Checking lz4 temp files purge conditions before proceeding...")
reason = {}
for instance in env.instances.values():
plain_target = get_real_instance(instance).target_lz4_file
temp_target = os.path.join(os.path.dirname(plain_target), 'temp',
os.path.basename(plain_target))
if exists(plain_target):
if exists(temp_target) and files.getmtime(
temp_target) > files.getmtime(plain_target):
reason[instance.name] = "{} is more recent than {}".format(
temp_target, plain_target)
elif exists(temp_target):
reason[instance.name] = "{} does not exists".format(
plain_target)
if reason:
print(yellow("Error: Can't purge lz4 temp files, reasons:"))
for k, v in reason.iteritems():
print(" {}: {}".format(k, v))
exit(1)
for instance in env.instances.values():
plain_target = get_real_instance(instance).target_lz4_file
temp_target = os.path.join(os.path.dirname(plain_target), 'temp',
os.path.basename(plain_target))
if exists(temp_target):
files.remove(temp_target)
def purge_data_nav(force=False):
"""
purge temp/data.nav.lz4 files
the whole process will be skipped as soon as a single condition is encountered:
- temp data file is more recent than actual data file
- temp data file exists but actual data file is missing
"""
if not force:
print("Checking lz4 temp files purge conditions before proceeding...")
reason = {}
for instance in env.instances.values():
plain_target = get_real_instance(instance).target_lz4_file
temp_target = os.path.join(os.path.dirname(plain_target), 'temp', os.path.basename(plain_target))
if exists(plain_target):
if exists(temp_target) and files.getmtime(temp_target) > files.getmtime(plain_target):
reason[instance.name] = "{} is more recent than {}".format(temp_target, plain_target)
elif exists(temp_target):
reason[instance.name] = "{} does not exists".format(plain_target)
if reason:
print(yellow("Error: Can't purge lz4 temp files, reasons:"))
for k, v in reason.iteritems():
print(" {}: {}".format(k, v))
exit(1)
for instance in env.instances.values():
plain_target = get_real_instance(instance).target_lz4_file
temp_target = os.path.join(os.path.dirname(plain_target), 'temp', os.path.basename(plain_target))
if exists(temp_target):
files.remove(temp_target)
def remove_ed_instance(instance):
"""Remove a ed instance entirely"""
ed_dir = get_real_instance(instance).base_ed_dir
destination_dir = get_real_instance(instance).base_destination_dir
backup_dir = get_real_instance(instance).backup_dir
run("rm -rf %s" % ed_dir)
run("rm -rf %s" % destination_dir)
run("rm -rf %s" % backup_dir)
def remove_ed_instance(instance):
"""Remove a ed instance entirely"""
ed_dir = get_real_instance(instance).base_ed_dir
destination_dir = get_real_instance(instance).base_destination_dir
backup_dir = get_real_instance(instance).backup_dir
run("rm -rf %s" % ed_dir)
run("rm -rf %s" % destination_dir)
run("rm -rf %s" % backup_dir)
def remove_instance(instance, admin=False):
"""Completely remove all components for a given instance
Remove instance in jormungandr db manually
TODO: remove instance db automatically
"""
execute(db.remove_instance_from_jormun_database, instance)
execute(db.remove_postgresql_database, get_real_instance(instance).db_name)
execute(db.remove_postgresql_user, get_real_instance(instance).db_user)
execute(tyr.remove_ed_instance, instance)
execute(tyr.remove_tyr_instance, instance)
execute(kraken.remove_kraken_instance, instance)
execute(jormungandr.remove_jormungandr_instance, instance)
if admin and env.use_load_balancer:
execute(remove_kraken_vip, instance)
def create_privileges_instance_db(instance):
postgres_database = get_real_instance(instance).db_name
postgres_connection_user = '******'.format(postgres_database)
for schema in env.postgres_schemas:
run('{} -c "GRANT SELECT ON ALL TABLES IN SCHEMA {} TO {};"'
.format(postgres_connection_user, schema, env.postgres_read_only_user))
def update_eng_instance_conf(instance):
instance = get_real_instance(instance)
for host in instance.kraken_engines:
with settings(host_string=host):
_upload_template("kraken/kraken.ini.jinja", "%s/%s/kraken.ini" %
(env.kraken_basedir, instance.name),
context={
'env': env,
'instance': instance,
}
)
if env.use_systemd:
_upload_template("kraken/systemd_kraken.jinja",
"{}".format(env.service_name('kraken_{}'.format(instance.name))),
context={'env': env,
'instance': instance.name,
'kraken_base_conf': env.kraken_basedir,
},
mode='644'
)
else:
_upload_template("kraken/kraken.initscript.jinja",
"{}".format(env.service_name('kraken_{}'.format(instance.name))),
context={'env': env,
'instance': instance.name,
'kraken_base_conf': env.kraken_basedir,
},
mode='755'
)
# TODO check this, make it consistent with env.use_systemd
update_init(host='eng')
def create_tyr_instance(instance):
""" Create a *private* tyr instance based on the given name
* postgresql user + dedicated database (1 time)
* /etc/tyr.d/instance.ini
* create /srv/ed/<instance> + target-file basedir
"""
instance = get_real_instance(instance)
execute(db.create_instance_db, instance)
# /srv/ed/destination/$instance & /srv/ed/backup/$instance
require_directory(instance.source_dir,
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
require_directory(instance.backup_dir, is_on_nfs4=True,
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
require_directory(instance.base_destination_dir, is_on_nfs4=True,
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
require_directory(env.tyr_base_logdir,
owner=env.TYR_USER, group=env.TYR_USER,
mode='755', use_sudo=True)
require.files.file(os.path.join(env.tyr_base_logdir, instance.name + '.log'),
owner=env.TYR_USER, group=env.TYR_USER,
mode='644', use_sudo=True)
update_tyr_instance_conf(instance) # Note it is not called as a task, for it needs to be done on the same server
if not env.standalone:
#@TODO: change user for non standalone
pass
def restart_kraken(instance, wait='serial'):
""" Restart all krakens of an instance (using pool), serially or in parallel,
then test them. Testing serially assures that krakens are restarted serially.
:param wait: string.
Possible values=False or None: restart in parallel, no test
'serial': restart serially and test
'parallel': restart in parallel and test
'no_test': explicitely skip tests (faster but dangerous)
The default value is 'serial' because it is the safest scenario
to restart the krakens of an instance in production.
"""
if wait not in ('serial', 'parallel', 'no_test'):
abort(
yellow(
"Error: wait parameter must be 'serial', 'parallel' or 'no_test', found '{}'"
.format(wait)))
instance = get_real_instance(instance)
excluded = instance.name in env.excluded_instances
# restart krakens of this instance that are also in the eng role,
# this works with the "pool" switch mechanism used in upgrade_all()
for host in set(instance.kraken_engines).intersection(env.roledefs['eng']):
restart_kraken_on_host(instance, host)
if wait == 'serial' and not excluded:
test_kraken(instance, fail_if_error=False, wait=True, hosts=[host])
if wait == 'parallel' and not excluded:
test_kraken(instance, fail_if_error=False, wait=True)
if wait != 'no_test' and excluded:
print(
yellow("Coverage '{}' has no data, not testing it".format(
instance.name)))
if wait == 'no_test':
print(
yellow(
"Warning Coverage '{}' not tested: parameter wait='no_test'".
format(instance.name)))
def create_tyr_instance(instance):
""" Create a *private* tyr instance based on the given name
* postgresql user + dedicated database (1 time)
* /etc/tyr.d/instance.ini
* create /srv/ed/<instance> + target-file basedir
"""
instance = get_real_instance(instance)
execute(db.create_instance_db, instance)
# /srv/ed/destination/$instance & /srv/ed/backup/$instance
require_directory(instance.source_dir,
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
require_directory(instance.backup_dir, is_on_nfs4=True,
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
require_directory(instance.base_destination_dir, is_on_nfs4=True,
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
require_directory(env.tyr_base_logdir,
owner=env.TYR_USER, group=env.TYR_USER,
mode='755', use_sudo=True)
require.files.file(os.path.join(env.tyr_base_logdir, instance.name + '.log'),
owner=env.TYR_USER, group=env.TYR_USER,
mode='644', use_sudo=True)
update_tyr_instance_conf(instance) # Note it is not called as a task, for it needs to be done on the same server
if not env.standalone:
#@TODO: change user for non standalone
pass
def update_eng_instance_conf(instance, host=None):
instance = get_real_instance(instance)
hosts = [host] if host else instance.kraken_engines
for host in hosts:
with settings(host_string=host):
require.files.directory(os.path.join(instance.kraken_basedir, instance.name),
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
_upload_template("kraken/kraken.ini.jinja", "%s/%s/kraken.ini" %
(env.kraken_basedir, instance.name),
context={
'env': env,
'instance': instance,
}
)
if env.use_systemd:
_upload_template("kraken/systemd_kraken.jinja",
"{}".format(env.service_name('kraken_{}'.format(instance.name))),
context={'env': env,
'instance': instance.name,
'kraken_base_conf': env.kraken_basedir,
},
mode='644'
)
else:
_upload_template("kraken/kraken.initscript.jinja",
"{}".format(env.service_name('kraken_{}'.format(instance.name))),
context={'env': env,
'instance': instance.name,
'kraken_base_conf': env.kraken_basedir,
},
mode='755'
)
# TODO check this, make it consistent with env.use_systemd
update_init(host='eng')
def delete_kraken_queue_to_rabbitmq(instance, apply_on='reverse'):
"""
Remove queue for a kraken
"""
instance = get_real_instance(instance)
if apply_on == 'engines':
hosts, exclude_hosts = instance.kraken_engines, ()
elif apply_on == 'reverse':
hosts, exclude_hosts = env.roledefs['eng'], instance.kraken_engines
elif apply_on == 'all':
hosts, exclude_hosts = env.roledefs['eng'], ()
else:
abort("Bad 'apply_on' parameter value: {}".format(apply_on))
if env.rabbitmq_host_api == 'localhost':
host_string = env.roledefs['tyr_master'][0]
else:
host_string = env.rabbitmq_host_api
for host in set(hosts) - set(exclude_hosts):
with settings(host_string=host_string):
run('curl -i -u {}:{} -XDELETE "http://localhost:{}/api/queues/%2F/kraken_{}_{}_rt"'
.format(env.rabbitmq_user, env.rabbitmq_pass, env.rabbitmq_port_api,
get_host_addr(host).split('.')[0], instance))
run('curl -i -u {}:{} -XDELETE "http://localhost:{}/api/queues/%2F/kraken_{}_{}_task"'
.format(env.rabbitmq_user, env.rabbitmq_pass, env.rabbitmq_port_api,
get_host_addr(host).split('.')[0], instance))
def delete_kraken_queue_to_rabbitmq(instance, apply_on='reverse'):
"""
Remove queue for a kraken
"""
instance = get_real_instance(instance)
if apply_on == 'engines':
hosts, exclude_hosts = instance.kraken_engines, ()
elif apply_on == 'reverse':
hosts, exclude_hosts = env.roledefs['eng'], instance.kraken_engines
elif apply_on == 'all':
hosts, exclude_hosts = env.roledefs['eng'], ()
else:
abort("Bad 'apply_on' parameter value: {}".format(apply_on))
host_string = env.rabbitmq_kraken_host
for host in set(hosts) - set(exclude_hosts):
with settings(host_string=host_string):
run('curl -i -u {}:{} -XDELETE "http://localhost:{}/api/queues/{}/kraken_{}_{}_rt"'
.format(env.rabbitmq_kraken_user, env.rabbitmq_kraken_pass,
env.rabbitmq_port_api, rabbitmq_kraken_vhost,
get_host_addr(host).split('.')[0], instance))
run('curl -i -u {}:{} -XDELETE "http://localhost:{}/api/queues/{}/kraken_{}_{}_task"'
.format(env.rabbitmq_kraken_user, env.rabbitmq_kraken_pass,
env.rabbitmq_port_api, rabbitmq_kraken_vhost,
get_host_addr(host).split('.')[0], instance))
def restart_kraken(instance, wait='serial'):
""" Restart all krakens of an instance (using pool), serially or in parallel,
then test them. Testing serially assures that krakens are restarted serially.
:param wait: string.
Possible values=False or None: restart in parallel, no test
'serial': restart serially and test
'parallel': restart in parallel and test
'no_test': explicitely skip tests (faster but dangerous)
The default value is 'serial' because it is the safest scenario
to restart the krakens of an instance in production.
"""
if wait not in ('serial', 'parallel', 'no_test'):
abort(yellow("Error: wait parameter must be 'serial', 'parallel' or 'no_test', found '{}'".format(wait)))
instance = get_real_instance(instance)
excluded = instance.name in env.excluded_instances
# restart krakens of this instance that are also in the eng role,
# this works with the "pool" switch mechanism used in upgrade_all()
for host in set(instance.kraken_engines).intersection(env.roledefs['eng']):
restart_kraken_on_host(instance, host)
if wait == 'serial' and not excluded:
test_kraken(instance, fail_if_error=False, wait=True, hosts=[host])
if wait == 'parallel' and not excluded:
test_kraken(instance, fail_if_error=False, wait=True)
if wait != 'no_test' and excluded:
print(yellow("Coverage '{}' has no data, not testing it".format(instance.name)))
if wait == 'no_test':
print(yellow("Warning Coverage '{}' not tested: parameter wait='no_test'".format(instance.name)))
def get_kraken_config(server, instance):
"""Get kraken configuration of a given instance"""
# TODO this task is never used and it looks like a function (inconsistent)
instance = get_real_instance(instance)
with settings(host_string=env.make_ssh_url(server)):
config_path = "%s/%s/kraken.ini" % (env.kraken_basedir, instance.name)
# first get the configfile here
temp_file = StringIO.StringIO()
if exists(config_path):
get(config_path, temp_file)
else:
print(red("ERROR: can't find %s" % config_path))
exit(1)
config = ConfigParser.RawConfigParser(allow_no_value=True)
config_text = temp_file.getvalue()
config.readfp(BytesIO(config_text))
if 'GENERAL' in config.sections():
return config
else:
return None
def get_kraken_config(server, instance):
"""Get kraken configuration of a given instance"""
# TODO this task is never used and it looks like a function (inconsistent)
instance = get_real_instance(instance)
with settings(host_string=env.make_ssh_url(server)):
config_path = "%s/%s/kraken.ini" % (env.kraken_basedir, instance.name)
# first get the configfile here
temp_file = StringIO.StringIO()
if exists(config_path):
get(config_path, temp_file)
else:
print(red("ERROR: can't find %s" % config_path))
exit(1)
config = ConfigParser.RawConfigParser(allow_no_value=True)
config_text = temp_file.getvalue()
config.readfp(BytesIO(config_text))
if 'GENERAL' in config.sections():
return config
else:
return None
def remove_jormungandr_instance(instance):
""" Remove a jormungandr instance entirely
* Remove json file which declare the instance
* Reload apache
"""
instance = get_real_instance(instance)
run("rm --force {}".format(instance.jormungandr_config_file))
def update_eng_instance_conf(instance, host=None):
instance = get_real_instance(instance)
hosts = [host] if host else instance.kraken_engines
for host in hosts:
with settings(host_string=host):
require.files.directory(os.path.join(instance.kraken_basedir, instance.name),
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
_upload_template("kraken/kraken.ini.jinja", "%s/%s/kraken.ini" %
(env.kraken_basedir, instance.name),
context={
'env': env,
'instance': instance,
}
)
if env.use_systemd:
_upload_template("kraken/systemd_kraken.jinja",
"{}".format(env.service_name('kraken_{}'.format(instance.name))),
context={'env': env,
'instance': instance.name,
'kraken_base_conf': env.kraken_basedir,
},
mode='644'
)
else:
_upload_template("kraken/kraken.initscript.jinja",
"{}".format(env.service_name('kraken_{}'.format(instance.name))),
context={'env': env,
'instance': instance.name,
'kraken_base_conf': env.kraken_basedir,
},
mode='755'
)
# TODO check this, make it consistent with env.use_systemd
update_init(host='eng')
def deploy_jormungandr_instance_conf(instance):
""" Deploy or redeploy one jormungander coverage:
* Deploy the json configuration file
* Do not reload apache
"""
instance = get_real_instance(instance)
config = {
'key': instance.name,
'zmq_socket': instance.jormungandr_zmq_socket_for_instance,
'realtime_proxies': instance.realtime_proxies
}
if instance.pt_zmq_socket:
config['pt_zmq_socket'] = instance.pt_zmq_socket
if instance.street_network:
config["street_network"] = instance.street_network
if instance.ridesharing:
config["ridesharing"] = instance.ridesharing
if instance.autocomplete:
config["default_autocomplete"] = instance.autocomplete
if instance.equipment_details_providers:
config[
"equipment_details_providers"] = instance.equipment_details_providers
_upload_template("jormungandr/instance.json.jinja",
instance.jormungandr_config_file,
context={'json': json.dumps(config, indent=4)},
use_sudo=True)
def remove_kraken_instance(instance, purge_logs=False, apply_on='engines'):
"""
Remove a kraken instance entirely
* Stop the service
* Remove startup at boot time
* Remove initscript
* Remove configuration and pid directory
apply_on values:
- engines: apply on instance.kraken_engines
- reverse: apply on all engines except instance.kraken_engines
- all: apply on all engines
"""
instance = get_real_instance(instance)
if apply_on == 'engines':
hosts, exclude_hosts = instance.kraken_engines, ()
elif apply_on == 'reverse':
hosts, exclude_hosts = env.roledefs['eng'], instance.kraken_engines
elif apply_on == 'all':
hosts, exclude_hosts = env.roledefs['eng'], ()
else:
abort("Bad 'apply_on' parameter value: {}".format(apply_on))
for host in set(hosts) - set(exclude_hosts):
with settings(
host_string=host,
warn_only=True
):
sudo("service kraken_{} stop && sleep 3".format(instance.name))
if not env.use_systemd:
run("update-rc.d -f kraken_{} remove".format(instance.name))
sudo("rm -f {}/kraken_{}".format(env.service_path(), instance.name))
sudo("rm -rf {}/{}/".format(env.kraken_basedir, instance.name))
if purge_logs:
sudo("rm -f {}/{}.log".format(env.kraken_log_basedir, instance.name))
def create_eng_instance(instance):
""" Create a new kraken instance (idempotent)
* Install requirements
* Deploy the binary, the templatized ini configuration in a dedicated
directory with rights to www-data and the logdir
* Deploy initscript and add it to startup
* Start the service
"""
instance = get_real_instance(instance)
for host in instance.kraken_engines:
with settings(host_string=host):
# base_conf
require.files.directory(instance.kraken_basedir,
owner=env.KRAKEN_USER,
group=env.KRAKEN_USER,
use_sudo=True)
# logs
require.files.directory(env.kraken_log_basedir,
owner=env.KRAKEN_USER,
group=env.KRAKEN_USER,
use_sudo=True)
update_eng_instance_conf(instance, host)
# kraken.ini, pid and binary symlink
kraken_bin = "{}/{}/kraken".format(env.kraken_basedir,
instance.name)
if not is_link(kraken_bin):
idempotent_symlink("/usr/bin/kraken",
kraken_bin,
use_sudo=True)
sudo('chown -h {user} {bin}'.format(user=env.KRAKEN_USER,
bin=kraken_bin))
kraken = "kraken_{}".format(instance.name)
if not service.is_running(kraken):
# TODO test this on systemd machines
if env.use_systemd:
sudo("systemctl enable kraken_{}.service".format(
instance.name))
else:
sudo("update-rc.d kraken_{} defaults".format(
instance.name))
print(
blue(
"INFO: kraken {instance} instance is starting on {server}, "
"waiting 5 seconds, we will check if processus is running"
.format(instance=instance.name,
server=get_host_addr(env.host_string))))
service.start(kraken)
run("sleep 5") # we wait a bit for the kraken to pop
with settings(warn_only=True):
run("pgrep --list-name --full /srv/kraken/{}/kraken".format(
instance.name))
print(
blue("INFO: kraken {instance} instance is running on {server}".
format(instance=instance.name,
server=get_host_addr(host))))
def rollback_instance(instance, test=True):
""" Use this only if something goes wrong during deployment of an instance
"""
test = get_bool_from_cli(test)
instance = get_real_instance(instance)
execute(swap_data_nav, instance, force=True)
execute(set_kraken_binary, instance, old=True)
execute(restart_kraken, instance, wait=env.KRAKEN_RESTART_SCHEME if test else 'no_test')
def restart_kraken_on_host(instance, host):
""" Restart a kraken of an instance on a given server
"""
instance = get_real_instance(instance)
with settings(host_string=host):
kraken = 'kraken_' + instance.name
start_or_stop_with_delay(kraken, 4000, 500, start=False, only_once=True)
start_or_stop_with_delay(kraken, 4000, 500, only_once=env.KRAKEN_START_ONLY_ONCE)
def require_kraken_started(instance):
"""start a kraken instance on all servers if it is not already started
"""
instance = get_real_instance(instance)
kraken = 'kraken_' + instance.name
for host in instance.kraken_engines:
with settings(host_string=host):
start_or_stop_with_delay(kraken, 4000, 500, only_once=True)
def stop_kraken(instance):
"""Stop a kraken instance on all servers
"""
instance = get_real_instance(instance)
kraken = 'kraken_' + instance.name
for host in instance.kraken_engines:
with settings(host_string=host):
start_or_stop_with_delay(kraken, 4000, 500, start=False, only_once=True)
def stop_kraken(instance):
"""Stop a kraken instance on all servers
"""
instance = get_real_instance(instance)
kraken = 'kraken_' + instance.name
for host in instance.kraken_engines:
with settings(host_string=host):
start_or_stop_with_delay(kraken, 4000, 500, start=False, only_once=True)
def restart_kraken_on_host(instance, host):
""" Restart a kraken of an instance on a given server
"""
instance = get_real_instance(instance)
with settings(host_string=host):
kraken = 'kraken_' + instance.name
start_or_stop_with_delay(kraken, 4000, 500, start=False, only_once=True)
start_or_stop_with_delay(kraken, 4000, 500, only_once=env.KRAKEN_START_ONLY_ONCE)
def create_privileges_instance_db(instance):
postgres_database = get_real_instance(instance).db_name
postgres_connection_user = '******'.format(
postgres_database)
for schema in env.postgres_schemas:
run('{} -c "GRANT SELECT ON ALL TABLES IN SCHEMA {} TO {};"'.format(
postgres_connection_user, schema, env.postgres_read_only_user))
def rollback_instance(instance, test=True):
""" Use this only if something goes wrong during deployment of an instance
"""
test = get_bool_from_cli(test)
instance = get_real_instance(instance)
execute(swap_data_nav, instance, force=True)
execute(set_kraken_binary, instance, old=True)
execute(restart_kraken, instance, wait=env.KRAKEN_RESTART_SCHEME if test else 'no_test')
def require_kraken_started(instance):
"""start a kraken instance on all servers if it is not already started
"""
instance = get_real_instance(instance)
kraken = 'kraken_' + instance.name
for host in instance.kraken_engines:
with settings(host_string=host):
start_or_stop_with_delay(kraken, 4000, 500, only_once=True)
def set_kraken_binary(instance, old=False):
instance = get_real_instance(instance)
for host in instance.kraken_engines:
with settings(host_string=host):
kraken_bin = "{}/{}/kraken".format(env.kraken_basedir,
instance.name)
idempotent_symlink('/usr/bin/kraken' + ('.old' if old else ''),
kraken_bin,
use_sudo=True)
def remove_jormungandr_instance(instance):
""" Remove a jormungandr instance entirely
* Remove json file which declare the instance
* Reload apache
"""
instance = get_real_instance(instance)
run("rm --force %s" % (instance.jormungandr_config_file))
reload_jormun_safe_all()
def test_kraken(instance, fail_if_error=True, wait=False, loaded_is_ok=None):
"""Test kraken with '?instance='"""
instance = get_real_instance(instance)
wait = get_bool_from_cli(wait)
# env.host will call the monitor kraken on the current host
request = Request('http://{}:{}/{}/?instance={}'.format(env.host,
env.kraken_monitor_port, env.kraken_monitor_location_dir, instance.name))
if wait:
# we wait until we get a gestion and the instance is 'loaded'
try:
result = Retrying(stop_max_delay=env.KRAKEN_RESTART_DELAY * 1000,
wait_fixed=1000, retry_on_result=lambda x: x is None or not x['loaded']) \
.call(_test_kraken, request, fail_if_error)
except Exception as e:
print(red("ERROR: could not reach {}, too many retries ! ({})".format(instance.name, e)))
result = {'status': False}
else:
result = _test_kraken(request, fail_if_error)
if result['status'] != 'running':
if result['status'] == 'no_data':
print(yellow("WARNING: instance {} has no loaded data".format(instance.name)))
return False
if fail_if_error:
print(red("ERROR: Instance {} is not running ! ({})".format(instance.name, result)))
return False
print(yellow("WARNING: Instance {} is not running ! ({})".format(instance.name, result)))
return False
if not result['is_connected_to_rabbitmq']:
print(yellow("WARNING: Instance {} is not connected to rabbitmq".format(instance.name)))
return False
if loaded_is_ok is None:
loaded_is_ok = wait
if not loaded_is_ok:
if result['loaded']:
print(yellow("WARNING: instance {} has loaded data".format(instance.name)))
return True
else:
print(green("OK: instance {} has correct values: {}".format(instance.name, result)))
return False
else:
if result['loaded']:
print(green("OK: instance {} has correct values: {}".format(instance.name, result)))
return True
elif fail_if_error:
print(red("CRITICAL: instance {} has no loaded data".format(instance.name)))
exit(1)
else:
print(yellow("WARNING: instance {} has no loaded data".format(instance.name)))
return False
def create_eng_instance(instance):
"""Create a new kraken instance
* Install requirements (idem potem)
* Deploy the binary, the templatized ini configuration in a dedicated
directory with rights to www-data and the logdir
* Deploy initscript and add it to startup
* Start the service
"""
instance = get_real_instance(instance)
for host in instance.kraken_engines:
with settings(host_string=host):
# base_conf
require.files.directory(instance.kraken_basedir,
owner=env.KRAKEN_USER,
group=env.KRAKEN_USER,
use_sudo=True)
# logs
require.files.directory(env.kraken_log_basedir,
owner=env.KRAKEN_USER,
group=env.KRAKEN_USER,
use_sudo=True)
update_eng_instance_conf(instance)
# kraken.ini, pid and binary symlink
kraken_bin = "{}/{}/kraken".format(env.kraken_basedir,
instance.name)
if not is_link(kraken_bin):
files.symlink("/usr/bin/kraken", kraken_bin, use_sudo=True)
sudo('chown -h {user} {bin}'.format(user=env.KRAKEN_USER,
bin=kraken_bin))
#run("chmod 755 /etc/init.d/kraken_{}".format(instance))
# TODO refactor this and test it on systemd and non-systemd machines
if not env.use_systemd:
sudo("update-rc.d kraken_{} defaults".format(instance.name))
print(
blue(
"INFO: Kraken {instance} instance is starting on {server}, "
"waiting 5 seconds, we will check if processus is running".
format(instance=instance.name,
server=get_host_addr(env.host_string))))
service.start("kraken_{}".format(instance.name))
run("sleep 5") # we wait a bit for the kraken to pop
# test it !
# execute(test_kraken, get_host_addr(env.host_string), instance, fail_if_error=False)
with settings(warn_only=True):
run("pgrep --list-name --full {}".format(instance.name))
print(
blue("INFO: kraken {instance} instance is running on {server}".
format(instance=instance.name,
server=get_host_addr(env.host_string))))
def redeploy_kraken(instance, create=True):
"""
Redistributes an existing kraken on eng engines.
Call this task when the zmq_server parameter of add_instance is changed.
Use create=False if krakens mapping is reduced (this avoids restarting them).
Use create=True if krakens are displaced or mapping is expanded.
"""
instance = get_real_instance(instance)
create = get_bool_from_cli(create)
if create:
execute(create_eng_instance, instance)
execute(remove_kraken_instance, instance, purge_logs=True, apply_on='reverse')
def redeploy_kraken(instance, create=True):
"""
Redistributes an existing kraken on eng engines.
Call this task when the zmq_server parameter of add_instance is changed.
Use create=False if krakens mapping is reduced (this avoids restarting them).
Use create=True if krakens are displaced or mapping is expanded.
"""
instance = get_real_instance(instance)
create = get_bool_from_cli(create)
if create:
execute(create_eng_instance, instance)
execute(remove_kraken_instance, instance, purge_logs=True, apply_on='reverse')
def restart_kraken(instance, test=True, wait=True):
"""Restart a kraken instance on a given server
To let us not restart all kraken servers in the farm
"""
instance = get_real_instance(instance)
wait = get_bool_from_cli(wait)
if instance.name not in env.excluded_instances:
for host in set(instance.kraken_engines).intersection(env.roledefs['eng']):
restart_kraken_on_host(instance, host)
if test:
test_kraken(instance, fail_if_error=False, wait=wait)
else:
print(yellow("{} has no data, not testing it".format(instance.name)))
def swap_data_nav(instance, force=False):
""" swap old/new data.nav.lz4, only if new is still in temp directory
"""
plain_target = get_real_instance(instance).target_lz4_file
temp_target = os.path.join(os.path.dirname(plain_target), 'temp', os.path.basename(plain_target))
if exists(plain_target):
if exists(temp_target) and \
(force or (files.getmtime(temp_target) > files.getmtime(plain_target))):
swap_temp = os.path.join(os.path.dirname(temp_target), 'x')
files.move(plain_target, swap_temp)
files.move(temp_target, plain_target)
files.move(swap_temp, temp_target)
elif exists(temp_target):
files.move(temp_target, plain_target)
def restart_kraken(instance, test=True, wait=True):
"""Restart a kraken instance on a given server
To let us not restart all kraken servers in the farm
"""
instance = get_real_instance(instance)
wait = get_bool_from_cli(wait)
if instance.name not in env.excluded_instances:
kraken = 'kraken_' + instance.name
start_or_stop_with_delay(kraken, 4000, 500, start=False, only_once=True)
start_or_stop_with_delay(kraken, 4000, 500, only_once=env.KRAKEN_START_ONLY_ONCE)
if test:
test_kraken(instance.name, fail_if_error=False, wait=wait)
else:
print(yellow("{} has no data, not testing it".format(instance.name)))
def swap_data_nav(instance, force=False):
""" swap old/new data.nav.lz4, only if new is still in temp directory
"""
plain_target = get_real_instance(instance).target_lz4_file
temp_target = os.path.join(os.path.dirname(plain_target), 'temp', os.path.basename(plain_target))
if exists(plain_target):
if exists(temp_target) and \
(force or (files.getmtime(temp_target) > files.getmtime(plain_target))):
swap_temp = os.path.join(os.path.dirname(temp_target), 'x')
files.move(plain_target, swap_temp)
files.move(temp_target, plain_target)
files.move(swap_temp, temp_target)
elif exists(temp_target):
files.move(temp_target, plain_target)
def update_instance(instance):
"""
param (instance) - update all configuration and restart all services
does not deploy any packages
"""
instance = utils.get_real_instance(instance) # since it might be a endpoint we might need to get the real instance
print(blue('updating {}'.format(instance.name)))
#first of all we compute the instance status, it will be helpfull later
execute(utils.compute_instance_status, instance)
execute(tyr.create_tyr_instance, instance)
execute(db.postgis_initdb, instance.db_name)
execute(tyr.update_ed_db, instance.name)
execute(jormungandr.deploy_jormungandr_instance_conf, instance)
execute(kraken.create_eng_instance, instance)
execute(tyr.deploy_default_synonyms, instance)
def deploy_jormungandr_instance_conf(instance):
instance = get_real_instance(instance)
config = {'key': instance.name, 'zmq_socket': instance.jormungandr_zmq_socket_for_instance}
config['realtime_proxies'] = instance.realtime_proxies
_upload_template("jormungandr/instance.json.jinja",
instance.jormungandr_config_file,
context={
'json': json.dumps(config, indent=4)
},
use_sudo=True
)
# the old configuration file were .ini, now it's json, we need to clean up
if fabtools.files.is_file(instance.jormungandr_old_ini_config_file):
fabtools.files.remove(instance.jormungandr_old_ini_config_file)
def deploy_jormungandr_instance_conf(instance):
instance = get_real_instance(instance)
config = {'key': instance.name, 'zmq_socket': instance.jormungandr_zmq_socket_for_instance}
config['realtime_proxies'] = instance.realtime_proxies
_upload_template("jormungandr/instance.json.jinja",
instance.jormungandr_config_file,
context={
'json': json.dumps(config, indent=4)
},
use_sudo=True
)
# the old configuration file were .ini, now it's json, we need to clean up
if fabtools.files.is_file(instance.jormungandr_old_ini_config_file):
fabtools.files.remove(instance.jormungandr_old_ini_config_file)
def deploy_jormungandr_instance_conf(instance):
""" Deploy or redeploy one jormungander coverage:
* Deploy the json configuration file
* Do not reload apache
"""
instance = get_real_instance(instance)
config = {'key': instance.name,
'zmq_socket': instance.jormungandr_zmq_socket_for_instance,
'realtime_proxies': instance.realtime_proxies}
_upload_template("jormungandr/instance.json.jinja",
instance.jormungandr_config_file,
context={
'json': json.dumps(config, indent=4)
},
use_sudo=True
)
def remove_kraken_instance(instance, purge_logs=False):
"""Remove a kraken instance entirely
* Stop the service
* Remove startup at boot time
* Remove initscript
* Remove configuration and pid directory
"""
instance = get_real_instance(instance)
sudo("service kraken_%s stop; sleep 3" % instance.name)
run("update-rc.d -f kraken_%s remove" % instance.name)
run("rm --force /etc/init.d/kraken_%s" % instance.name)
run("rm --recursive --force %s/%s/" % (env.kraken_basedir, instance.name))
if purge_logs:
# ex.: /var/log/kraken/navitia-bretagne.log
run("rm --force %s-%s.log" % (env.kraken_log_name, instance.name))
def update_eng_instance_conf(instance):
instance = get_real_instance(instance)
_upload_template("kraken/kraken.ini.jinja",
"%s/%s/kraken.ini" % (env.kraken_basedir, instance.name),
context={
'env': env,
'instance': instance,
})
_upload_template("kraken/kraken.initscript.jinja",
"/etc/init.d/kraken_%s" % instance.name,
context={
'env': env,
'instance': instance.name,
'kraken_base_conf': env.kraken_basedir,
},
mode='755')
def remove_kraken_instance(instance, purge_logs=False):
"""Remove a kraken instance entirely
* Stop the service
* Remove startup at boot time
* Remove initscript
* Remove configuration and pid directory
"""
instance = get_real_instance(instance)
sudo("service kraken_%s stop; sleep 3" % instance.name)
run("update-rc.d -f kraken_%s remove" % instance.name)
run("rm --force /etc/init.d/kraken_%s" % instance.name)
run("rm --recursive --force %s/%s/" % (env.kraken_basedir, instance.name))
if purge_logs:
# ex.: /var/log/kraken/navitia-bretagne.log
run("rm --force %s-%s.log" % (env.kraken_log_name, instance.name))
def update_instance(instance):
"""
param (instance) - update all configuration and restart all services
does not deploy any packages
"""
instance = utils.get_real_instance(
instance
) # since it might be a endpoint we might need to get the real instance
print(blue('updating {}'.format(instance.name)))
#first of all we compute the instance status, it will be helpfull later
execute(utils.compute_instance_status, instance)
execute(tyr.create_tyr_instance, instance)
execute(db.postgis_initdb, instance.db_name)
execute(tyr.update_ed_db, instance.name)
execute(jormungandr.deploy_jormungandr_instance_conf, instance)
execute(kraken.create_eng_instance, instance)
execute(tyr.deploy_default_synonyms, instance)
def update_eng_instance_conf(instance):
instance = get_real_instance(instance)
_upload_template("kraken/kraken.ini.jinja", "%s/%s/kraken.ini" %
(env.kraken_basedir, instance.name),
context={
'env': env,
'instance': instance,
}
)
_upload_template("kraken/kraken.initscript.jinja",
"/etc/init.d/kraken_%s" % instance.name,
context={'env': env,
'instance': instance.name,
'kraken_base_conf': env.kraken_basedir,
},
mode='755'
)
def create_eng_instance(instance):
"""Create a new kraken instance
* Install requirements (idem potem)
* Deploy the binary, the templatized ini configuration in a dedicated
directory with rights to www-data and the logdir
* Deploy initscript and add it to startup
* Start the service
"""
instance = get_real_instance(instance)
# base_conf
require.files.directory(instance.kraken_basedir,
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
# logs
require.files.directory(env.kraken_log_basedir,
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
require.files.directory(instance.base_destination_dir,
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
update_eng_instance_conf(instance)
# kraken.ini, pid and binary symlink
if not exists("{}/{}/kraken".format(env.kraken_basedir, instance.name)):
kraken_bin = "{}/{}/kraken".format(env.kraken_basedir, instance.name)
files.symlink("/usr/bin/kraken", kraken_bin, use_sudo=True)
sudo('chown {user} {bin}'.format(user=env.KRAKEN_USER, bin=kraken_bin))
#run("chmod 755 /etc/init.d/kraken_{}".format(instance))
sudo("update-rc.d kraken_{} defaults".format(instance.name))
print(blue("INFO: Kraken {instance} instance is starting on {server}, "
"waiting 5 seconds, we will check if processus is running".format(
instance=instance.name, server=get_host_addr(env.host_string))))
service.start("kraken_{} start".format(instance.name))
run("sleep 5") # we wait a bit for the kraken to pop
# test it !
# execute(test_kraken, get_host_addr(env.host_string), instance, fail_if_error=False)
print("server: {}".format(env.host_string))
run("pgrep --list-name --full {}".format(instance.name))
print(blue("INFO: kraken {instance} instance is running on {server}".
format(instance=instance.name, server=get_host_addr(env.host_string))))
def deploy_jormungandr_instance_conf(instance):
""" Deploy or redeploy one jormungander coverage:
* Deploy the json configuration file
* Do not reload apache
"""
instance = get_real_instance(instance)
config = {
"key": instance.name,
"zmq_socket": instance.jormungandr_zmq_socket_for_instance,
"realtime_proxies": instance.realtime_proxies,
}
if instance.street_network:
config["street_network"] = instance.street_network
_upload_template(
"jormungandr/instance.json.jinja",
instance.jormungandr_config_file,
context={"json": json.dumps(config, indent=4)},
use_sudo=True,
)
def remove_kraken_instance(instance, purge_logs=False, apply_on='engines'):
"""
Remove a kraken instance entirely
* Stop the service
* Remove startup at boot time
* Remove initscript
* Remove configuration and pid directory
apply_on values:
- engines: apply on instance.kraken_engines
- reverse: apply on all engines except instance.kraken_engines
- all: apply on all engines
"""
instance = get_real_instance(instance)
if apply_on == 'engines':
hosts, exclude_hosts = instance.kraken_engines, ()
elif apply_on == 'reverse':
hosts, exclude_hosts = env.roledefs['eng'], instance.kraken_engines
elif apply_on == 'all':
hosts, exclude_hosts = env.roledefs['eng'], ()
else:
abort("Bad 'apply_on' parameter value: {}".format(apply_on))
for host in set(hosts) - set(exclude_hosts):
with settings(host_string=host, warn_only=True):
print("INFO: removing kraken instance {} from {}".format(
instance.name, get_host_addr(host)))
service.stop('kraken_{}'.format(instance.name))
run("sleep 3")
# TODO test this on systemd machines
if env.use_systemd:
run("systemctl disable kraken_{}.service".format(
instance.name))
run('systemctl daemon-reload')
run("rm -f {}/kraken_{}.service".format(
env.service_path(), instance.name))
else:
run("rm -f {}/kraken_{}".format(env.service_path(),
instance.name))
run("rm -rf {}/{}/".format(env.kraken_basedir, instance.name))
if purge_logs:
run("rm -f {}/{}.log".format(env.kraken_log_basedir,
instance.name))
def update_instance(instance, reload_jormun=True):
"""
param (instance) - update all configuration and restart all services
does not deploy any packages
"""
instance = get_real_instance(instance)
reload_jormun = get_bool_from_cli(reload_jormun)
print(blue('updating {}'.format(instance.name)))
#first of all we compute the instance status, it will be helpfull later
execute(compute_instance_status, instance)
execute(tyr.create_tyr_instance, instance)
execute(db.postgis_initdb, instance.db_name)
execute(tyr.update_ed_db, instance.name)
execute(jormungandr.deploy_jormungandr_instance_conf, instance)
execute(kraken.create_eng_instance, instance)
execute(tyr.deploy_default_synonyms, instance)
execute(db.create_privileges_instance_db, instance)
if reload_jormun:
execute(jormungandr.reload_jormun_safe_all)
def create_eng_instance(instance):
""" Create a new kraken instance (idempotent)
* Install requirements
* Deploy the binary, the templatized ini configuration in a dedicated
directory with rights to www-data and the logdir
* Deploy initscript and add it to startup
* Start the service
"""
instance = get_real_instance(instance)
for host in instance.kraken_engines:
with settings(host_string=host):
# base_conf
require.files.directory(instance.kraken_basedir,
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
# logs
require.files.directory(env.kraken_log_basedir,
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
update_eng_instance_conf(instance, host)
# kraken.ini, pid and binary symlink
kraken_bin = "{}/{}/kraken".format(env.kraken_basedir, instance.name)
if not is_link(kraken_bin):
idempotent_symlink("/usr/bin/kraken", kraken_bin, use_sudo=True)
sudo('chown -h {user} {bin}'.format(user=env.KRAKEN_USER, bin=kraken_bin))
kraken = "kraken_{}".format(instance.name)
if not service.is_running(kraken):
# TODO test this on systemd machines
if env.use_systemd:
sudo("systemctl enable kraken_{}.service".format(instance.name))
else:
sudo("update-rc.d kraken_{} defaults".format(instance.name))
print(blue("INFO: kraken {instance} instance is starting on {server}, "
"waiting 5 seconds, we will check if processus is running"
.format(instance=instance.name, server=get_host_addr(env.host_string))))
service.start(kraken)
run("sleep 5") # we wait a bit for the kraken to pop
with settings(warn_only=True):
run("pgrep --list-name --full /srv/kraken/{}/kraken".format(instance.name))
print(blue("INFO: kraken {instance} instance is running on {server}".
format(instance=instance.name, server=get_host_addr(host))))
def restart_kraken(instance, test=True, wait=True):
"""Restart a kraken instance on a given server
To let us not restart all kraken servers in the farm
"""
instance = get_real_instance(instance)
wait = get_bool_from_cli(wait)
if instance.name not in env.excluded_instances:
kraken = 'kraken_' + instance.name
start_or_stop_with_delay(kraken,
4000,
500,
start=False,
only_once=True)
start_or_stop_with_delay(kraken,
4000,
500,
only_once=env.KRAKEN_START_ONLY_ONCE)
if test:
test_kraken(instance.name, fail_if_error=False, wait=wait)
else:
print(yellow("{} has no data, not testing it".format(instance.name)))
def is_not_synchronized(instance, hosts=None):
"""
test if an instance has unsynchronized kraken
to test that we consider 'publication_date' of the kraken's data
for the moment it's a manually called function
"""
instance = get_real_instance(instance)
hosts = [e.split('@')[1] for e in hosts or instance.kraken_engines]
publication_dates = set()
for host in hosts:
request = 'http://{}:{}/{}/?instance={}'.format(host,
env.kraken_monitor_port, env.kraken_monitor_location_dir, instance.name)
result = _test_kraken(request, fail_if_error=False)
publication_dates.add(result.get('publication_date'))
print('publication date for {}, : {}'.format(instance, publication_dates))
if len(publication_dates) != 1:
print(red('all the {} krakens do not have the same data loaded, publication dates: {}'.format(
instance, publication_dates)))
return 1
return 0
def get_kraken_config(server, instance):
"""Get kraken configuration of a given instance"""
instance = get_real_instance(instance)
with settings(host_string=server):
config_path = "%s/%s/kraken.ini" % (env.kraken_basedir, instance.name)
# first get the configfile here
temp_file = StringIO.StringIO()
if exists(config_path):
get(config_path, temp_file)
else:
print(red("ERROR: can't find %s" % config_path))
exit(1)
config = ConfigParser.RawConfigParser(allow_no_value=True)
config.readfp(BytesIO(temp_file.getvalue()))
if 'GENERAL' in config.sections():
return config
else:
return None
def update_eng_instance_conf(instance):
instance = get_real_instance(instance)
for host in instance.kraken_engines:
with settings(host_string=host):
_upload_template("kraken/kraken.ini.jinja",
"%s/%s/kraken.ini" %
(env.kraken_basedir, instance.name),
context={
'env': env,
'instance': instance,
})
if env.use_systemd:
_upload_template("kraken/systemd_kraken.jinja",
"{}".format(
env.service_name('kraken_{}'.format(
instance.name))),
context={
'env': env,
'instance': instance.name,
'kraken_base_conf': env.kraken_basedir,
},
mode='644')
else:
_upload_template("kraken/kraken.initscript.jinja",
"{}".format(
env.service_name('kraken_{}'.format(
instance.name))),
context={
'env': env,
'instance': instance.name,
'kraken_base_conf': env.kraken_basedir,
},
mode='755')
# TODO check this, make it consistent with env.use_systemd
update_init(host='eng')
def remove_kraken_instance(instance, purge_logs=False, apply_on='engines'):
"""
Remove a kraken instance entirely
* Stop the service
* Remove startup at boot time
* Remove initscript
* Remove configuration and pid directory
apply_on values:
- engines: apply on instance.kraken_engines
- reverse: apply on all engines except instance.kraken_engines
- all: apply on all engines
"""
instance = get_real_instance(instance)
if apply_on == 'engines':
hosts, exclude_hosts = instance.kraken_engines, ()
elif apply_on == 'reverse':
hosts, exclude_hosts = env.roledefs['eng'], instance.kraken_engines
elif apply_on == 'all':
hosts, exclude_hosts = env.roledefs['eng'], ()
else:
abort("Bad 'apply_on' parameter value: {}".format(apply_on))
for host in set(hosts) - set(exclude_hosts):
with settings(host_string=host, warn_only=True):
sudo("service kraken_{} stop; sleep 3".format(instance.name))
if not env.use_systemd:
run("update-rc.d -f kraken_{} remove".format(instance.name))
sudo("rm --force {}/kraken_{}".format(env.service_path(),
instance.name))
sudo("rm --recursive --force {}/{}/".format(
env.kraken_basedir, instance.name))
if purge_logs:
# ex.: /var/log/kraken/navitia-bretagne.log
sudo("rm --force {}-{}.log".format(env.kraken_log_name,
instance.name))
def test_kraken(instance,
fail_if_error=True,
wait=False,
loaded_is_ok=None,
hosts=None):
"""Test kraken with '?instance='"""
instance = get_real_instance(instance)
wait = get_bool_from_cli(wait)
hosts = [h.split('@')[1] for h in hosts or instance.kraken_engines]
will_return = len(hosts) == 1
for host in hosts:
request = 'http://{}:{}/{}/?instance={}'.format(
host, env.kraken_monitor_port, env.kraken_monitor_location_dir,
instance.name)
if wait:
# we wait until we get a response and the instance is 'loaded'
try:
result = Retrying(stop_max_delay=env.KRAKEN_RESTART_DELAY * 1000,
wait_fixed=1000, retry_on_result=lambda x: x is None or not x['loaded']) \
.call(_test_kraken, request, fail_if_error)
except Exception as ex:
print(
red("ERROR: could not reach {}, too many retries ! ({})".
format(instance.name, ex)))
result = {'status': False}
else:
result = _test_kraken(request, fail_if_error)
try:
if result['status'] != 'running':
if result['status'] == 'no_data':
print(
yellow(
"WARNING: instance {} has no loaded data".format(
instance.name)))
if will_return:
return False
if fail_if_error:
print(
red("ERROR: Instance {} is not running ! ({})".format(
instance.name, result)))
if will_return:
return False
print(
yellow("WARNING: Instance {} is not running ! ({})".format(
instance.name, result)))
if will_return:
return False
if not result['is_connected_to_rabbitmq']:
print(
yellow("WARNING: Instance {} is not connected to rabbitmq".
format(instance.name)))
if will_return:
return False
if loaded_is_ok is None:
loaded_is_ok = wait
if not loaded_is_ok:
if result['loaded']:
print(
yellow("WARNING: instance {} has loaded data".format(
instance.name)))
if will_return:
return True
else:
print(
green("OK: instance {} has correct values: {}".format(
instance.name, result)))
if will_return:
return False
else:
if result['loaded']:
print(
green("OK: instance {} has correct values: {}".format(
instance.name, result)))
if will_return:
return True
elif fail_if_error:
abort(
red("CRITICAL: instance {} has no loaded data".format(
instance.name)))
else:
print(
yellow(
"WARNING: instance {} has no loaded data".format(
instance.name)))
if will_return:
return False
except KeyError:
print(
red("CRITICAL: instance {} does not return a correct result".
format(instance.name)))
print(result)
if fail_if_error:
abort('')
return False
