def purge_data_nav(force=False): """ purge temp/data.nav.lz4 files the whole process will be skipped as soon as a single condition is encountered: - temp data file is more recent than actual data file - temp data file exists but actual data file is missing """ if not force: print("Checking lz4 temp files purge conditions before proceeding...") reason = {} for instance in env.instances.values(): plain_target = get_real_instance(instance).target_lz4_file temp_target = os.path.join(os.path.dirname(plain_target), 'temp', os.path.basename(plain_target)) if exists(plain_target): if exists(temp_target) and files.getmtime( temp_target) > files.getmtime(plain_target): reason[instance.name] = "{} is more recent than {}".format( temp_target, plain_target) elif exists(temp_target): reason[instance.name] = "{} does not exists".format( plain_target) if reason: print(yellow("Error: Can't purge lz4 temp files, reasons:")) for k, v in reason.iteritems(): print(" {}: {}".format(k, v)) exit(1) for instance in env.instances.values(): plain_target = get_real_instance(instance).target_lz4_file temp_target = os.path.join(os.path.dirname(plain_target), 'temp', os.path.basename(plain_target)) if exists(temp_target): files.remove(temp_target)
def purge_data_nav(force=False): """ purge temp/data.nav.lz4 files the whole process will be skipped as soon as a single condition is encountered: - temp data file is more recent than actual data file - temp data file exists but actual data file is missing """ if not force: print("Checking lz4 temp files purge conditions before proceeding...") reason = {} for instance in env.instances.values(): plain_target = get_real_instance(instance).target_lz4_file temp_target = os.path.join(os.path.dirname(plain_target), 'temp', os.path.basename(plain_target)) if exists(plain_target): if exists(temp_target) and files.getmtime(temp_target) > files.getmtime(plain_target): reason[instance.name] = "{} is more recent than {}".format(temp_target, plain_target) elif exists(temp_target): reason[instance.name] = "{} does not exists".format(plain_target) if reason: print(yellow("Error: Can't purge lz4 temp files, reasons:")) for k, v in reason.iteritems(): print(" {}: {}".format(k, v)) exit(1) for instance in env.instances.values(): plain_target = get_real_instance(instance).target_lz4_file temp_target = os.path.join(os.path.dirname(plain_target), 'temp', os.path.basename(plain_target)) if exists(temp_target): files.remove(temp_target)
def remove_ed_instance(instance): """Remove a ed instance entirely""" ed_dir = get_real_instance(instance).base_ed_dir destination_dir = get_real_instance(instance).base_destination_dir backup_dir = get_real_instance(instance).backup_dir run("rm -rf %s" % ed_dir) run("rm -rf %s" % destination_dir) run("rm -rf %s" % backup_dir)
def remove_ed_instance(instance): """Remove a ed instance entirely""" ed_dir = get_real_instance(instance).base_ed_dir destination_dir = get_real_instance(instance).base_destination_dir backup_dir = get_real_instance(instance).backup_dir run("rm -rf %s" % ed_dir) run("rm -rf %s" % destination_dir) run("rm -rf %s" % backup_dir)
def remove_instance(instance, admin=False): """Completely remove all components for a given instance Remove instance in jormungandr db manually TODO: remove instance db automatically """ execute(db.remove_instance_from_jormun_database, instance) execute(db.remove_postgresql_database, get_real_instance(instance).db_name) execute(db.remove_postgresql_user, get_real_instance(instance).db_user) execute(tyr.remove_ed_instance, instance) execute(tyr.remove_tyr_instance, instance) execute(kraken.remove_kraken_instance, instance) execute(jormungandr.remove_jormungandr_instance, instance) if admin and env.use_load_balancer: execute(remove_kraken_vip, instance)
def create_privileges_instance_db(instance): postgres_database = get_real_instance(instance).db_name postgres_connection_user = '******'.format(postgres_database) for schema in env.postgres_schemas: run('{} -c "GRANT SELECT ON ALL TABLES IN SCHEMA {} TO {};"' .format(postgres_connection_user, schema, env.postgres_read_only_user))
def update_eng_instance_conf(instance): instance = get_real_instance(instance) for host in instance.kraken_engines: with settings(host_string=host): _upload_template("kraken/kraken.ini.jinja", "%s/%s/kraken.ini" % (env.kraken_basedir, instance.name), context={ 'env': env, 'instance': instance, } ) if env.use_systemd: _upload_template("kraken/systemd_kraken.jinja", "{}".format(env.service_name('kraken_{}'.format(instance.name))), context={'env': env, 'instance': instance.name, 'kraken_base_conf': env.kraken_basedir, }, mode='644' ) else: _upload_template("kraken/kraken.initscript.jinja", "{}".format(env.service_name('kraken_{}'.format(instance.name))), context={'env': env, 'instance': instance.name, 'kraken_base_conf': env.kraken_basedir, }, mode='755' ) # TODO check this, make it consistent with env.use_systemd update_init(host='eng')
def create_tyr_instance(instance): """ Create a *private* tyr instance based on the given name * postgresql user + dedicated database (1 time) * /etc/tyr.d/instance.ini * create /srv/ed/<instance> + target-file basedir """ instance = get_real_instance(instance) execute(db.create_instance_db, instance) # /srv/ed/destination/$instance & /srv/ed/backup/$instance require_directory(instance.source_dir, owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True) require_directory(instance.backup_dir, is_on_nfs4=True, owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True) require_directory(instance.base_destination_dir, is_on_nfs4=True, owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True) require_directory(env.tyr_base_logdir, owner=env.TYR_USER, group=env.TYR_USER, mode='755', use_sudo=True) require.files.file(os.path.join(env.tyr_base_logdir, instance.name + '.log'), owner=env.TYR_USER, group=env.TYR_USER, mode='644', use_sudo=True) update_tyr_instance_conf(instance) # Note it is not called as a task, for it needs to be done on the same server if not env.standalone: #@TODO: change user for non standalone pass
def restart_kraken(instance, wait='serial'): """ Restart all krakens of an instance (using pool), serially or in parallel, then test them. Testing serially assures that krakens are restarted serially. :param wait: string. Possible values=False or None: restart in parallel, no test 'serial': restart serially and test 'parallel': restart in parallel and test 'no_test': explicitely skip tests (faster but dangerous) The default value is 'serial' because it is the safest scenario to restart the krakens of an instance in production. """ if wait not in ('serial', 'parallel', 'no_test'): abort( yellow( "Error: wait parameter must be 'serial', 'parallel' or 'no_test', found '{}'" .format(wait))) instance = get_real_instance(instance) excluded = instance.name in env.excluded_instances # restart krakens of this instance that are also in the eng role, # this works with the "pool" switch mechanism used in upgrade_all() for host in set(instance.kraken_engines).intersection(env.roledefs['eng']): restart_kraken_on_host(instance, host) if wait == 'serial' and not excluded: test_kraken(instance, fail_if_error=False, wait=True, hosts=[host]) if wait == 'parallel' and not excluded: test_kraken(instance, fail_if_error=False, wait=True) if wait != 'no_test' and excluded: print( yellow("Coverage '{}' has no data, not testing it".format( instance.name))) if wait == 'no_test': print( yellow( "Warning Coverage '{}' not tested: parameter wait='no_test'". format(instance.name)))
def create_tyr_instance(instance): """ Create a *private* tyr instance based on the given name * postgresql user + dedicated database (1 time) * /etc/tyr.d/instance.ini * create /srv/ed/<instance> + target-file basedir """ instance = get_real_instance(instance) execute(db.create_instance_db, instance) # /srv/ed/destination/$instance & /srv/ed/backup/$instance require_directory(instance.source_dir, owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True) require_directory(instance.backup_dir, is_on_nfs4=True, owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True) require_directory(instance.base_destination_dir, is_on_nfs4=True, owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True) require_directory(env.tyr_base_logdir, owner=env.TYR_USER, group=env.TYR_USER, mode='755', use_sudo=True) require.files.file(os.path.join(env.tyr_base_logdir, instance.name + '.log'), owner=env.TYR_USER, group=env.TYR_USER, mode='644', use_sudo=True) update_tyr_instance_conf(instance) # Note it is not called as a task, for it needs to be done on the same server if not env.standalone: #@TODO: change user for non standalone pass
def update_eng_instance_conf(instance, host=None): instance = get_real_instance(instance) hosts = [host] if host else instance.kraken_engines for host in hosts: with settings(host_string=host): require.files.directory(os.path.join(instance.kraken_basedir, instance.name), owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True) _upload_template("kraken/kraken.ini.jinja", "%s/%s/kraken.ini" % (env.kraken_basedir, instance.name), context={ 'env': env, 'instance': instance, } ) if env.use_systemd: _upload_template("kraken/systemd_kraken.jinja", "{}".format(env.service_name('kraken_{}'.format(instance.name))), context={'env': env, 'instance': instance.name, 'kraken_base_conf': env.kraken_basedir, }, mode='644' ) else: _upload_template("kraken/kraken.initscript.jinja", "{}".format(env.service_name('kraken_{}'.format(instance.name))), context={'env': env, 'instance': instance.name, 'kraken_base_conf': env.kraken_basedir, }, mode='755' ) # TODO check this, make it consistent with env.use_systemd update_init(host='eng')
def delete_kraken_queue_to_rabbitmq(instance, apply_on='reverse'): """ Remove queue for a kraken """ instance = get_real_instance(instance) if apply_on == 'engines': hosts, exclude_hosts = instance.kraken_engines, () elif apply_on == 'reverse': hosts, exclude_hosts = env.roledefs['eng'], instance.kraken_engines elif apply_on == 'all': hosts, exclude_hosts = env.roledefs['eng'], () else: abort("Bad 'apply_on' parameter value: {}".format(apply_on)) if env.rabbitmq_host_api == 'localhost': host_string = env.roledefs['tyr_master'][0] else: host_string = env.rabbitmq_host_api for host in set(hosts) - set(exclude_hosts): with settings(host_string=host_string): run('curl -i -u {}:{} -XDELETE "http://localhost:{}/api/queues/%2F/kraken_{}_{}_rt"' .format(env.rabbitmq_user, env.rabbitmq_pass, env.rabbitmq_port_api, get_host_addr(host).split('.')[0], instance)) run('curl -i -u {}:{} -XDELETE "http://localhost:{}/api/queues/%2F/kraken_{}_{}_task"' .format(env.rabbitmq_user, env.rabbitmq_pass, env.rabbitmq_port_api, get_host_addr(host).split('.')[0], instance))
def delete_kraken_queue_to_rabbitmq(instance, apply_on='reverse'): """ Remove queue for a kraken """ instance = get_real_instance(instance) if apply_on == 'engines': hosts, exclude_hosts = instance.kraken_engines, () elif apply_on == 'reverse': hosts, exclude_hosts = env.roledefs['eng'], instance.kraken_engines elif apply_on == 'all': hosts, exclude_hosts = env.roledefs['eng'], () else: abort("Bad 'apply_on' parameter value: {}".format(apply_on)) host_string = env.rabbitmq_kraken_host for host in set(hosts) - set(exclude_hosts): with settings(host_string=host_string): run('curl -i -u {}:{} -XDELETE "http://localhost:{}/api/queues/{}/kraken_{}_{}_rt"' .format(env.rabbitmq_kraken_user, env.rabbitmq_kraken_pass, env.rabbitmq_port_api, rabbitmq_kraken_vhost, get_host_addr(host).split('.')[0], instance)) run('curl -i -u {}:{} -XDELETE "http://localhost:{}/api/queues/{}/kraken_{}_{}_task"' .format(env.rabbitmq_kraken_user, env.rabbitmq_kraken_pass, env.rabbitmq_port_api, rabbitmq_kraken_vhost, get_host_addr(host).split('.')[0], instance))
def restart_kraken(instance, wait='serial'): """ Restart all krakens of an instance (using pool), serially or in parallel, then test them. Testing serially assures that krakens are restarted serially. :param wait: string. Possible values=False or None: restart in parallel, no test 'serial': restart serially and test 'parallel': restart in parallel and test 'no_test': explicitely skip tests (faster but dangerous) The default value is 'serial' because it is the safest scenario to restart the krakens of an instance in production. """ if wait not in ('serial', 'parallel', 'no_test'): abort(yellow("Error: wait parameter must be 'serial', 'parallel' or 'no_test', found '{}'".format(wait))) instance = get_real_instance(instance) excluded = instance.name in env.excluded_instances # restart krakens of this instance that are also in the eng role, # this works with the "pool" switch mechanism used in upgrade_all() for host in set(instance.kraken_engines).intersection(env.roledefs['eng']): restart_kraken_on_host(instance, host) if wait == 'serial' and not excluded: test_kraken(instance, fail_if_error=False, wait=True, hosts=[host]) if wait == 'parallel' and not excluded: test_kraken(instance, fail_if_error=False, wait=True) if wait != 'no_test' and excluded: print(yellow("Coverage '{}' has no data, not testing it".format(instance.name))) if wait == 'no_test': print(yellow("Warning Coverage '{}' not tested: parameter wait='no_test'".format(instance.name)))
def get_kraken_config(server, instance): """Get kraken configuration of a given instance""" # TODO this task is never used and it looks like a function (inconsistent) instance = get_real_instance(instance) with settings(host_string=env.make_ssh_url(server)): config_path = "%s/%s/kraken.ini" % (env.kraken_basedir, instance.name) # first get the configfile here temp_file = StringIO.StringIO() if exists(config_path): get(config_path, temp_file) else: print(red("ERROR: can't find %s" % config_path)) exit(1) config = ConfigParser.RawConfigParser(allow_no_value=True) config_text = temp_file.getvalue() config.readfp(BytesIO(config_text)) if 'GENERAL' in config.sections(): return config else: return None
def get_kraken_config(server, instance): """Get kraken configuration of a given instance""" # TODO this task is never used and it looks like a function (inconsistent) instance = get_real_instance(instance) with settings(host_string=env.make_ssh_url(server)): config_path = "%s/%s/kraken.ini" % (env.kraken_basedir, instance.name) # first get the configfile here temp_file = StringIO.StringIO() if exists(config_path): get(config_path, temp_file) else: print(red("ERROR: can't find %s" % config_path)) exit(1) config = ConfigParser.RawConfigParser(allow_no_value=True) config_text = temp_file.getvalue() config.readfp(BytesIO(config_text)) if 'GENERAL' in config.sections(): return config else: return None
def remove_jormungandr_instance(instance): """ Remove a jormungandr instance entirely * Remove json file which declare the instance * Reload apache """ instance = get_real_instance(instance) run("rm --force {}".format(instance.jormungandr_config_file))
def update_eng_instance_conf(instance, host=None): instance = get_real_instance(instance) hosts = [host] if host else instance.kraken_engines for host in hosts: with settings(host_string=host): require.files.directory(os.path.join(instance.kraken_basedir, instance.name), owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True) _upload_template("kraken/kraken.ini.jinja", "%s/%s/kraken.ini" % (env.kraken_basedir, instance.name), context={ 'env': env, 'instance': instance, } ) if env.use_systemd: _upload_template("kraken/systemd_kraken.jinja", "{}".format(env.service_name('kraken_{}'.format(instance.name))), context={'env': env, 'instance': instance.name, 'kraken_base_conf': env.kraken_basedir, }, mode='644' ) else: _upload_template("kraken/kraken.initscript.jinja", "{}".format(env.service_name('kraken_{}'.format(instance.name))), context={'env': env, 'instance': instance.name, 'kraken_base_conf': env.kraken_basedir, }, mode='755' ) # TODO check this, make it consistent with env.use_systemd update_init(host='eng')
def deploy_jormungandr_instance_conf(instance): """ Deploy or redeploy one jormungander coverage: * Deploy the json configuration file * Do not reload apache """ instance = get_real_instance(instance) config = { 'key': instance.name, 'zmq_socket': instance.jormungandr_zmq_socket_for_instance, 'realtime_proxies': instance.realtime_proxies } if instance.pt_zmq_socket: config['pt_zmq_socket'] = instance.pt_zmq_socket if instance.street_network: config["street_network"] = instance.street_network if instance.ridesharing: config["ridesharing"] = instance.ridesharing if instance.autocomplete: config["default_autocomplete"] = instance.autocomplete if instance.equipment_details_providers: config[ "equipment_details_providers"] = instance.equipment_details_providers _upload_template("jormungandr/instance.json.jinja", instance.jormungandr_config_file, context={'json': json.dumps(config, indent=4)}, use_sudo=True)
def remove_kraken_instance(instance, purge_logs=False, apply_on='engines'): """ Remove a kraken instance entirely * Stop the service * Remove startup at boot time * Remove initscript * Remove configuration and pid directory apply_on values: - engines: apply on instance.kraken_engines - reverse: apply on all engines except instance.kraken_engines - all: apply on all engines """ instance = get_real_instance(instance) if apply_on == 'engines': hosts, exclude_hosts = instance.kraken_engines, () elif apply_on == 'reverse': hosts, exclude_hosts = env.roledefs['eng'], instance.kraken_engines elif apply_on == 'all': hosts, exclude_hosts = env.roledefs['eng'], () else: abort("Bad 'apply_on' parameter value: {}".format(apply_on)) for host in set(hosts) - set(exclude_hosts): with settings( host_string=host, warn_only=True ): sudo("service kraken_{} stop && sleep 3".format(instance.name)) if not env.use_systemd: run("update-rc.d -f kraken_{} remove".format(instance.name)) sudo("rm -f {}/kraken_{}".format(env.service_path(), instance.name)) sudo("rm -rf {}/{}/".format(env.kraken_basedir, instance.name)) if purge_logs: sudo("rm -f {}/{}.log".format(env.kraken_log_basedir, instance.name))
def create_eng_instance(instance): """ Create a new kraken instance (idempotent) * Install requirements * Deploy the binary, the templatized ini configuration in a dedicated directory with rights to www-data and the logdir * Deploy initscript and add it to startup * Start the service """ instance = get_real_instance(instance) for host in instance.kraken_engines: with settings(host_string=host): # base_conf require.files.directory(instance.kraken_basedir, owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True) # logs require.files.directory(env.kraken_log_basedir, owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True) update_eng_instance_conf(instance, host) # kraken.ini, pid and binary symlink kraken_bin = "{}/{}/kraken".format(env.kraken_basedir, instance.name) if not is_link(kraken_bin): idempotent_symlink("/usr/bin/kraken", kraken_bin, use_sudo=True) sudo('chown -h {user} {bin}'.format(user=env.KRAKEN_USER, bin=kraken_bin)) kraken = "kraken_{}".format(instance.name) if not service.is_running(kraken): # TODO test this on systemd machines if env.use_systemd: sudo("systemctl enable kraken_{}.service".format( instance.name)) else: sudo("update-rc.d kraken_{} defaults".format( instance.name)) print( blue( "INFO: kraken {instance} instance is starting on {server}, " "waiting 5 seconds, we will check if processus is running" .format(instance=instance.name, server=get_host_addr(env.host_string)))) service.start(kraken) run("sleep 5") # we wait a bit for the kraken to pop with settings(warn_only=True): run("pgrep --list-name --full /srv/kraken/{}/kraken".format( instance.name)) print( blue("INFO: kraken {instance} instance is running on {server}". format(instance=instance.name, server=get_host_addr(host))))
def rollback_instance(instance, test=True): """ Use this only if something goes wrong during deployment of an instance """ test = get_bool_from_cli(test) instance = get_real_instance(instance) execute(swap_data_nav, instance, force=True) execute(set_kraken_binary, instance, old=True) execute(restart_kraken, instance, wait=env.KRAKEN_RESTART_SCHEME if test else 'no_test')
def restart_kraken_on_host(instance, host): """ Restart a kraken of an instance on a given server """ instance = get_real_instance(instance) with settings(host_string=host): kraken = 'kraken_' + instance.name start_or_stop_with_delay(kraken, 4000, 500, start=False, only_once=True) start_or_stop_with_delay(kraken, 4000, 500, only_once=env.KRAKEN_START_ONLY_ONCE)
def require_kraken_started(instance): """start a kraken instance on all servers if it is not already started """ instance = get_real_instance(instance) kraken = 'kraken_' + instance.name for host in instance.kraken_engines: with settings(host_string=host): start_or_stop_with_delay(kraken, 4000, 500, only_once=True)
def stop_kraken(instance): """Stop a kraken instance on all servers """ instance = get_real_instance(instance) kraken = 'kraken_' + instance.name for host in instance.kraken_engines: with settings(host_string=host): start_or_stop_with_delay(kraken, 4000, 500, start=False, only_once=True)
def stop_kraken(instance): """Stop a kraken instance on all servers """ instance = get_real_instance(instance) kraken = 'kraken_' + instance.name for host in instance.kraken_engines: with settings(host_string=host): start_or_stop_with_delay(kraken, 4000, 500, start=False, only_once=True)
def restart_kraken_on_host(instance, host): """ Restart a kraken of an instance on a given server """ instance = get_real_instance(instance) with settings(host_string=host): kraken = 'kraken_' + instance.name start_or_stop_with_delay(kraken, 4000, 500, start=False, only_once=True) start_or_stop_with_delay(kraken, 4000, 500, only_once=env.KRAKEN_START_ONLY_ONCE)
def create_privileges_instance_db(instance): postgres_database = get_real_instance(instance).db_name postgres_connection_user = '******'.format( postgres_database) for schema in env.postgres_schemas: run('{} -c "GRANT SELECT ON ALL TABLES IN SCHEMA {} TO {};"'.format( postgres_connection_user, schema, env.postgres_read_only_user))
def rollback_instance(instance, test=True): """ Use this only if something goes wrong during deployment of an instance """ test = get_bool_from_cli(test) instance = get_real_instance(instance) execute(swap_data_nav, instance, force=True) execute(set_kraken_binary, instance, old=True) execute(restart_kraken, instance, wait=env.KRAKEN_RESTART_SCHEME if test else 'no_test')
def require_kraken_started(instance): """start a kraken instance on all servers if it is not already started """ instance = get_real_instance(instance) kraken = 'kraken_' + instance.name for host in instance.kraken_engines: with settings(host_string=host): start_or_stop_with_delay(kraken, 4000, 500, only_once=True)
def set_kraken_binary(instance, old=False): instance = get_real_instance(instance) for host in instance.kraken_engines: with settings(host_string=host): kraken_bin = "{}/{}/kraken".format(env.kraken_basedir, instance.name) idempotent_symlink('/usr/bin/kraken' + ('.old' if old else ''), kraken_bin, use_sudo=True)
def remove_jormungandr_instance(instance): """ Remove a jormungandr instance entirely * Remove json file which declare the instance * Reload apache """ instance = get_real_instance(instance) run("rm --force %s" % (instance.jormungandr_config_file)) reload_jormun_safe_all()
def test_kraken(instance, fail_if_error=True, wait=False, loaded_is_ok=None): """Test kraken with '?instance='""" instance = get_real_instance(instance) wait = get_bool_from_cli(wait) # env.host will call the monitor kraken on the current host request = Request('http://{}:{}/{}/?instance={}'.format(env.host, env.kraken_monitor_port, env.kraken_monitor_location_dir, instance.name)) if wait: # we wait until we get a gestion and the instance is 'loaded' try: result = Retrying(stop_max_delay=env.KRAKEN_RESTART_DELAY * 1000, wait_fixed=1000, retry_on_result=lambda x: x is None or not x['loaded']) \ .call(_test_kraken, request, fail_if_error) except Exception as e: print(red("ERROR: could not reach {}, too many retries ! ({})".format(instance.name, e))) result = {'status': False} else: result = _test_kraken(request, fail_if_error) if result['status'] != 'running': if result['status'] == 'no_data': print(yellow("WARNING: instance {} has no loaded data".format(instance.name))) return False if fail_if_error: print(red("ERROR: Instance {} is not running ! ({})".format(instance.name, result))) return False print(yellow("WARNING: Instance {} is not running ! ({})".format(instance.name, result))) return False if not result['is_connected_to_rabbitmq']: print(yellow("WARNING: Instance {} is not connected to rabbitmq".format(instance.name))) return False if loaded_is_ok is None: loaded_is_ok = wait if not loaded_is_ok: if result['loaded']: print(yellow("WARNING: instance {} has loaded data".format(instance.name))) return True else: print(green("OK: instance {} has correct values: {}".format(instance.name, result))) return False else: if result['loaded']: print(green("OK: instance {} has correct values: {}".format(instance.name, result))) return True elif fail_if_error: print(red("CRITICAL: instance {} has no loaded data".format(instance.name))) exit(1) else: print(yellow("WARNING: instance {} has no loaded data".format(instance.name))) return False
def create_eng_instance(instance): """Create a new kraken instance * Install requirements (idem potem) * Deploy the binary, the templatized ini configuration in a dedicated directory with rights to www-data and the logdir * Deploy initscript and add it to startup * Start the service """ instance = get_real_instance(instance) for host in instance.kraken_engines: with settings(host_string=host): # base_conf require.files.directory(instance.kraken_basedir, owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True) # logs require.files.directory(env.kraken_log_basedir, owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True) update_eng_instance_conf(instance) # kraken.ini, pid and binary symlink kraken_bin = "{}/{}/kraken".format(env.kraken_basedir, instance.name) if not is_link(kraken_bin): files.symlink("/usr/bin/kraken", kraken_bin, use_sudo=True) sudo('chown -h {user} {bin}'.format(user=env.KRAKEN_USER, bin=kraken_bin)) #run("chmod 755 /etc/init.d/kraken_{}".format(instance)) # TODO refactor this and test it on systemd and non-systemd machines if not env.use_systemd: sudo("update-rc.d kraken_{} defaults".format(instance.name)) print( blue( "INFO: Kraken {instance} instance is starting on {server}, " "waiting 5 seconds, we will check if processus is running". format(instance=instance.name, server=get_host_addr(env.host_string)))) service.start("kraken_{}".format(instance.name)) run("sleep 5") # we wait a bit for the kraken to pop # test it ! # execute(test_kraken, get_host_addr(env.host_string), instance, fail_if_error=False) with settings(warn_only=True): run("pgrep --list-name --full {}".format(instance.name)) print( blue("INFO: kraken {instance} instance is running on {server}". format(instance=instance.name, server=get_host_addr(env.host_string))))
def redeploy_kraken(instance, create=True): """ Redistributes an existing kraken on eng engines. Call this task when the zmq_server parameter of add_instance is changed. Use create=False if krakens mapping is reduced (this avoids restarting them). Use create=True if krakens are displaced or mapping is expanded. """ instance = get_real_instance(instance) create = get_bool_from_cli(create) if create: execute(create_eng_instance, instance) execute(remove_kraken_instance, instance, purge_logs=True, apply_on='reverse')
def redeploy_kraken(instance, create=True): """ Redistributes an existing kraken on eng engines. Call this task when the zmq_server parameter of add_instance is changed. Use create=False if krakens mapping is reduced (this avoids restarting them). Use create=True if krakens are displaced or mapping is expanded. """ instance = get_real_instance(instance) create = get_bool_from_cli(create) if create: execute(create_eng_instance, instance) execute(remove_kraken_instance, instance, purge_logs=True, apply_on='reverse')
def restart_kraken(instance, test=True, wait=True): """Restart a kraken instance on a given server To let us not restart all kraken servers in the farm """ instance = get_real_instance(instance) wait = get_bool_from_cli(wait) if instance.name not in env.excluded_instances: for host in set(instance.kraken_engines).intersection(env.roledefs['eng']): restart_kraken_on_host(instance, host) if test: test_kraken(instance, fail_if_error=False, wait=wait) else: print(yellow("{} has no data, not testing it".format(instance.name)))
def swap_data_nav(instance, force=False): """ swap old/new data.nav.lz4, only if new is still in temp directory """ plain_target = get_real_instance(instance).target_lz4_file temp_target = os.path.join(os.path.dirname(plain_target), 'temp', os.path.basename(plain_target)) if exists(plain_target): if exists(temp_target) and \ (force or (files.getmtime(temp_target) > files.getmtime(plain_target))): swap_temp = os.path.join(os.path.dirname(temp_target), 'x') files.move(plain_target, swap_temp) files.move(temp_target, plain_target) files.move(swap_temp, temp_target) elif exists(temp_target): files.move(temp_target, plain_target)
def restart_kraken(instance, test=True, wait=True): """Restart a kraken instance on a given server To let us not restart all kraken servers in the farm """ instance = get_real_instance(instance) wait = get_bool_from_cli(wait) if instance.name not in env.excluded_instances: kraken = 'kraken_' + instance.name start_or_stop_with_delay(kraken, 4000, 500, start=False, only_once=True) start_or_stop_with_delay(kraken, 4000, 500, only_once=env.KRAKEN_START_ONLY_ONCE) if test: test_kraken(instance.name, fail_if_error=False, wait=wait) else: print(yellow("{} has no data, not testing it".format(instance.name)))
def swap_data_nav(instance, force=False): """ swap old/new data.nav.lz4, only if new is still in temp directory """ plain_target = get_real_instance(instance).target_lz4_file temp_target = os.path.join(os.path.dirname(plain_target), 'temp', os.path.basename(plain_target)) if exists(plain_target): if exists(temp_target) and \ (force or (files.getmtime(temp_target) > files.getmtime(plain_target))): swap_temp = os.path.join(os.path.dirname(temp_target), 'x') files.move(plain_target, swap_temp) files.move(temp_target, plain_target) files.move(swap_temp, temp_target) elif exists(temp_target): files.move(temp_target, plain_target)
def update_instance(instance): """ param (instance) - update all configuration and restart all services does not deploy any packages """ instance = utils.get_real_instance(instance) # since it might be a endpoint we might need to get the real instance print(blue('updating {}'.format(instance.name))) #first of all we compute the instance status, it will be helpfull later execute(utils.compute_instance_status, instance) execute(tyr.create_tyr_instance, instance) execute(db.postgis_initdb, instance.db_name) execute(tyr.update_ed_db, instance.name) execute(jormungandr.deploy_jormungandr_instance_conf, instance) execute(kraken.create_eng_instance, instance) execute(tyr.deploy_default_synonyms, instance)
def deploy_jormungandr_instance_conf(instance): instance = get_real_instance(instance) config = {'key': instance.name, 'zmq_socket': instance.jormungandr_zmq_socket_for_instance} config['realtime_proxies'] = instance.realtime_proxies _upload_template("jormungandr/instance.json.jinja", instance.jormungandr_config_file, context={ 'json': json.dumps(config, indent=4) }, use_sudo=True ) # the old configuration file were .ini, now it's json, we need to clean up if fabtools.files.is_file(instance.jormungandr_old_ini_config_file): fabtools.files.remove(instance.jormungandr_old_ini_config_file)
def deploy_jormungandr_instance_conf(instance): instance = get_real_instance(instance) config = {'key': instance.name, 'zmq_socket': instance.jormungandr_zmq_socket_for_instance} config['realtime_proxies'] = instance.realtime_proxies _upload_template("jormungandr/instance.json.jinja", instance.jormungandr_config_file, context={ 'json': json.dumps(config, indent=4) }, use_sudo=True ) # the old configuration file were .ini, now it's json, we need to clean up if fabtools.files.is_file(instance.jormungandr_old_ini_config_file): fabtools.files.remove(instance.jormungandr_old_ini_config_file)
def deploy_jormungandr_instance_conf(instance): """ Deploy or redeploy one jormungander coverage: * Deploy the json configuration file * Do not reload apache """ instance = get_real_instance(instance) config = {'key': instance.name, 'zmq_socket': instance.jormungandr_zmq_socket_for_instance, 'realtime_proxies': instance.realtime_proxies} _upload_template("jormungandr/instance.json.jinja", instance.jormungandr_config_file, context={ 'json': json.dumps(config, indent=4) }, use_sudo=True )
def remove_kraken_instance(instance, purge_logs=False): """Remove a kraken instance entirely * Stop the service * Remove startup at boot time * Remove initscript * Remove configuration and pid directory """ instance = get_real_instance(instance) sudo("service kraken_%s stop; sleep 3" % instance.name) run("update-rc.d -f kraken_%s remove" % instance.name) run("rm --force /etc/init.d/kraken_%s" % instance.name) run("rm --recursive --force %s/%s/" % (env.kraken_basedir, instance.name)) if purge_logs: # ex.: /var/log/kraken/navitia-bretagne.log run("rm --force %s-%s.log" % (env.kraken_log_name, instance.name))
def update_eng_instance_conf(instance): instance = get_real_instance(instance) _upload_template("kraken/kraken.ini.jinja", "%s/%s/kraken.ini" % (env.kraken_basedir, instance.name), context={ 'env': env, 'instance': instance, }) _upload_template("kraken/kraken.initscript.jinja", "/etc/init.d/kraken_%s" % instance.name, context={ 'env': env, 'instance': instance.name, 'kraken_base_conf': env.kraken_basedir, }, mode='755')
def remove_kraken_instance(instance, purge_logs=False): """Remove a kraken instance entirely * Stop the service * Remove startup at boot time * Remove initscript * Remove configuration and pid directory """ instance = get_real_instance(instance) sudo("service kraken_%s stop; sleep 3" % instance.name) run("update-rc.d -f kraken_%s remove" % instance.name) run("rm --force /etc/init.d/kraken_%s" % instance.name) run("rm --recursive --force %s/%s/" % (env.kraken_basedir, instance.name)) if purge_logs: # ex.: /var/log/kraken/navitia-bretagne.log run("rm --force %s-%s.log" % (env.kraken_log_name, instance.name))
def update_instance(instance): """ param (instance) - update all configuration and restart all services does not deploy any packages """ instance = utils.get_real_instance( instance ) # since it might be a endpoint we might need to get the real instance print(blue('updating {}'.format(instance.name))) #first of all we compute the instance status, it will be helpfull later execute(utils.compute_instance_status, instance) execute(tyr.create_tyr_instance, instance) execute(db.postgis_initdb, instance.db_name) execute(tyr.update_ed_db, instance.name) execute(jormungandr.deploy_jormungandr_instance_conf, instance) execute(kraken.create_eng_instance, instance) execute(tyr.deploy_default_synonyms, instance)
def update_eng_instance_conf(instance): instance = get_real_instance(instance) _upload_template("kraken/kraken.ini.jinja", "%s/%s/kraken.ini" % (env.kraken_basedir, instance.name), context={ 'env': env, 'instance': instance, } ) _upload_template("kraken/kraken.initscript.jinja", "/etc/init.d/kraken_%s" % instance.name, context={'env': env, 'instance': instance.name, 'kraken_base_conf': env.kraken_basedir, }, mode='755' )
def create_eng_instance(instance): """Create a new kraken instance * Install requirements (idem potem) * Deploy the binary, the templatized ini configuration in a dedicated directory with rights to www-data and the logdir * Deploy initscript and add it to startup * Start the service """ instance = get_real_instance(instance) # base_conf require.files.directory(instance.kraken_basedir, owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True) # logs require.files.directory(env.kraken_log_basedir, owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True) require.files.directory(instance.base_destination_dir, owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True) update_eng_instance_conf(instance) # kraken.ini, pid and binary symlink if not exists("{}/{}/kraken".format(env.kraken_basedir, instance.name)): kraken_bin = "{}/{}/kraken".format(env.kraken_basedir, instance.name) files.symlink("/usr/bin/kraken", kraken_bin, use_sudo=True) sudo('chown {user} {bin}'.format(user=env.KRAKEN_USER, bin=kraken_bin)) #run("chmod 755 /etc/init.d/kraken_{}".format(instance)) sudo("update-rc.d kraken_{} defaults".format(instance.name)) print(blue("INFO: Kraken {instance} instance is starting on {server}, " "waiting 5 seconds, we will check if processus is running".format( instance=instance.name, server=get_host_addr(env.host_string)))) service.start("kraken_{} start".format(instance.name)) run("sleep 5") # we wait a bit for the kraken to pop # test it ! # execute(test_kraken, get_host_addr(env.host_string), instance, fail_if_error=False) print("server: {}".format(env.host_string)) run("pgrep --list-name --full {}".format(instance.name)) print(blue("INFO: kraken {instance} instance is running on {server}". format(instance=instance.name, server=get_host_addr(env.host_string))))
def deploy_jormungandr_instance_conf(instance): """ Deploy or redeploy one jormungander coverage: * Deploy the json configuration file * Do not reload apache """ instance = get_real_instance(instance) config = { "key": instance.name, "zmq_socket": instance.jormungandr_zmq_socket_for_instance, "realtime_proxies": instance.realtime_proxies, } if instance.street_network: config["street_network"] = instance.street_network _upload_template( "jormungandr/instance.json.jinja", instance.jormungandr_config_file, context={"json": json.dumps(config, indent=4)}, use_sudo=True, )
def remove_kraken_instance(instance, purge_logs=False, apply_on='engines'): """ Remove a kraken instance entirely * Stop the service * Remove startup at boot time * Remove initscript * Remove configuration and pid directory apply_on values: - engines: apply on instance.kraken_engines - reverse: apply on all engines except instance.kraken_engines - all: apply on all engines """ instance = get_real_instance(instance) if apply_on == 'engines': hosts, exclude_hosts = instance.kraken_engines, () elif apply_on == 'reverse': hosts, exclude_hosts = env.roledefs['eng'], instance.kraken_engines elif apply_on == 'all': hosts, exclude_hosts = env.roledefs['eng'], () else: abort("Bad 'apply_on' parameter value: {}".format(apply_on)) for host in set(hosts) - set(exclude_hosts): with settings(host_string=host, warn_only=True): print("INFO: removing kraken instance {} from {}".format( instance.name, get_host_addr(host))) service.stop('kraken_{}'.format(instance.name)) run("sleep 3") # TODO test this on systemd machines if env.use_systemd: run("systemctl disable kraken_{}.service".format( instance.name)) run('systemctl daemon-reload') run("rm -f {}/kraken_{}.service".format( env.service_path(), instance.name)) else: run("rm -f {}/kraken_{}".format(env.service_path(), instance.name)) run("rm -rf {}/{}/".format(env.kraken_basedir, instance.name)) if purge_logs: run("rm -f {}/{}.log".format(env.kraken_log_basedir, instance.name))
def update_instance(instance, reload_jormun=True): """ param (instance) - update all configuration and restart all services does not deploy any packages """ instance = get_real_instance(instance) reload_jormun = get_bool_from_cli(reload_jormun) print(blue('updating {}'.format(instance.name))) #first of all we compute the instance status, it will be helpfull later execute(compute_instance_status, instance) execute(tyr.create_tyr_instance, instance) execute(db.postgis_initdb, instance.db_name) execute(tyr.update_ed_db, instance.name) execute(jormungandr.deploy_jormungandr_instance_conf, instance) execute(kraken.create_eng_instance, instance) execute(tyr.deploy_default_synonyms, instance) execute(db.create_privileges_instance_db, instance) if reload_jormun: execute(jormungandr.reload_jormun_safe_all)
def create_eng_instance(instance): """ Create a new kraken instance (idempotent) * Install requirements * Deploy the binary, the templatized ini configuration in a dedicated directory with rights to www-data and the logdir * Deploy initscript and add it to startup * Start the service """ instance = get_real_instance(instance) for host in instance.kraken_engines: with settings(host_string=host): # base_conf require.files.directory(instance.kraken_basedir, owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True) # logs require.files.directory(env.kraken_log_basedir, owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True) update_eng_instance_conf(instance, host) # kraken.ini, pid and binary symlink kraken_bin = "{}/{}/kraken".format(env.kraken_basedir, instance.name) if not is_link(kraken_bin): idempotent_symlink("/usr/bin/kraken", kraken_bin, use_sudo=True) sudo('chown -h {user} {bin}'.format(user=env.KRAKEN_USER, bin=kraken_bin)) kraken = "kraken_{}".format(instance.name) if not service.is_running(kraken): # TODO test this on systemd machines if env.use_systemd: sudo("systemctl enable kraken_{}.service".format(instance.name)) else: sudo("update-rc.d kraken_{} defaults".format(instance.name)) print(blue("INFO: kraken {instance} instance is starting on {server}, " "waiting 5 seconds, we will check if processus is running" .format(instance=instance.name, server=get_host_addr(env.host_string)))) service.start(kraken) run("sleep 5") # we wait a bit for the kraken to pop with settings(warn_only=True): run("pgrep --list-name --full /srv/kraken/{}/kraken".format(instance.name)) print(blue("INFO: kraken {instance} instance is running on {server}". format(instance=instance.name, server=get_host_addr(host))))
def restart_kraken(instance, test=True, wait=True): """Restart a kraken instance on a given server To let us not restart all kraken servers in the farm """ instance = get_real_instance(instance) wait = get_bool_from_cli(wait) if instance.name not in env.excluded_instances: kraken = 'kraken_' + instance.name start_or_stop_with_delay(kraken, 4000, 500, start=False, only_once=True) start_or_stop_with_delay(kraken, 4000, 500, only_once=env.KRAKEN_START_ONLY_ONCE) if test: test_kraken(instance.name, fail_if_error=False, wait=wait) else: print(yellow("{} has no data, not testing it".format(instance.name)))
def is_not_synchronized(instance, hosts=None): """ test if an instance has unsynchronized kraken to test that we consider 'publication_date' of the kraken's data for the moment it's a manually called function """ instance = get_real_instance(instance) hosts = [e.split('@')[1] for e in hosts or instance.kraken_engines] publication_dates = set() for host in hosts: request = 'http://{}:{}/{}/?instance={}'.format(host, env.kraken_monitor_port, env.kraken_monitor_location_dir, instance.name) result = _test_kraken(request, fail_if_error=False) publication_dates.add(result.get('publication_date')) print('publication date for {}, : {}'.format(instance, publication_dates)) if len(publication_dates) != 1: print(red('all the {} krakens do not have the same data loaded, publication dates: {}'.format( instance, publication_dates))) return 1 return 0
def get_kraken_config(server, instance): """Get kraken configuration of a given instance""" instance = get_real_instance(instance) with settings(host_string=server): config_path = "%s/%s/kraken.ini" % (env.kraken_basedir, instance.name) # first get the configfile here temp_file = StringIO.StringIO() if exists(config_path): get(config_path, temp_file) else: print(red("ERROR: can't find %s" % config_path)) exit(1) config = ConfigParser.RawConfigParser(allow_no_value=True) config.readfp(BytesIO(temp_file.getvalue())) if 'GENERAL' in config.sections(): return config else: return None
def update_eng_instance_conf(instance): instance = get_real_instance(instance) for host in instance.kraken_engines: with settings(host_string=host): _upload_template("kraken/kraken.ini.jinja", "%s/%s/kraken.ini" % (env.kraken_basedir, instance.name), context={ 'env': env, 'instance': instance, }) if env.use_systemd: _upload_template("kraken/systemd_kraken.jinja", "{}".format( env.service_name('kraken_{}'.format( instance.name))), context={ 'env': env, 'instance': instance.name, 'kraken_base_conf': env.kraken_basedir, }, mode='644') else: _upload_template("kraken/kraken.initscript.jinja", "{}".format( env.service_name('kraken_{}'.format( instance.name))), context={ 'env': env, 'instance': instance.name, 'kraken_base_conf': env.kraken_basedir, }, mode='755') # TODO check this, make it consistent with env.use_systemd update_init(host='eng')
def remove_kraken_instance(instance, purge_logs=False, apply_on='engines'): """ Remove a kraken instance entirely * Stop the service * Remove startup at boot time * Remove initscript * Remove configuration and pid directory apply_on values: - engines: apply on instance.kraken_engines - reverse: apply on all engines except instance.kraken_engines - all: apply on all engines """ instance = get_real_instance(instance) if apply_on == 'engines': hosts, exclude_hosts = instance.kraken_engines, () elif apply_on == 'reverse': hosts, exclude_hosts = env.roledefs['eng'], instance.kraken_engines elif apply_on == 'all': hosts, exclude_hosts = env.roledefs['eng'], () else: abort("Bad 'apply_on' parameter value: {}".format(apply_on)) for host in set(hosts) - set(exclude_hosts): with settings(host_string=host, warn_only=True): sudo("service kraken_{} stop; sleep 3".format(instance.name)) if not env.use_systemd: run("update-rc.d -f kraken_{} remove".format(instance.name)) sudo("rm --force {}/kraken_{}".format(env.service_path(), instance.name)) sudo("rm --recursive --force {}/{}/".format( env.kraken_basedir, instance.name)) if purge_logs: # ex.: /var/log/kraken/navitia-bretagne.log sudo("rm --force {}-{}.log".format(env.kraken_log_name, instance.name))
def test_kraken(instance, fail_if_error=True, wait=False, loaded_is_ok=None, hosts=None): """Test kraken with '?instance='""" instance = get_real_instance(instance) wait = get_bool_from_cli(wait) hosts = [h.split('@')[1] for h in hosts or instance.kraken_engines] will_return = len(hosts) == 1 for host in hosts: request = 'http://{}:{}/{}/?instance={}'.format( host, env.kraken_monitor_port, env.kraken_monitor_location_dir, instance.name) if wait: # we wait until we get a response and the instance is 'loaded' try: result = Retrying(stop_max_delay=env.KRAKEN_RESTART_DELAY * 1000, wait_fixed=1000, retry_on_result=lambda x: x is None or not x['loaded']) \ .call(_test_kraken, request, fail_if_error) except Exception as ex: print( red("ERROR: could not reach {}, too many retries ! ({})". format(instance.name, ex))) result = {'status': False} else: result = _test_kraken(request, fail_if_error) try: if result['status'] != 'running': if result['status'] == 'no_data': print( yellow( "WARNING: instance {} has no loaded data".format( instance.name))) if will_return: return False if fail_if_error: print( red("ERROR: Instance {} is not running ! ({})".format( instance.name, result))) if will_return: return False print( yellow("WARNING: Instance {} is not running ! ({})".format( instance.name, result))) if will_return: return False if not result['is_connected_to_rabbitmq']: print( yellow("WARNING: Instance {} is not connected to rabbitmq". format(instance.name))) if will_return: return False if loaded_is_ok is None: loaded_is_ok = wait if not loaded_is_ok: if result['loaded']: print( yellow("WARNING: instance {} has loaded data".format( instance.name))) if will_return: return True else: print( green("OK: instance {} has correct values: {}".format( instance.name, result))) if will_return: return False else: if result['loaded']: print( green("OK: instance {} has correct values: {}".format( instance.name, result))) if will_return: return True elif fail_if_error: abort( red("CRITICAL: instance {} has no loaded data".format( instance.name))) else: print( yellow( "WARNING: instance {} has no loaded data".format( instance.name))) if will_return: return False except KeyError: print( red("CRITICAL: instance {} does not return a correct result". format(instance.name))) print(result) if fail_if_error: abort('') return False