def upgrade_tyr_packages(): packages = [ 'logrotate', 'git', ] if env.distrib == 'ubuntu14.04': packages += ['libpython2.7-dev'] elif env.distrib == 'debian7': packages += ['python2.7-dev'] elif env.distrib == 'debian8': packages += ['python2.7-dev', 'g++'] require.deb.packages(packages, update=True) package_filter_list = ['navitia-tyr*deb', 'navitia-common*deb'] _install_packages(package_filter_list) if not python.is_pip_installed(): python.install_pip() #we want the version of the system for these packages run('''sed -e "/protobuf/d" -e "/psycopg2/d" /usr/share/tyr/requirements.txt > /tmp/tyr_requirements.txt''') run('git config --global url."https://".insteadOf git://') require.python.install_requirements('/tmp/tyr_requirements.txt', use_sudo=True, exists_action='w') if env.use_systemd: _upload_template('tyr/systemd_tyr_worker.jinja', env.service_name('tyr_worker'), user='******', mode='644', context={'env': env}) else: _upload_template('tyr/tyr_worker.jinja', env.service_name('tyr_worker'), user='******', mode='755', context={'env': env}) restart_tyr_worker() update_init(host='tyr')
def deploy_jormungandr_instance_conf(instance): """ Deploy or redeploy one jormungander coverage: * Deploy the json configuration file * Do not reload apache """ instance = get_real_instance(instance) config = { 'key': instance.name, 'zmq_socket': instance.jormungandr_zmq_socket_for_instance, 'realtime_proxies': instance.realtime_proxies } if instance.pt_zmq_socket: config['pt_zmq_socket'] = instance.pt_zmq_socket if instance.street_network: config["street_network"] = instance.street_network if instance.ridesharing: config["ridesharing"] = instance.ridesharing if instance.autocomplete: config["default_autocomplete"] = instance.autocomplete if instance.equipment_details_providers: config[ "equipment_details_providers"] = instance.equipment_details_providers _upload_template("jormungandr/instance.json.jinja", instance.jormungandr_config_file, context={'json': json.dumps(config, indent=4)}, use_sudo=True)
def update_tyr_config_file(): _upload_template("tyr/settings.py.jinja", env.tyr_settings_file, context={ 'env': env, 'tyr_broker_username': env.tyr_broker_username, 'tyr_broker_password': env.tyr_broker_password, 'rabbitmq_host': env.rabbitmq_host, 'rabbitmq_port': env.rabbitmq_port, 'tyr_postgresql_user': env.tyr_postgresql_user, 'tyr_postgresql_password': env.tyr_postgresql_password, 'postgresql_database_host': env.postgresql_database_host, 'tyr_postgresql_database': env.tyr_postgresql_database, 'tyr_base_instances_dir': env.tyr_base_instances_dir, 'tyr_base_logfile': env.tyr_base_logfile, 'redis_host': env.redis_host, 'redis_port': env.redis_port, 'tyr_redis_password': env.tyr_redis_password, 'tyr_redis_db': env.tyr_redis_db }) _upload_template('tyr/tyr.wsgi.jinja', env.tyr_wsgi_file, context={'tyr_settings_file': env.tyr_settings_file})
def configure(): _upload_template('haproxy/haproxy.cfg', '/etc/haproxy/haproxy.cfg', context={ 'env': env, }, user='******')
def upgrade_tyr_packages(): packages = [ 'sudo', 'apache2', 'libapache2-mod-wsgi', 'logrotate', 'python2.7', 'git', 'postgresql-server-dev-all' ] if env.distrib == 'ubuntu14.04': packages += ['libpython2.7-dev', 'postgresql-9.3-postgis-2.1'] elif env.distrib == 'debian7': packages += ['python2.7-dev', 'postgresql-9.1-postgis'] elif env.distrib == 'debian8': packages += ['python2.7-dev', 'g++', 'postgresql-9.4-postgis-2.1'] require.deb.packages(packages, update=True) package_filter_list = ['navitia-tyr*deb', 'navitia-common*deb'] _install_packages(package_filter_list) if not python.is_pip_installed(): python.install_pip() require.python.install_requirements('/usr/share/tyr/requirements.txt', use_sudo=True, exists_action='w') _upload_template('tyr/tyr_beat.jinja', env.tyr_beat_service_file, context={'env': env}, mode='755') _upload_template('tyr/tyr_worker.jinja', env.tyr_worker_service_file, context={'env': env}, mode='755')
def upload_config(): utils._upload_template("stat_persistor.json.jinja", env.settings_file, context={'env': env}) utils._upload_template('alembic.ini.jinja', env.alembic_file, context={'env': env})
def update_monitor_configuration(): _upload_template('kraken/monitor_kraken.wsgi.jinja', env.kraken_monitor_wsgi_file, context={'env': env}) _upload_template('kraken/monitor_settings.py.jinja', env.kraken_monitor_config_file, context={'env': env})
def setup_tyr_master(): require_directory(env.ed_basedir, owner='www-data', group='www-data', use_sudo=True) if env.use_systemd: _upload_template('tyr/systemd_tyr_beat.jinja', env.service_name('tyr_beat'), user='******', mode='644', context={'env': env}) else: _upload_template('tyr/tyr_beat.jinja',env.service_name('tyr_beat'), user='******', mode='755', context={'env': env}) update_init(host='tyr')
def deploy_jormungandr_instance_conf(instance): _upload_template("jormungandr/jormungandr.ini.jinja", instance.jormungandr_config_file, context={ 'env': env, 'instance': instance, }, use_sudo=True)
def deploy_jormungandr_instance_conf(instance): _upload_template("jormungandr/jormungandr.ini.jinja", instance.jormungandr_config_file, context={ 'env': env, 'instance': instance, }, use_sudo=True )
def setup_kraken(): require.users.user('www-data') require.files.directories([env.kraken_basedir, env.kraken_log_basedir, env.kraken_monitor_basedir], owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True) update_monitor_configuration() if env.setup_apache: _upload_template('kraken/monitor_apache_config.jinja', env.kraken_monitor_apache_config_file, context={'env': env}, backup=False) require.service.started('apache2')
def create_jormungandr_instance(instance, cheap_journey=False): #DEPRECATED """Create a jormungandr instance * Deploy /etc/jormungadr.d/<instance>.ini template * Reload apache """ # get the port config from the kraken engine config = kraken.get_kraken_config(env.roledefs['eng'][0], instance) zmq_socket = config.get('GENERAL', 'zmq_socket') instance_port = env.KRAKEN_RE_PORT.match(zmq_socket) port = instance_port.group('port') _upload_template("jormungandr/jormungandr.ini.jinja", instance.jormungandr_config_file, context={ 'env': env, 'instance': instance, 'socket': "tcp://{}:{}".format(env.jormungandr_instance_socket, port), 'cheap_journey': cheap_journey, }, use_sudo=True ) # testing if instance appears in JSON return on URI /v1/coverage on each # Jormungandr headers = {'Host': env.jormungandr_url } server = env.host_string print("→ server: {}".format(server)) execute(reload_jormun_safe, server) request_str = 'http://{}/v1/coverage/{}/status'.format(get_host_addr(server), instance) print("request_string: {}".format(request_str)) try: response = requests.get(request_str, headers=headers) except (ConnectionError, HTTPError) as e: print(red("HTTP Error %s: %s" % (e.code, e.readlines()[0]))) exit(1) except Exception as e: print(red("Error when connecting to %s: %s" % (env.jormungandr_url, e))) exit(1) try: result = response.json() except JSONDecodeError: print(red("cannot read json response : {}".format(response.text))) exit(1) if result['status']['kraken_version']: print(green("OK: Test {} OK".format(instance))) # really test the instance, warning, maybe no data so 503 returned test_jormungandr(get_host_addr(env.host_string), instance, fail_if_error=False)
def setup_jormungandr(): require.users.user('www-data') execute(update_jormungandr_conf) if env.setup_apache: sudo('sudo a2enmod rewrite') _upload_template('jormungandr/jormungandr_apache_config.jinja', env.jormungandr_apache_config_file, context={'env': env}, backup=False) execute(start_jormungandr_all)
def get_tyr_last_done_job_id(instance_id): """ Return the last done job for an instance """ _upload_template("templates/db/last-job-instance.sql.jinja", "/var/lib/postgresql/postgres_{}.sql".format(instance_id), use_jinja=True, context={'instance_id': instance_id}, ) job_id = run('su - postgres --command="psql {} --quiet --tuples-only < /var/lib/postgresql/postgres_{}.sql"' .format(env.jormungandr_postgresql_database, instance_id)) run("rm -f /var/lib/postgresql/postgres_{}.sql".format(instance_id)) return job_id
def get_tyr_last_pt_data_set(instance_id): """Return the data_set used for """ _upload_template("templates/db/job-id-data-set.sql.jinja", "/var/lib/postgresql/postgres_{}.sql".format(instance_id), use_jinja=True, context={'instance_id': instance_id}, ) data_set = run('su - postgres --command="psql {} --quiet --tuples-only < /var/lib/postgresql/postgres_{}.sql"' .format(env.jormungandr_postgresql_database, instance_id)) run("rm -f /var/lib/postgresql/postgres_{}.sql".format(instance_id)) return data_set
def setup_tyr(): require.users.user('www-data') require_directories([env.tyr_base_instances_dir, env.tyr_basedir, ], owner=env.TYR_USER, group=env.TYR_USER, mode='755', use_sudo=True) require_directory(env.tyr_base_logdir, owner=env.TYR_USER, group=env.TYR_USER, mode='777', use_sudo=True) require_directory(env.tyr_base_destination_dir, is_on_nfs4=False, owner=env.TYR_USER, group=env.TYR_USER, mode='755', use_sudo=True) require.files.file(env.tyr_base_logfile, owner=env.TYR_USER, group=env.TYR_USER, mode='766', use_sudo=True) require.files.file(env.tyr_logfile_pattern, owner=env.TYR_USER, group=env.TYR_USER, mode='766', use_sudo=True) update_tyr_confs() if env.use_systemd: _upload_template('tyr/systemd_tyr_worker.jinja', env.service_name('tyr_worker'), user='******', mode='644', context={'env': env}) else: _upload_template('tyr/tyr_worker.jinja', env.service_name('tyr_worker'), user='******', mode='755', context={'env': env}) update_init(host='tyr') if not files.is_dir(env.tyr_migration_dir): idempotent_symlink('/usr/share/tyr/migrations/', env.tyr_migration_dir, use_sudo=True) sudo("chown www-data:www-data {}".format(env.tyr_migration_dir)) # we create a symlink for tyr manage_py tyr_symlink = os.path.join(env.tyr_basedir, 'manage.py') if not files.is_file(tyr_symlink): idempotent_symlink('/usr/bin/manage_tyr.py', tyr_symlink, use_sudo=True)
def get_instance_id(instance): """ Return the id of a given instance """ _upload_template("templates/db/instance_id.sql.jinja", "/var/lib/postgresql/postgres_{}.sql".format(instance), use_jinja=True, context={'instance': instance}, ) instance_id = run('su - postgres --command="psql {} --quiet --tuples-only < /var/lib/postgresql/postgres_{}.sql"' .format(env.jormungandr_postgresql_database, instance)) run("rm -f /var/lib/postgresql/postgres_{}.sql".format(instance)) return instance_id
def remove_ed_database(instance): """Remove a given ed instance in jormungandr PostgreSQL db http://jira.canaltp.fr/browse/NAVITIAII-1098 """ _upload_template("db/remove_instance.sql.jinja", \ "/var/lib/postgresql/postgres_{}.sql".format(instance), context={ 'instance': instance, } ) run('su - postgres --command="psql jormungandr < /var/lib/postgresql/postgres_{}.sql"'.format(instance)) run("rm -f /var/lib/postgresql/postgres_{}.sql".format(instance))
def rename_postgresql_database(current_database, new_database): """ Rename a postgresql database and the SAME corresponding username""" _upload_template("db/rename_postgresql_database_user.sql.jinja", \ "/var/lib/postgresql/postgres_{}.sql".format(current_database), context={ 'current_database': current_database, 'new_database': new_database, } ) run('su - postgres --command="psql postgres < /var/lib/postgresql/postgres_{}.sql"'.format(current_database)) run("rm -f /var/lib/postgresql/postgres_{}.sql".format(current_database))
def remove_instance_from_jormun_database(instance): """Remove a given ed instance in jormungandr PostgreSQL db http://jira.canaltp.fr/browse/NAVITIAII-1098 """ _upload_template("db/remove_instance.sql.jinja", \ "/var/lib/postgresql/postgres_{}.sql".format(instance), context={ 'instance': instance, } ) run('su - postgres --command="psql jormungandr < /var/lib/postgresql/postgres_{}.sql"'.format(instance)) run("rm -f /var/lib/postgresql/postgres_{}.sql".format(instance))
def rename_postgresql_database(current_database, new_database): """ Rename a postgresql database and the SAME corresponding username""" _upload_template("db/rename_postgresql_database_user.sql.jinja", "/var/lib/postgresql/postgres_{}.sql".format(current_database), context={ 'current_database': current_database, 'new_database': new_database, } ) run('su - postgres --command="psql postgres < /var/lib/postgresql/postgres_{}.sql"'.format(current_database)) run("rm -f /var/lib/postgresql/postgres_{}.sql".format(current_database))
def set_tyr_is_free(instance, is_free=True): """Set is_free flag in jormungandr database for a given instance""" _upload_template("db/is-free.sql.jinja", "/var/lib/postgresql/postgres_{}.sql".format(instance), context={ 'instance': instance, 'is_free': is_free, }) run('su - postgres --command="psql {} < /var/lib/postgresql/postgres_{}.sql"' .format(env.jormungandr_postgresql_database, instance)) run("rm -f /var/lib/postgresql/postgres_{}.sql".format(instance))
def rename_tyr_jormungandr_database(current_instance, new_instance): """ Rename the instance id in the jormungandr database """ _upload_template("db/rename_tyr_jormungandr_database.sql.jinja", \ "/var/lib/postgresql/postgres_{}.sql".format(current_instance), context={ 'current_database': current_instance, 'new_database': new_instance, } ) run('su - postgres --command="psql {} < /var/lib/postgresql/postgres_{}.sql"' .format(env.jormungandr_postgresql_database, current_instance)) run("rm -f /var/lib/postgresql/postgres_{}.sql".format(current_instance))
def set_tyr_is_free(instance, is_free=True): """Set is_free flag in jormungandr database for a given instance""" _upload_template("db/is-free.sql.jinja", \ "/var/lib/postgresql/postgres_{}.sql".format(instance), context={ 'instance': instance, 'is_free': is_free, } ) run('su - postgres --command="psql {} < /var/lib/postgresql/postgres_{}.sql"' .format(env.jormungandr_postgresql_database, instance)) run("rm -f /var/lib/postgresql/postgres_{}.sql".format(instance))
def update_jormungandr_conf(): """ update the jormungandr configuration """ require.files.directories([env.jormungandr_base_dir, env.jormungandr_instances_dir, env.jormungandr_log_dir], owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True) _upload_template('jormungandr/jormungandr.wsgi.jinja', env.jormungandr_wsgi_file, context={ 'env': env }) _upload_template('jormungandr/settings.py.jinja', env.jormungandr_settings_file, context={'env': env})
def setup_kraken(): require.users.user('www-data') require.files.directories([env.kraken_basedir, env.kraken_log_basedir, env.kraken_monitor_basedir], owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True) update_monitor_configuration() if env.setup_apache: apache_conf_path = env.apache_conf_path('monitor-kraken') _upload_template('kraken/monitor_apache_config.jinja', apache_conf_path, context={'env': env}, backup=False) if env.apache_version() >= 2.4: sudo('a2enconf monitor-kraken.conf') sudo("service apache2 reload") require.service.started('apache2')
def init_app(): require.files.directories( [env.deploy_to, '{}/releases'.format(env.deploy_to), env.log_dir], owner=env.USER) require.deb.packages([ 'python2.7', 'python-psycopg2', 'sudo', 'protobuf-compiler', ]) utils._upload_template('stat_persistor.jinja', '/etc/init.d/stat_persistor', mode='755', context={'env': env})
def check_is_postgresql_user_exist(username): """Check if a given postgresql user exist""" _upload_template("db/check_is_postgresql_user_exist.sql.jinja", \ "/var/lib/postgresql/postgres_{}.sql".format(username), context={ 'username': username, } ) user = run('su - postgres --command="psql --tuples-only postgres < /var/lib/postgresql/postgres_{}.sql"'.format(username)) run("rm -f /var/lib/postgresql/postgres_{}.sql".format(username)) if username == user: return True else: return False
def update_jormungandr_conf(): """ update the jormungandr configuration """ require.files.directories([ env.jormungandr_base_dir, env.jormungandr_instances_dir, env.jormungandr_log_dir ], owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True) _upload_template('jormungandr/jormungandr.wsgi.jinja', env.jormungandr_wsgi_file, context={'env': env}) _upload_template('jormungandr/settings.py.jinja', env.jormungandr_settings_file, context={'env': env}) if env.newrelic_key: # Only upload newrelic config if a license key is provided _upload_template('jormungandr/newrelic.ini.jinja', env.jormungandr_newrelic_config_file, context={'env': env}) if env.uwsgi_enable: # Add uwsgi config for jormungandr _upload_template('jormungandr/jormungandr.ini.jinja', env.jormungandr_uwsgi_config_file, context={'env': env}, backup=False)
def check_is_postgresql_user_exist(username): """Check if a given postgresql user exist""" _upload_template("db/check_is_postgresql_user_exist.sql.jinja", "/var/lib/postgresql/postgres_{}.sql".format(username), context={ 'username': username, } ) user = run('su - postgres --command="psql --tuples-only postgres < /var/lib/postgresql/postgres_{}.sql"' .format(username)) run("rm -f /var/lib/postgresql/postgres_{}.sql".format(username)) if username == user: return True else: return False
def setup_jormungandr(): require.users.user("www-data") execute(update_jormungandr_conf) if env.setup_apache: sudo("sudo a2enmod rewrite") _upload_template( "jormungandr/jormungandr_apache_config.jinja", env.jormungandr_apache_config_file, context={"env": env}, backup=False, ) execute(start_jormungandr_all)
def setup_kraken(): require.users.user('www-data') require.files.directories([ env.kraken_basedir, env.kraken_log_basedir, env.kraken_monitor_basedir ], owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True) update_monitor_configuration() if env.setup_apache: _upload_template('kraken/monitor_apache_config.jinja', env.kraken_monitor_apache_config_file, context={'env': env}, backup=False) require.service.started('apache2')
def deploy_jormungandr_instance_conf(instance): instance = get_real_instance(instance) config = {'key': instance.name, 'zmq_socket': instance.jormungandr_zmq_socket_for_instance} config['realtime_proxies'] = instance.realtime_proxies _upload_template("jormungandr/instance.json.jinja", instance.jormungandr_config_file, context={ 'json': json.dumps(config, indent=4) }, use_sudo=True ) # the old configuration file were .ini, now it's json, we need to clean up if fabtools.files.is_file(instance.jormungandr_old_ini_config_file): fabtools.files.remove(instance.jormungandr_old_ini_config_file)
def deploy_jormungandr_instance_conf(instance): """ Deploy or redeploy one jormungander coverage: * Deploy the json configuration file * Do not reload apache """ instance = get_real_instance(instance) config = {'key': instance.name, 'zmq_socket': instance.jormungandr_zmq_socket_for_instance, 'realtime_proxies': instance.realtime_proxies} _upload_template("jormungandr/instance.json.jinja", instance.jormungandr_config_file, context={ 'json': json.dumps(config, indent=4) }, use_sudo=True )
def create_postgresql_user(username, password): """ Create a postgresql user""" run('su - postgres --command="createuser {} --no-createdb --no-createrole --no-superuser"'.format(username)) # set the password _upload_template("db/set_user_password.sql.jinja", \ "/var/lib/postgresql/postgres_{}.sql".format(username), context={ 'username': username, 'password': password, } ) run('su - postgres --command="psql postgres < /var/lib/postgresql/postgres_{}.sql"'.format(username)) run("rm -f /var/lib/postgresql/postgres_{}.sql".format(username)) # test the user access run('PGPASSWORD="******" psql --tuples-only --host localhost --username {} postgres --command="SELECT * FROM pg_catalog.pg_database;"'.format(password, username))
def update_eng_instance_conf(instance): instance = get_real_instance(instance) _upload_template("kraken/kraken.ini.jinja", "%s/%s/kraken.ini" % (env.kraken_basedir, instance.name), context={ 'env': env, 'instance': instance, }) _upload_template("kraken/kraken.initscript.jinja", "/etc/init.d/kraken_%s" % instance.name, context={ 'env': env, 'instance': instance.name, 'kraken_base_conf': env.kraken_basedir, }, mode='755')
def cities_integration(): """ Setup the cities module see https://github.com/CanalTP/puppet-navitia/pull/45 for more information """ run("apt-get --yes install navitia-cities") run("pip install python-dateutil") # postgresql user + dedicated database postgresql_user = '******' postgresql_database = postgresql_user password = utils._random_generator() execute(db.create_postgresql_user, "cities", password) execute(db.create_postgresql_database, "cities") # init_db.sh execute(db.postgis_initdb, "cities") utils._upload_template( "tyr/cities_alembic.ini.jinja", "{}/cities_alembic.ini".format(env.tyr_basedir), context={ 'env': env, 'postgresql_database': postgresql_database, 'postgresql_user': postgresql_user, 'postgresql_password': password, }, ) raw_input( "Please add \"CITIES_DATABASE_URI = 'user={user} password={password} " "host={database_host} dbname={dbname}'\" in /srv/tyr/settings.py and press " "enter when finished.".format( password=password, database_host=env.postgresql_database_host, dbname=postgresql_database, user=postgresql_user)) with cd(env.tyr_basedir): run("alembic --config cities_alembic.ini upgrade head") if exists("/srv/ed/france-latest.osm.pbf"): run("TYR_CONFIG_FILE=/srv/tyr/settings.py ./manage.py {} " "/srv/ed/france-latest.osm.pbf".format(postgresql_database)) execute(tyr.restart_tyr_worker)
def update_eng_instance_conf(instance): instance = get_real_instance(instance) _upload_template("kraken/kraken.ini.jinja", "%s/%s/kraken.ini" % (env.kraken_basedir, instance.name), context={ 'env': env, 'instance': instance, } ) _upload_template("kraken/kraken.initscript.jinja", "/etc/init.d/kraken_%s" % instance.name, context={'env': env, 'instance': instance.name, 'kraken_base_conf': env.kraken_basedir, }, mode='755' )
def create_postgresql_user(username, password): """ Create a postgresql user""" run('su - postgres --command="createuser {} --no-createdb --no-createrole --no-superuser"'.format(username)) # set the password _upload_template("db/set_user_password.sql.jinja", "/var/lib/postgresql/postgres_{}.sql".format(username), context={ 'username': username, 'password': password, } ) run('su - postgres --command="psql postgres < /var/lib/postgresql/postgres_{}.sql"'.format(username)) run("rm -f /var/lib/postgresql/postgres_{}.sql".format(username)) # test the user access run('PGPASSWORD="******" psql --tuples-only --host localhost --username {} ' 'postgres --command="SELECT * FROM pg_catalog.pg_database;"'.format(password, username))
def deploy_jormungandr_instance_conf(instance): """ Deploy or redeploy one jormungander coverage: * Deploy the json configuration file * Do not reload apache """ instance = get_real_instance(instance) config = { "key": instance.name, "zmq_socket": instance.jormungandr_zmq_socket_for_instance, "realtime_proxies": instance.realtime_proxies, } if instance.street_network: config["street_network"] = instance.street_network _upload_template( "jormungandr/instance.json.jinja", instance.jormungandr_config_file, context={"json": json.dumps(config, indent=4)}, use_sudo=True, )
def cities_integration(): """ Setup the cities module see https://github.com/CanalTP/puppet-navitia/pull/45 for more information """ run("apt-get --yes install navitia-cities") run("pip install python-dateutil") # postgresql user + dedicated database postgresql_user = '******' postgresql_database = postgresql_user password = utils._random_generator() execute(db.create_postgresql_user, "cities", password) execute(db.create_postgresql_database, "cities") # init_db.sh execute(db.postgis_initdb, "cities") utils._upload_template("tyr/cities_alembic.ini.jinja", "{}/cities_alembic.ini".format(env.tyr_basedir), context={ 'env': env, 'postgresql_database': postgresql_database, 'postgresql_user': postgresql_user, 'postgresql_password': password, }, ) raw_input("Please add \"CITIES_DATABASE_URI = 'user={user} password={password} " "host={database_host} dbname={dbname}'\" in /srv/tyr/settings.py and press " "enter when finished.".format(password=password, database_host=env.postgresql_database_host, dbname=postgresql_database, user=postgresql_user)) with cd(env.tyr_basedir): run("alembic --config cities_alembic.ini upgrade head") if exists("/srv/ed/france-latest.osm.pbf"): run("TYR_CONFIG_FILE=/srv/tyr/settings.py ./manage.py {} " "/srv/ed/france-latest.osm.pbf".format(postgresql_database)) execute(tyr.restart_tyr_worker)
def upgrade_tyr_packages(): packages = [ 'logrotate', 'git', ] if env.distrib == 'ubuntu14.04': packages += ['libpython2.7-dev'] elif env.distrib == 'debian7': packages += ['python2.7-dev'] elif env.distrib == 'debian8': packages += ['python2.7-dev', 'g++'] require.deb.packages(packages, update=True) package_filter_list = ['navitia-tyr*deb', 'navitia-common*deb'] _install_packages(package_filter_list) if not python.is_pip_installed(): python.install_pip() require.python.install_requirements('/usr/share/tyr/requirements.txt', use_sudo=True, exists_action='w') _upload_template('tyr/tyr_worker.jinja', env.tyr_worker_service_file, user='******', mode='755', context={'env': env})
def update_eng_instance_conf(instance, host=None): instance = get_real_instance(instance) hosts = [host] if host else instance.kraken_engines for host in hosts: with settings(host_string=host): require.files.directory(os.path.join(instance.kraken_basedir, instance.name), owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True) _upload_template("kraken/kraken.ini.jinja", "%s/%s/kraken.ini" % (env.kraken_basedir, instance.name), context={ 'env': env, 'instance': instance, } ) if env.use_systemd: _upload_template("kraken/systemd_kraken.jinja", "{}".format(env.service_name('kraken_{}'.format(instance.name))), context={'env': env, 'instance': instance.name, 'kraken_base_conf': env.kraken_basedir, }, mode='644' ) else: _upload_template("kraken/kraken.initscript.jinja", "{}".format(env.service_name('kraken_{}'.format(instance.name))), context={'env': env, 'instance': instance.name, 'kraken_base_conf': env.kraken_basedir, }, mode='755' ) # TODO check this, make it consistent with env.use_systemd update_init(host='eng')
def update_eng_instance_conf(instance): instance = get_real_instance(instance) for host in instance.kraken_engines: with settings(host_string=host): _upload_template("kraken/kraken.ini.jinja", "%s/%s/kraken.ini" % (env.kraken_basedir, instance.name), context={ 'env': env, 'instance': instance, } ) if env.use_systemd: _upload_template("kraken/systemd_kraken.jinja", "{}".format(env.service_name('kraken_{}'.format(instance.name))), context={'env': env, 'instance': instance.name, 'kraken_base_conf': env.kraken_basedir, }, mode='644' ) else: _upload_template("kraken/kraken.initscript.jinja", "{}".format(env.service_name('kraken_{}'.format(instance.name))), context={'env': env, 'instance': instance.name, 'kraken_base_conf': env.kraken_basedir, }, mode='755' ) # TODO check this, make it consistent with env.use_systemd update_init(host='eng')
def update_tyr_instance_conf(instance): _upload_template( "tyr/instance.ini.jinja", "{}/{}.ini".format(env.tyr_base_instances_dir, instance.name), context={ 'env': env, 'instance': instance, }, ) require_directory(instance.base_ed_dir, owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True) # /srv/ed/$instance/alembic.ini, used by update_ed_db() _upload_template( "tyr/ed_alembic.ini.jinja", "{}/alembic.ini".format(instance.base_ed_dir), context={ 'env': env, 'instance': instance, }, ) #we need a settings file to init the db with postgis # will be deprecated when migrating to postgis 2.1 _upload_template( "tyr/ed_settings.sh.jinja", "{}/settings.sh".format(instance.base_ed_dir), context={ 'env': env, 'instance': instance, }, )
def update_tyr_config_file(): _upload_template("tyr/settings.py.jinja", env.tyr_settings_file, context={ 'env': env, 'tyr_postgresql_user': env.tyr_postgresql_user, 'tyr_postgresql_password': env.tyr_postgresql_password, 'ed_postgresql_database_host': env.ed_postgresql_database_host, 'tyr_postgresql_database_host': env.tyr_postgresql_database_host, 'tyr_postgresql_database': env.tyr_postgresql_database, 'tyr_base_instances_dir': env.tyr_base_instances_dir, 'tyr_base_logfile': env.tyr_base_logfile, 'redis_host': env.redis_host, 'redis_port': env.redis_port, 'tyr_redis_password': env.tyr_redis_password, 'tyr_redis_db': env.tyr_redis_db }) _upload_template('tyr/tyr.wsgi.jinja', env.tyr_wsgi_file, context={'tyr_settings_file': env.tyr_settings_file}) if env.uwsgi_enable: _upload_template('tyr/tyr.ini.jinja', env.tyr_uwsgi_config_file, context={'env': env})
def update_tyr_instance_conf(instance): _upload_template("tyr/instance.ini.jinja", "{}/{}.ini".format(env.tyr_base_instances_dir, instance.name), context={ 'env': env, 'instance': instance, }, ) require_directory(instance.base_ed_dir, owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True) # /srv/ed/$instance/alembic.ini, used by update_ed_db() _upload_template("tyr/ed_alembic.ini.jinja", "{}/alembic.ini".format(instance.base_ed_dir), context={ 'env': env, 'instance': instance, }, ) #we need a settings file to init the db with postgis # will be deprecated when migrating to postgis 2.1 _upload_template("tyr/ed_settings.sh.jinja", "{}/settings.sh".format(instance.base_ed_dir), context={ 'env': env, 'instance': instance, }, )
def update_tyr_config_file(): _upload_template("tyr/settings.py.jinja", env.tyr_settings_file, context={ 'env': env, 'tyr_broker_username': env.tyr_broker_username, 'tyr_broker_password': env.tyr_broker_password, 'rabbitmq_host': env.rabbitmq_host, 'rabbitmq_port': env.rabbitmq_port, 'tyr_postgresql_user': env.tyr_postgresql_user, 'tyr_postgresql_password': env.tyr_postgresql_password, 'postgresql_database_host': env.postgresql_database_host, 'tyr_postgresql_database': env.tyr_postgresql_database, 'tyr_base_instances_dir': env.tyr_base_instances_dir, 'tyr_base_logfile': env.tyr_base_logfile, 'redis_host': env.redis_host, 'redis_port': env.redis_port, 'tyr_redis_password': env.tyr_redis_password, 'tyr_redis_db': env.tyr_redis_db }) _upload_template('tyr/tyr.wsgi.jinja', env.tyr_wsgi_file, context={ 'tyr_settings_file': env.tyr_settings_file })