def upgrade_tyr_packages():
packages = [
'logrotate',
'git',
]
if env.distrib == 'ubuntu14.04':
packages += ['libpython2.7-dev']
elif env.distrib == 'debian7':
packages += ['python2.7-dev']
elif env.distrib == 'debian8':
packages += ['python2.7-dev', 'g++']
require.deb.packages(packages, update=True)
package_filter_list = ['navitia-tyr*deb',
'navitia-common*deb']
_install_packages(package_filter_list)
if not python.is_pip_installed():
python.install_pip()
#we want the version of the system for these packages
run('''sed -e "/protobuf/d" -e "/psycopg2/d" /usr/share/tyr/requirements.txt > /tmp/tyr_requirements.txt''')
run('git config --global url."https://".insteadOf git://')
require.python.install_requirements('/tmp/tyr_requirements.txt', use_sudo=True, exists_action='w')
if env.use_systemd:
_upload_template('tyr/systemd_tyr_worker.jinja', env.service_name('tyr_worker'),
user='******', mode='644', context={'env': env})
else:
_upload_template('tyr/tyr_worker.jinja', env.service_name('tyr_worker'),
user='******', mode='755', context={'env': env})
restart_tyr_worker()
update_init(host='tyr')
def deploy_jormungandr_instance_conf(instance):
""" Deploy or redeploy one jormungander coverage:
* Deploy the json configuration file
* Do not reload apache
"""
instance = get_real_instance(instance)
config = {
'key': instance.name,
'zmq_socket': instance.jormungandr_zmq_socket_for_instance,
'realtime_proxies': instance.realtime_proxies
}
if instance.pt_zmq_socket:
config['pt_zmq_socket'] = instance.pt_zmq_socket
if instance.street_network:
config["street_network"] = instance.street_network
if instance.ridesharing:
config["ridesharing"] = instance.ridesharing
if instance.autocomplete:
config["default_autocomplete"] = instance.autocomplete
if instance.equipment_details_providers:
config[
"equipment_details_providers"] = instance.equipment_details_providers
_upload_template("jormungandr/instance.json.jinja",
instance.jormungandr_config_file,
context={'json': json.dumps(config, indent=4)},
use_sudo=True)
def update_tyr_config_file():
_upload_template("tyr/settings.py.jinja",
env.tyr_settings_file,
context={
'env': env,
'tyr_broker_username': env.tyr_broker_username,
'tyr_broker_password': env.tyr_broker_password,
'rabbitmq_host': env.rabbitmq_host,
'rabbitmq_port': env.rabbitmq_port,
'tyr_postgresql_user': env.tyr_postgresql_user,
'tyr_postgresql_password':
env.tyr_postgresql_password,
'postgresql_database_host':
env.postgresql_database_host,
'tyr_postgresql_database':
env.tyr_postgresql_database,
'tyr_base_instances_dir': env.tyr_base_instances_dir,
'tyr_base_logfile': env.tyr_base_logfile,
'redis_host': env.redis_host,
'redis_port': env.redis_port,
'tyr_redis_password': env.tyr_redis_password,
'tyr_redis_db': env.tyr_redis_db
})
_upload_template('tyr/tyr.wsgi.jinja',
env.tyr_wsgi_file,
context={'tyr_settings_file': env.tyr_settings_file})
def configure():
_upload_template('haproxy/haproxy.cfg',
'/etc/haproxy/haproxy.cfg',
context={
'env': env,
},
user='******')
def upgrade_tyr_packages():
packages = [
'sudo',
'apache2',
'libapache2-mod-wsgi',
'logrotate',
'python2.7',
'git',
'postgresql-server-dev-all'
]
if env.distrib == 'ubuntu14.04':
packages += ['libpython2.7-dev', 'postgresql-9.3-postgis-2.1']
elif env.distrib == 'debian7':
packages += ['python2.7-dev', 'postgresql-9.1-postgis']
elif env.distrib == 'debian8':
packages += ['python2.7-dev', 'g++', 'postgresql-9.4-postgis-2.1']
require.deb.packages(packages, update=True)
package_filter_list = ['navitia-tyr*deb',
'navitia-common*deb']
_install_packages(package_filter_list)
if not python.is_pip_installed():
python.install_pip()
require.python.install_requirements('/usr/share/tyr/requirements.txt', use_sudo=True, exists_action='w')
_upload_template('tyr/tyr_beat.jinja', env.tyr_beat_service_file,
context={'env': env}, mode='755')
_upload_template('tyr/tyr_worker.jinja', env.tyr_worker_service_file,
context={'env': env}, mode='755')
def upgrade_tyr_packages():
packages = [
'sudo', 'apache2', 'libapache2-mod-wsgi', 'logrotate', 'python2.7',
'git', 'postgresql-server-dev-all'
]
if env.distrib == 'ubuntu14.04':
packages += ['libpython2.7-dev', 'postgresql-9.3-postgis-2.1']
elif env.distrib == 'debian7':
packages += ['python2.7-dev', 'postgresql-9.1-postgis']
elif env.distrib == 'debian8':
packages += ['python2.7-dev', 'g++', 'postgresql-9.4-postgis-2.1']
require.deb.packages(packages, update=True)
package_filter_list = ['navitia-tyr*deb', 'navitia-common*deb']
_install_packages(package_filter_list)
if not python.is_pip_installed():
python.install_pip()
require.python.install_requirements('/usr/share/tyr/requirements.txt',
use_sudo=True,
exists_action='w')
_upload_template('tyr/tyr_beat.jinja',
env.tyr_beat_service_file,
context={'env': env},
mode='755')
_upload_template('tyr/tyr_worker.jinja',
env.tyr_worker_service_file,
context={'env': env},
mode='755')
def upgrade_tyr_packages():
packages = [
'logrotate',
'git',
]
if env.distrib == 'ubuntu14.04':
packages += ['libpython2.7-dev']
elif env.distrib == 'debian7':
packages += ['python2.7-dev']
elif env.distrib == 'debian8':
packages += ['python2.7-dev', 'g++']
require.deb.packages(packages, update=True)
package_filter_list = ['navitia-tyr*deb',
'navitia-common*deb']
_install_packages(package_filter_list)
if not python.is_pip_installed():
python.install_pip()
#we want the version of the system for these packages
run('''sed -e "/protobuf/d" -e "/psycopg2/d" /usr/share/tyr/requirements.txt > /tmp/tyr_requirements.txt''')
run('git config --global url."https://".insteadOf git://')
require.python.install_requirements('/tmp/tyr_requirements.txt', use_sudo=True, exists_action='w')
if env.use_systemd:
_upload_template('tyr/systemd_tyr_worker.jinja', env.service_name('tyr_worker'),
user='******', mode='644', context={'env': env})
else:
_upload_template('tyr/tyr_worker.jinja', env.service_name('tyr_worker'),
user='******', mode='755', context={'env': env})
restart_tyr_worker()
update_init(host='tyr')
def upload_config():
utils._upload_template("stat_persistor.json.jinja",
env.settings_file,
context={'env': env})
utils._upload_template('alembic.ini.jinja',
env.alembic_file,
context={'env': env})
def update_monitor_configuration():
_upload_template('kraken/monitor_kraken.wsgi.jinja',
env.kraken_monitor_wsgi_file,
context={'env': env})
_upload_template('kraken/monitor_settings.py.jinja',
env.kraken_monitor_config_file,
context={'env': env})
def setup_tyr_master():
require_directory(env.ed_basedir, owner='www-data', group='www-data', use_sudo=True)
if env.use_systemd:
_upload_template('tyr/systemd_tyr_beat.jinja', env.service_name('tyr_beat'),
user='******', mode='644', context={'env': env})
else:
_upload_template('tyr/tyr_beat.jinja',env.service_name('tyr_beat'),
user='******', mode='755', context={'env': env})
update_init(host='tyr')
def setup_tyr_master():
require_directory(env.ed_basedir, owner='www-data', group='www-data', use_sudo=True)
if env.use_systemd:
_upload_template('tyr/systemd_tyr_beat.jinja', env.service_name('tyr_beat'),
user='******', mode='644', context={'env': env})
else:
_upload_template('tyr/tyr_beat.jinja',env.service_name('tyr_beat'),
user='******', mode='755', context={'env': env})
update_init(host='tyr')
def deploy_jormungandr_instance_conf(instance):
_upload_template("jormungandr/jormungandr.ini.jinja",
instance.jormungandr_config_file,
context={
'env': env,
'instance': instance,
},
use_sudo=True)
def deploy_jormungandr_instance_conf(instance):
_upload_template("jormungandr/jormungandr.ini.jinja",
instance.jormungandr_config_file,
context={
'env': env,
'instance': instance,
},
use_sudo=True
)
def setup_kraken():
require.users.user('www-data')
require.files.directories([env.kraken_basedir, env.kraken_log_basedir,
env.kraken_monitor_basedir], owner=env.KRAKEN_USER, group=env.KRAKEN_USER,
use_sudo=True)
update_monitor_configuration()
if env.setup_apache:
_upload_template('kraken/monitor_apache_config.jinja', env.kraken_monitor_apache_config_file,
context={'env': env}, backup=False)
require.service.started('apache2')
def create_jormungandr_instance(instance, cheap_journey=False):
#DEPRECATED
"""Create a jormungandr instance
* Deploy /etc/jormungadr.d/<instance>.ini template
* Reload apache
"""
# get the port config from the kraken engine
config = kraken.get_kraken_config(env.roledefs['eng'][0], instance)
zmq_socket = config.get('GENERAL', 'zmq_socket')
instance_port = env.KRAKEN_RE_PORT.match(zmq_socket)
port = instance_port.group('port')
_upload_template("jormungandr/jormungandr.ini.jinja",
instance.jormungandr_config_file,
context={
'env': env,
'instance': instance,
'socket': "tcp://{}:{}".format(env.jormungandr_instance_socket, port),
'cheap_journey': cheap_journey,
},
use_sudo=True
)
# testing if instance appears in JSON return on URI /v1/coverage on each
# Jormungandr
headers = {'Host': env.jormungandr_url }
server = env.host_string
print("→ server: {}".format(server))
execute(reload_jormun_safe, server)
request_str = 'http://{}/v1/coverage/{}/status'.format(get_host_addr(server), instance)
print("request_string: {}".format(request_str))
try:
response = requests.get(request_str, headers=headers)
except (ConnectionError, HTTPError) as e:
print(red("HTTP Error %s: %s" % (e.code, e.readlines()[0])))
exit(1)
except Exception as e:
print(red("Error when connecting to %s: %s" % (env.jormungandr_url, e)))
exit(1)
try:
result = response.json()
except JSONDecodeError:
print(red("cannot read json response : {}".format(response.text)))
exit(1)
if result['status']['kraken_version']:
print(green("OK: Test {} OK".format(instance)))
# really test the instance, warning, maybe no data so 503 returned
test_jormungandr(get_host_addr(env.host_string), instance, fail_if_error=False)
def setup_jormungandr():
require.users.user('www-data')
execute(update_jormungandr_conf)
if env.setup_apache:
sudo('sudo a2enmod rewrite')
_upload_template('jormungandr/jormungandr_apache_config.jinja', env.jormungandr_apache_config_file,
context={'env': env}, backup=False)
execute(start_jormungandr_all)
def get_tyr_last_done_job_id(instance_id):
""" Return the last done job for an instance """
_upload_template("templates/db/last-job-instance.sql.jinja",
"/var/lib/postgresql/postgres_{}.sql".format(instance_id),
use_jinja=True,
context={'instance_id': instance_id},
)
job_id = run('su - postgres --command="psql {} --quiet --tuples-only < /var/lib/postgresql/postgres_{}.sql"'
.format(env.jormungandr_postgresql_database, instance_id))
run("rm -f /var/lib/postgresql/postgres_{}.sql".format(instance_id))
return job_id
def setup_jormungandr():
require.users.user('www-data')
execute(update_jormungandr_conf)
if env.setup_apache:
sudo('sudo a2enmod rewrite')
_upload_template('jormungandr/jormungandr_apache_config.jinja', env.jormungandr_apache_config_file,
context={'env': env}, backup=False)
execute(start_jormungandr_all)
def get_tyr_last_pt_data_set(instance_id):
"""Return the data_set used for """
_upload_template("templates/db/job-id-data-set.sql.jinja",
"/var/lib/postgresql/postgres_{}.sql".format(instance_id),
use_jinja=True,
context={'instance_id': instance_id},
)
data_set = run('su - postgres --command="psql {} --quiet --tuples-only < /var/lib/postgresql/postgres_{}.sql"'
.format(env.jormungandr_postgresql_database, instance_id))
run("rm -f /var/lib/postgresql/postgres_{}.sql".format(instance_id))
return data_set
def setup_tyr():
require.users.user('www-data')
require_directories([env.tyr_base_instances_dir,
env.tyr_basedir,
],
owner=env.TYR_USER,
group=env.TYR_USER,
mode='755',
use_sudo=True)
require_directory(env.tyr_base_logdir,
owner=env.TYR_USER,
group=env.TYR_USER,
mode='777',
use_sudo=True)
require_directory(env.tyr_base_destination_dir,
is_on_nfs4=False,
owner=env.TYR_USER,
group=env.TYR_USER,
mode='755',
use_sudo=True)
require.files.file(env.tyr_base_logfile,
owner=env.TYR_USER,
group=env.TYR_USER,
mode='766',
use_sudo=True)
require.files.file(env.tyr_logfile_pattern,
owner=env.TYR_USER,
group=env.TYR_USER,
mode='766',
use_sudo=True)
update_tyr_confs()
if env.use_systemd:
_upload_template('tyr/systemd_tyr_worker.jinja', env.service_name('tyr_worker'),
user='******', mode='644', context={'env': env})
else:
_upload_template('tyr/tyr_worker.jinja', env.service_name('tyr_worker'),
user='******', mode='755', context={'env': env})
update_init(host='tyr')
if not files.is_dir(env.tyr_migration_dir):
idempotent_symlink('/usr/share/tyr/migrations/', env.tyr_migration_dir, use_sudo=True)
sudo("chown www-data:www-data {}".format(env.tyr_migration_dir))
# we create a symlink for tyr manage_py
tyr_symlink = os.path.join(env.tyr_basedir, 'manage.py')
if not files.is_file(tyr_symlink):
idempotent_symlink('/usr/bin/manage_tyr.py', tyr_symlink, use_sudo=True)
def setup_tyr():
require.users.user('www-data')
require_directories([env.tyr_base_instances_dir,
env.tyr_basedir,
],
owner=env.TYR_USER,
group=env.TYR_USER,
mode='755',
use_sudo=True)
require_directory(env.tyr_base_logdir,
owner=env.TYR_USER,
group=env.TYR_USER,
mode='777',
use_sudo=True)
require_directory(env.tyr_base_destination_dir,
is_on_nfs4=False,
owner=env.TYR_USER,
group=env.TYR_USER,
mode='755',
use_sudo=True)
require.files.file(env.tyr_base_logfile,
owner=env.TYR_USER,
group=env.TYR_USER,
mode='766',
use_sudo=True)
require.files.file(env.tyr_logfile_pattern,
owner=env.TYR_USER,
group=env.TYR_USER,
mode='766',
use_sudo=True)
update_tyr_confs()
if env.use_systemd:
_upload_template('tyr/systemd_tyr_worker.jinja', env.service_name('tyr_worker'),
user='******', mode='644', context={'env': env})
else:
_upload_template('tyr/tyr_worker.jinja', env.service_name('tyr_worker'),
user='******', mode='755', context={'env': env})
update_init(host='tyr')
if not files.is_dir(env.tyr_migration_dir):
idempotent_symlink('/usr/share/tyr/migrations/', env.tyr_migration_dir, use_sudo=True)
sudo("chown www-data:www-data {}".format(env.tyr_migration_dir))
# we create a symlink for tyr manage_py
tyr_symlink = os.path.join(env.tyr_basedir, 'manage.py')
if not files.is_file(tyr_symlink):
idempotent_symlink('/usr/bin/manage_tyr.py', tyr_symlink, use_sudo=True)
def get_tyr_last_pt_data_set(instance_id):
"""Return the data_set used for """
_upload_template("templates/db/job-id-data-set.sql.jinja",
"/var/lib/postgresql/postgres_{}.sql".format(instance_id),
use_jinja=True,
context={'instance_id': instance_id},
)
data_set = run('su - postgres --command="psql {} --quiet --tuples-only < /var/lib/postgresql/postgres_{}.sql"'
.format(env.jormungandr_postgresql_database, instance_id))
run("rm -f /var/lib/postgresql/postgres_{}.sql".format(instance_id))
return data_set
def get_tyr_last_done_job_id(instance_id):
""" Return the last done job for an instance """
_upload_template("templates/db/last-job-instance.sql.jinja",
"/var/lib/postgresql/postgres_{}.sql".format(instance_id),
use_jinja=True,
context={'instance_id': instance_id},
)
job_id = run('su - postgres --command="psql {} --quiet --tuples-only < /var/lib/postgresql/postgres_{}.sql"'
.format(env.jormungandr_postgresql_database, instance_id))
run("rm -f /var/lib/postgresql/postgres_{}.sql".format(instance_id))
return job_id
def get_instance_id(instance):
""" Return the id of a given instance """
_upload_template("templates/db/instance_id.sql.jinja",
"/var/lib/postgresql/postgres_{}.sql".format(instance),
use_jinja=True,
context={'instance': instance},
)
instance_id = run('su - postgres --command="psql {} --quiet --tuples-only < /var/lib/postgresql/postgres_{}.sql"'
.format(env.jormungandr_postgresql_database, instance))
run("rm -f /var/lib/postgresql/postgres_{}.sql".format(instance))
return instance_id
def remove_ed_database(instance):
"""Remove a given ed instance in jormungandr PostgreSQL db
http://jira.canaltp.fr/browse/NAVITIAII-1098
"""
_upload_template("db/remove_instance.sql.jinja", \
"/var/lib/postgresql/postgres_{}.sql".format(instance),
context={
'instance': instance,
}
)
run('su - postgres --command="psql jormungandr < /var/lib/postgresql/postgres_{}.sql"'.format(instance))
run("rm -f /var/lib/postgresql/postgres_{}.sql".format(instance))
def rename_postgresql_database(current_database, new_database):
""" Rename a postgresql database and the SAME corresponding username"""
_upload_template("db/rename_postgresql_database_user.sql.jinja", \
"/var/lib/postgresql/postgres_{}.sql".format(current_database),
context={
'current_database': current_database,
'new_database': new_database,
}
)
run('su - postgres --command="psql postgres < /var/lib/postgresql/postgres_{}.sql"'.format(current_database))
run("rm -f /var/lib/postgresql/postgres_{}.sql".format(current_database))
def remove_instance_from_jormun_database(instance):
"""Remove a given ed instance in jormungandr PostgreSQL db
http://jira.canaltp.fr/browse/NAVITIAII-1098
"""
_upload_template("db/remove_instance.sql.jinja", \
"/var/lib/postgresql/postgres_{}.sql".format(instance),
context={
'instance': instance,
}
)
run('su - postgres --command="psql jormungandr < /var/lib/postgresql/postgres_{}.sql"'.format(instance))
run("rm -f /var/lib/postgresql/postgres_{}.sql".format(instance))
def get_instance_id(instance):
""" Return the id of a given instance """
_upload_template("templates/db/instance_id.sql.jinja",
"/var/lib/postgresql/postgres_{}.sql".format(instance),
use_jinja=True,
context={'instance': instance},
)
instance_id = run('su - postgres --command="psql {} --quiet --tuples-only < /var/lib/postgresql/postgres_{}.sql"'
.format(env.jormungandr_postgresql_database, instance))
run("rm -f /var/lib/postgresql/postgres_{}.sql".format(instance))
return instance_id
def rename_postgresql_database(current_database, new_database):
""" Rename a postgresql database and the SAME corresponding username"""
_upload_template("db/rename_postgresql_database_user.sql.jinja",
"/var/lib/postgresql/postgres_{}.sql".format(current_database),
context={
'current_database': current_database,
'new_database': new_database,
}
)
run('su - postgres --command="psql postgres < /var/lib/postgresql/postgres_{}.sql"'.format(current_database))
run("rm -f /var/lib/postgresql/postgres_{}.sql".format(current_database))
def set_tyr_is_free(instance, is_free=True):
"""Set is_free flag in jormungandr database for a given instance"""
_upload_template("db/is-free.sql.jinja",
"/var/lib/postgresql/postgres_{}.sql".format(instance),
context={
'instance': instance,
'is_free': is_free,
})
run('su - postgres --command="psql {} < /var/lib/postgresql/postgres_{}.sql"'
.format(env.jormungandr_postgresql_database, instance))
run("rm -f /var/lib/postgresql/postgres_{}.sql".format(instance))
def rename_tyr_jormungandr_database(current_instance, new_instance):
""" Rename the instance id in the jormungandr database """
_upload_template("db/rename_tyr_jormungandr_database.sql.jinja", \
"/var/lib/postgresql/postgres_{}.sql".format(current_instance),
context={
'current_database': current_instance,
'new_database': new_instance,
}
)
run('su - postgres --command="psql {} < /var/lib/postgresql/postgres_{}.sql"'
.format(env.jormungandr_postgresql_database, current_instance))
run("rm -f /var/lib/postgresql/postgres_{}.sql".format(current_instance))
def set_tyr_is_free(instance, is_free=True):
"""Set is_free flag in jormungandr database for a given instance"""
_upload_template("db/is-free.sql.jinja", \
"/var/lib/postgresql/postgres_{}.sql".format(instance),
context={
'instance': instance,
'is_free': is_free,
}
)
run('su - postgres --command="psql {} < /var/lib/postgresql/postgres_{}.sql"'
.format(env.jormungandr_postgresql_database, instance))
run("rm -f /var/lib/postgresql/postgres_{}.sql".format(instance))
def rename_tyr_jormungandr_database(current_instance, new_instance):
""" Rename the instance id in the jormungandr database """
_upload_template("db/rename_tyr_jormungandr_database.sql.jinja", \
"/var/lib/postgresql/postgres_{}.sql".format(current_instance),
context={
'current_database': current_instance,
'new_database': new_instance,
}
)
run('su - postgres --command="psql {} < /var/lib/postgresql/postgres_{}.sql"'
.format(env.jormungandr_postgresql_database, current_instance))
run("rm -f /var/lib/postgresql/postgres_{}.sql".format(current_instance))
def update_jormungandr_conf():
"""
update the jormungandr configuration
"""
require.files.directories([env.jormungandr_base_dir, env.jormungandr_instances_dir, env.jormungandr_log_dir],
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
_upload_template('jormungandr/jormungandr.wsgi.jinja', env.jormungandr_wsgi_file,
context={
'env': env
})
_upload_template('jormungandr/settings.py.jinja', env.jormungandr_settings_file,
context={'env': env})
def update_jormungandr_conf():
"""
update the jormungandr configuration
"""
require.files.directories([env.jormungandr_base_dir, env.jormungandr_instances_dir, env.jormungandr_log_dir],
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
_upload_template('jormungandr/jormungandr.wsgi.jinja', env.jormungandr_wsgi_file,
context={
'env': env
})
_upload_template('jormungandr/settings.py.jinja', env.jormungandr_settings_file,
context={'env': env})
def setup_kraken():
require.users.user('www-data')
require.files.directories([env.kraken_basedir, env.kraken_log_basedir,
env.kraken_monitor_basedir], owner=env.KRAKEN_USER, group=env.KRAKEN_USER,
use_sudo=True)
update_monitor_configuration()
if env.setup_apache:
apache_conf_path = env.apache_conf_path('monitor-kraken')
_upload_template('kraken/monitor_apache_config.jinja', apache_conf_path,
context={'env': env}, backup=False)
if env.apache_version() >= 2.4:
sudo('a2enconf monitor-kraken.conf')
sudo("service apache2 reload")
require.service.started('apache2')
def setup_kraken():
require.users.user('www-data')
require.files.directories([env.kraken_basedir, env.kraken_log_basedir,
env.kraken_monitor_basedir], owner=env.KRAKEN_USER, group=env.KRAKEN_USER,
use_sudo=True)
update_monitor_configuration()
if env.setup_apache:
apache_conf_path = env.apache_conf_path('monitor-kraken')
_upload_template('kraken/monitor_apache_config.jinja', apache_conf_path,
context={'env': env}, backup=False)
if env.apache_version() >= 2.4:
sudo('a2enconf monitor-kraken.conf')
sudo("service apache2 reload")
require.service.started('apache2')
def init_app():
require.files.directories(
[env.deploy_to, '{}/releases'.format(env.deploy_to), env.log_dir],
owner=env.USER)
require.deb.packages([
'python2.7',
'python-psycopg2',
'sudo',
'protobuf-compiler',
])
utils._upload_template('stat_persistor.jinja',
'/etc/init.d/stat_persistor',
mode='755',
context={'env': env})
def check_is_postgresql_user_exist(username):
"""Check if a given postgresql user exist"""
_upload_template("db/check_is_postgresql_user_exist.sql.jinja", \
"/var/lib/postgresql/postgres_{}.sql".format(username),
context={
'username': username,
}
)
user = run('su - postgres --command="psql --tuples-only postgres < /var/lib/postgresql/postgres_{}.sql"'.format(username))
run("rm -f /var/lib/postgresql/postgres_{}.sql".format(username))
if username == user:
return True
else:
return False
def update_jormungandr_conf():
"""
update the jormungandr configuration
"""
require.files.directories([
env.jormungandr_base_dir, env.jormungandr_instances_dir,
env.jormungandr_log_dir
],
owner=env.KRAKEN_USER,
group=env.KRAKEN_USER,
use_sudo=True)
_upload_template('jormungandr/jormungandr.wsgi.jinja',
env.jormungandr_wsgi_file,
context={'env': env})
_upload_template('jormungandr/settings.py.jinja',
env.jormungandr_settings_file,
context={'env': env})
if env.newrelic_key: # Only upload newrelic config if a license key is provided
_upload_template('jormungandr/newrelic.ini.jinja',
env.jormungandr_newrelic_config_file,
context={'env': env})
if env.uwsgi_enable:
# Add uwsgi config for jormungandr
_upload_template('jormungandr/jormungandr.ini.jinja',
env.jormungandr_uwsgi_config_file,
context={'env': env},
backup=False)
def check_is_postgresql_user_exist(username):
"""Check if a given postgresql user exist"""
_upload_template("db/check_is_postgresql_user_exist.sql.jinja",
"/var/lib/postgresql/postgres_{}.sql".format(username),
context={
'username': username,
}
)
user = run('su - postgres --command="psql --tuples-only postgres < /var/lib/postgresql/postgres_{}.sql"'
.format(username))
run("rm -f /var/lib/postgresql/postgres_{}.sql".format(username))
if username == user:
return True
else:
return False
def setup_jormungandr():
require.users.user("www-data")
execute(update_jormungandr_conf)
if env.setup_apache:
sudo("sudo a2enmod rewrite")
_upload_template(
"jormungandr/jormungandr_apache_config.jinja",
env.jormungandr_apache_config_file,
context={"env": env},
backup=False,
)
execute(start_jormungandr_all)
def setup_kraken():
require.users.user('www-data')
require.files.directories([
env.kraken_basedir, env.kraken_log_basedir, env.kraken_monitor_basedir
],
owner=env.KRAKEN_USER,
group=env.KRAKEN_USER,
use_sudo=True)
update_monitor_configuration()
if env.setup_apache:
_upload_template('kraken/monitor_apache_config.jinja',
env.kraken_monitor_apache_config_file,
context={'env': env},
backup=False)
require.service.started('apache2')
def deploy_jormungandr_instance_conf(instance):
instance = get_real_instance(instance)
config = {'key': instance.name, 'zmq_socket': instance.jormungandr_zmq_socket_for_instance}
config['realtime_proxies'] = instance.realtime_proxies
_upload_template("jormungandr/instance.json.jinja",
instance.jormungandr_config_file,
context={
'json': json.dumps(config, indent=4)
},
use_sudo=True
)
# the old configuration file were .ini, now it's json, we need to clean up
if fabtools.files.is_file(instance.jormungandr_old_ini_config_file):
fabtools.files.remove(instance.jormungandr_old_ini_config_file)
def deploy_jormungandr_instance_conf(instance):
""" Deploy or redeploy one jormungander coverage:
* Deploy the json configuration file
* Do not reload apache
"""
instance = get_real_instance(instance)
config = {'key': instance.name,
'zmq_socket': instance.jormungandr_zmq_socket_for_instance,
'realtime_proxies': instance.realtime_proxies}
_upload_template("jormungandr/instance.json.jinja",
instance.jormungandr_config_file,
context={
'json': json.dumps(config, indent=4)
},
use_sudo=True
)
def deploy_jormungandr_instance_conf(instance):
instance = get_real_instance(instance)
config = {'key': instance.name, 'zmq_socket': instance.jormungandr_zmq_socket_for_instance}
config['realtime_proxies'] = instance.realtime_proxies
_upload_template("jormungandr/instance.json.jinja",
instance.jormungandr_config_file,
context={
'json': json.dumps(config, indent=4)
},
use_sudo=True
)
# the old configuration file were .ini, now it's json, we need to clean up
if fabtools.files.is_file(instance.jormungandr_old_ini_config_file):
fabtools.files.remove(instance.jormungandr_old_ini_config_file)
def create_postgresql_user(username, password):
""" Create a postgresql user"""
run('su - postgres --command="createuser {} --no-createdb --no-createrole --no-superuser"'.format(username))
# set the password
_upload_template("db/set_user_password.sql.jinja", \
"/var/lib/postgresql/postgres_{}.sql".format(username),
context={
'username': username,
'password': password,
}
)
run('su - postgres --command="psql postgres < /var/lib/postgresql/postgres_{}.sql"'.format(username))
run("rm -f /var/lib/postgresql/postgres_{}.sql".format(username))
# test the user access
run('PGPASSWORD="******" psql --tuples-only --host localhost --username {} postgres --command="SELECT * FROM pg_catalog.pg_database;"'.format(password, username))
def update_eng_instance_conf(instance):
instance = get_real_instance(instance)
_upload_template("kraken/kraken.ini.jinja",
"%s/%s/kraken.ini" % (env.kraken_basedir, instance.name),
context={
'env': env,
'instance': instance,
})
_upload_template("kraken/kraken.initscript.jinja",
"/etc/init.d/kraken_%s" % instance.name,
context={
'env': env,
'instance': instance.name,
'kraken_base_conf': env.kraken_basedir,
},
mode='755')
def cities_integration():
""" Setup the cities module
see https://github.com/CanalTP/puppet-navitia/pull/45 for more information
"""
run("apt-get --yes install navitia-cities")
run("pip install python-dateutil")
# postgresql user + dedicated database
postgresql_user = '******'
postgresql_database = postgresql_user
password = utils._random_generator()
execute(db.create_postgresql_user, "cities", password)
execute(db.create_postgresql_database, "cities")
# init_db.sh
execute(db.postgis_initdb, "cities")
utils._upload_template(
"tyr/cities_alembic.ini.jinja",
"{}/cities_alembic.ini".format(env.tyr_basedir),
context={
'env': env,
'postgresql_database': postgresql_database,
'postgresql_user': postgresql_user,
'postgresql_password': password,
},
)
raw_input(
"Please add \"CITIES_DATABASE_URI = 'user={user} password={password} "
"host={database_host} dbname={dbname}'\" in /srv/tyr/settings.py and press "
"enter when finished.".format(
password=password,
database_host=env.postgresql_database_host,
dbname=postgresql_database,
user=postgresql_user))
with cd(env.tyr_basedir):
run("alembic --config cities_alembic.ini upgrade head")
if exists("/srv/ed/france-latest.osm.pbf"):
run("TYR_CONFIG_FILE=/srv/tyr/settings.py ./manage.py {} "
"/srv/ed/france-latest.osm.pbf".format(postgresql_database))
execute(tyr.restart_tyr_worker)
def update_eng_instance_conf(instance):
instance = get_real_instance(instance)
_upload_template("kraken/kraken.ini.jinja", "%s/%s/kraken.ini" %
(env.kraken_basedir, instance.name),
context={
'env': env,
'instance': instance,
}
)
_upload_template("kraken/kraken.initscript.jinja",
"/etc/init.d/kraken_%s" % instance.name,
context={'env': env,
'instance': instance.name,
'kraken_base_conf': env.kraken_basedir,
},
mode='755'
)
def create_postgresql_user(username, password):
""" Create a postgresql user"""
run('su - postgres --command="createuser {} --no-createdb --no-createrole --no-superuser"'.format(username))
# set the password
_upload_template("db/set_user_password.sql.jinja",
"/var/lib/postgresql/postgres_{}.sql".format(username),
context={
'username': username,
'password': password,
}
)
run('su - postgres --command="psql postgres < /var/lib/postgresql/postgres_{}.sql"'.format(username))
run("rm -f /var/lib/postgresql/postgres_{}.sql".format(username))
# test the user access
run('PGPASSWORD="******" psql --tuples-only --host localhost --username {} '
'postgres --command="SELECT * FROM pg_catalog.pg_database;"'.format(password, username))
def deploy_jormungandr_instance_conf(instance):
""" Deploy or redeploy one jormungander coverage:
* Deploy the json configuration file
* Do not reload apache
"""
instance = get_real_instance(instance)
config = {
"key": instance.name,
"zmq_socket": instance.jormungandr_zmq_socket_for_instance,
"realtime_proxies": instance.realtime_proxies,
}
if instance.street_network:
config["street_network"] = instance.street_network
_upload_template(
"jormungandr/instance.json.jinja",
instance.jormungandr_config_file,
context={"json": json.dumps(config, indent=4)},
use_sudo=True,
)
def cities_integration():
""" Setup the cities module
see https://github.com/CanalTP/puppet-navitia/pull/45 for more information
"""
run("apt-get --yes install navitia-cities")
run("pip install python-dateutil")
# postgresql user + dedicated database
postgresql_user = '******'
postgresql_database = postgresql_user
password = utils._random_generator()
execute(db.create_postgresql_user, "cities", password)
execute(db.create_postgresql_database, "cities")
# init_db.sh
execute(db.postgis_initdb, "cities")
utils._upload_template("tyr/cities_alembic.ini.jinja",
"{}/cities_alembic.ini".format(env.tyr_basedir),
context={
'env': env,
'postgresql_database': postgresql_database,
'postgresql_user': postgresql_user,
'postgresql_password': password,
},
)
raw_input("Please add \"CITIES_DATABASE_URI = 'user={user} password={password} "
"host={database_host} dbname={dbname}'\" in /srv/tyr/settings.py and press "
"enter when finished.".format(password=password,
database_host=env.postgresql_database_host,
dbname=postgresql_database,
user=postgresql_user))
with cd(env.tyr_basedir):
run("alembic --config cities_alembic.ini upgrade head")
if exists("/srv/ed/france-latest.osm.pbf"):
run("TYR_CONFIG_FILE=/srv/tyr/settings.py ./manage.py {} "
"/srv/ed/france-latest.osm.pbf".format(postgresql_database))
execute(tyr.restart_tyr_worker)
def upgrade_tyr_packages():
packages = [
'logrotate',
'git',
]
if env.distrib == 'ubuntu14.04':
packages += ['libpython2.7-dev']
elif env.distrib == 'debian7':
packages += ['python2.7-dev']
elif env.distrib == 'debian8':
packages += ['python2.7-dev', 'g++']
require.deb.packages(packages, update=True)
package_filter_list = ['navitia-tyr*deb',
'navitia-common*deb']
_install_packages(package_filter_list)
if not python.is_pip_installed():
python.install_pip()
require.python.install_requirements('/usr/share/tyr/requirements.txt', use_sudo=True, exists_action='w')
_upload_template('tyr/tyr_worker.jinja', env.tyr_worker_service_file,
user='******', mode='755', context={'env': env})
def update_eng_instance_conf(instance, host=None):
instance = get_real_instance(instance)
hosts = [host] if host else instance.kraken_engines
for host in hosts:
with settings(host_string=host):
require.files.directory(os.path.join(instance.kraken_basedir, instance.name),
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
_upload_template("kraken/kraken.ini.jinja", "%s/%s/kraken.ini" %
(env.kraken_basedir, instance.name),
context={
'env': env,
'instance': instance,
}
)
if env.use_systemd:
_upload_template("kraken/systemd_kraken.jinja",
"{}".format(env.service_name('kraken_{}'.format(instance.name))),
context={'env': env,
'instance': instance.name,
'kraken_base_conf': env.kraken_basedir,
},
mode='644'
)
else:
_upload_template("kraken/kraken.initscript.jinja",
"{}".format(env.service_name('kraken_{}'.format(instance.name))),
context={'env': env,
'instance': instance.name,
'kraken_base_conf': env.kraken_basedir,
},
mode='755'
)
# TODO check this, make it consistent with env.use_systemd
update_init(host='eng')
def update_eng_instance_conf(instance):
instance = get_real_instance(instance)
for host in instance.kraken_engines:
with settings(host_string=host):
_upload_template("kraken/kraken.ini.jinja", "%s/%s/kraken.ini" %
(env.kraken_basedir, instance.name),
context={
'env': env,
'instance': instance,
}
)
if env.use_systemd:
_upload_template("kraken/systemd_kraken.jinja",
"{}".format(env.service_name('kraken_{}'.format(instance.name))),
context={'env': env,
'instance': instance.name,
'kraken_base_conf': env.kraken_basedir,
},
mode='644'
)
else:
_upload_template("kraken/kraken.initscript.jinja",
"{}".format(env.service_name('kraken_{}'.format(instance.name))),
context={'env': env,
'instance': instance.name,
'kraken_base_conf': env.kraken_basedir,
},
mode='755'
)
# TODO check this, make it consistent with env.use_systemd
update_init(host='eng')
def update_tyr_instance_conf(instance):
_upload_template(
"tyr/instance.ini.jinja",
"{}/{}.ini".format(env.tyr_base_instances_dir, instance.name),
context={
'env': env,
'instance': instance,
},
)
require_directory(instance.base_ed_dir,
owner=env.KRAKEN_USER,
group=env.KRAKEN_USER,
use_sudo=True)
# /srv/ed/$instance/alembic.ini, used by update_ed_db()
_upload_template(
"tyr/ed_alembic.ini.jinja",
"{}/alembic.ini".format(instance.base_ed_dir),
context={
'env': env,
'instance': instance,
},
)
#we need a settings file to init the db with postgis
# will be deprecated when migrating to postgis 2.1
_upload_template(
"tyr/ed_settings.sh.jinja",
"{}/settings.sh".format(instance.base_ed_dir),
context={
'env': env,
'instance': instance,
},
)
def update_tyr_config_file():
_upload_template("tyr/settings.py.jinja",
env.tyr_settings_file,
context={
'env': env,
'tyr_postgresql_user': env.tyr_postgresql_user,
'tyr_postgresql_password':
env.tyr_postgresql_password,
'ed_postgresql_database_host':
env.ed_postgresql_database_host,
'tyr_postgresql_database_host':
env.tyr_postgresql_database_host,
'tyr_postgresql_database':
env.tyr_postgresql_database,
'tyr_base_instances_dir': env.tyr_base_instances_dir,
'tyr_base_logfile': env.tyr_base_logfile,
'redis_host': env.redis_host,
'redis_port': env.redis_port,
'tyr_redis_password': env.tyr_redis_password,
'tyr_redis_db': env.tyr_redis_db
})
_upload_template('tyr/tyr.wsgi.jinja',
env.tyr_wsgi_file,
context={'tyr_settings_file': env.tyr_settings_file})
if env.uwsgi_enable:
_upload_template('tyr/tyr.ini.jinja',
env.tyr_uwsgi_config_file,
context={'env': env})
def update_tyr_instance_conf(instance):
_upload_template("tyr/instance.ini.jinja",
"{}/{}.ini".format(env.tyr_base_instances_dir, instance.name),
context={
'env': env,
'instance': instance,
},
)
require_directory(instance.base_ed_dir,
owner=env.KRAKEN_USER, group=env.KRAKEN_USER, use_sudo=True)
# /srv/ed/$instance/alembic.ini, used by update_ed_db()
_upload_template("tyr/ed_alembic.ini.jinja",
"{}/alembic.ini".format(instance.base_ed_dir),
context={
'env': env,
'instance': instance,
},
)
#we need a settings file to init the db with postgis
# will be deprecated when migrating to postgis 2.1
_upload_template("tyr/ed_settings.sh.jinja",
"{}/settings.sh".format(instance.base_ed_dir),
context={
'env': env,
'instance': instance,
},
)
def update_tyr_config_file():
_upload_template("tyr/settings.py.jinja", env.tyr_settings_file,
context={
'env': env,
'tyr_broker_username': env.tyr_broker_username,
'tyr_broker_password': env.tyr_broker_password,
'rabbitmq_host': env.rabbitmq_host,
'rabbitmq_port': env.rabbitmq_port,
'tyr_postgresql_user': env.tyr_postgresql_user,
'tyr_postgresql_password': env.tyr_postgresql_password,
'postgresql_database_host': env.postgresql_database_host,
'tyr_postgresql_database': env.tyr_postgresql_database,
'tyr_base_instances_dir': env.tyr_base_instances_dir,
'tyr_base_logfile': env.tyr_base_logfile,
'redis_host': env.redis_host,
'redis_port': env.redis_port,
'tyr_redis_password': env.tyr_redis_password,
'tyr_redis_db': env.tyr_redis_db
})
_upload_template('tyr/tyr.wsgi.jinja', env.tyr_wsgi_file,
context={
'tyr_settings_file': env.tyr_settings_file
})
