def sync_project_to_server():
"""Synchronize project with webserver ignoring venv and sqlite db..
This is a handy way to get your secret key to the server too...
"""
base_path, code_path, git_url, repo_alias, site_name = get_vars()
update_venv(code_path)
#noinspection PyArgumentEqualDefault
rsync_project(
base_path,
delete=False,
exclude=[
'*.pyc',
'.git',
'*.dmp',
'.DS_Store',
'projecta.db',
'venv',
'django_project/static'])
update_migrations()
with cd(os.path.join(code_path, 'django_project')):
run('touch core/wsgi.py')
set_media_permissions(code_path)
set_db_permissions()
update_migrations()
collectstatic()
fastprint(blue(
'Your server is now in synchronised to your local project\n'))
def get_disk_space():
env.output_prefix = False
print("\n++++++++++ DiskSpace details++++++++++++")
with hide('output', 'running', 'warnings', 'aborts',
'status'), settings(warn_only=True):
fastprint(run('df -h'))
print("\n++++++++++++++++++++++++++++++++++++++++")
def setup_tilemill(proxy_url=None):
"""Set up tile mill - see http://www.mapbox.com/tilemill/ .
We use a pure node setup as described here:
https://github.com/mapbox/tilemill/issues/2103#issuecomment-37473991
in order to avoid potential conflicts with ubuntugis.
:param proxy_url: Optional parameters to specify the url to use for your
network proxy. It should be specified in the form: ``<host>:<port>``.
The same proxy will be used for both http and https urls. If ommitted
no proxy will be used.
:type proxy_url: str
"""
# Note raring seems not to be supported yet...
setup_env()
repo_alias = 'tilemill'
print env
update_git_checkout(env.fg.workspace,
'https://github.com/mapbox/tilemill.git', repo_alias)
work_path = os.path.join(env.fg.workspace, repo_alias)
setup_node(
work_path,
node_version='0.10.26', # known good version
proxy_url=proxy_url)
with cd(work_path):
run('env/bin/npm install')
fastprint(
green('Tilemill installed. Use the start_tilemill task to run it.'))
def list():
"""
List remote configurations.
"""
require('PROJECT')
fastprint(run('cat %(settings)s' % env.PROJECT, quiet=True))
def initialise_qgis_plugin_repo():
"""Initialise a QGIS plugin repo where we host test builds."""
base_path, code_path, git_url, repo_alias, site_name = get_vars()
sudo('apt-get update')
fabtools.require.deb.package('libapache2-mod-wsgi')
local_path = '%s/scripts/test-build-repo' % code_path
if not exists(base_path):
sudo('mkdir -p %s' % base_path)
sudo('chown %s.%s %s' % (env.user, env.user, base_path))
run('cp %s/plugin* %s' % (local_path, env.plugin_repo_path))
run('cp %s/icon* %s' % (code_path, env.plugin_repo_path))
run('cp %(local_path)s/inasafe-test.conf.templ '
'%(local_path)s/inasafe-test.conf' % {'local_path': local_path})
sed('%s/inasafe-test.conf' % local_path,
'inasafe-test.linfiniti.com',
env.repo_site_name)
with cd('/etc/apache2/sites-available/'):
if exists('inasafe-test.conf'):
sudo('a2dissite inasafe-test.conf')
fastprint('Removing old apache2 conf')
sudo('rm inasafe-test.conf')
sudo('ln -s %s/inasafe-test.conf .' % local_path)
# Add a hosts entry for local testing - only really useful for localhost
repo_hosts = '/etc/hosts'
if not contains(repo_hosts, 'inasafe-test'):
append(repo_hosts, '127.0.0.1 %s' % env.repo_site_name, use_sudo=True)
sudo('a2ensite inasafe-test.conf')
sudo('service apache2 reload')
def get_cpu_usage():
env.output_prefix = False
with hide('output', 'running', 'warnings', 'aborts',
'status'), settings(warn_only=True):
fastprint("CPU usage is :" + run(
'grep \'cpu \' /proc/stat | awk \'{usage=($2+$4)*100/($2+$4+$5)} END {print usage "%"}\''
))
def setup_env():
"""Things to do regardless of whether command is local or remote."""
if env.fg is not None:
fastprint('Environment already set!\n')
return
fastprint('Setting environment!\n')
env.fg = fdict()
with hide('output'):
env.fg.user = run('whoami')
# Workaround for
env.fg.hostname = run('hostname')
# this which fails in docker - see
# https://github.com/dotcloud/docker/issues/1301
#env.fg.hostname = fabtools.system.get_hostname()
env.fg.home = os.path.join('/home/', env.fg.user)
env.fg.workspace = os.path.join(env.fg.home, 'dev')
env.fg.inasafe_git_url = 'git://github.com/AIFDR/inasafe.git'
env.fg.qgis_git_url = 'git://github.com/qgis/QGIS.git'
env.fg.kandan_git_url = 'git://github.com/kandanapp/kandan.git'
env.fg.gdal_svn_url = 'https://svn.osgeo.org/gdal/trunk/gdal'
env.fg.tilemill_tarball_url = (
'http://tilemill.s3.amazonaws.com/latest/install-tilemill.tar.gz')
env.fg.inasafe_checkout_alias = 'inasafe-fabric'
env.fg.qgis_checkout_alias = 'qgis-fabric'
env.fg.inasafe_code_path = os.path.join(
env.fg.workspace, env.fg.inasafe_checkout_alias)
env.fg.qgis_code_path = os.path.join(
env.fg.workspace, env.fg.qgis_checkout_alias)
def setup_env():
"""Things to do regardless of whether command is local or remote."""
if env.fg is not None:
fastprint('Environment already set!\n')
return
fastprint('Setting environment!\n')
env.fg = fdict()
with hide('output'):
env.fg.user = run('whoami')
# Workaround for
env.fg.hostname = run('hostname')
# this which fails in docker - see
# https://github.com/dotcloud/docker/issues/1301
#env.fg.hostname = fabtools.system.get_hostname()
env.fg.home = os.path.join('/home/', env.fg.user)
env.fg.workspace = os.path.join(env.fg.home, 'dev')
env.fg.inasafe_git_url = 'git://github.com/AIFDR/inasafe.git'
env.fg.qgis_git_url = 'git://github.com/qgis/QGIS.git'
env.fg.kandan_git_url = 'git://github.com/kandanapp/kandan.git'
env.fg.gdal_svn_url = 'https://svn.osgeo.org/gdal/trunk/gdal'
env.fg.tilemill_tarball_url = (
'http://tilemill.s3.amazonaws.com/latest/install-tilemill.tar.gz')
env.fg.inasafe_checkout_alias = 'inasafe-fabric'
env.fg.qgis_checkout_alias = 'qgis-fabric'
env.fg.inasafe_code_path = os.path.join(env.fg.workspace,
env.fg.inasafe_checkout_alias)
env.fg.qgis_code_path = os.path.join(env.fg.workspace,
env.fg.qgis_checkout_alias)
def _get_commits_for_release(commits: List[Commit], auto: bool = False) -> List[Commit]:
candidate_commits = OrderedDict([
(commit.sha, commit)
for commit in commits
])
# do not prompt for release sha in auto mode
if not auto and len(commits) > 1:
commits_log = '\n'.join([
f'{commit.sha} {commit.msg}'
for i, commit in enumerate(commits)
])
release_commit_sha = None
fastprint(f'You are about to release commits (not tip of master branch):\n{commits_log}\n')
while not release_commit_sha:
commit_text_user_prompt = prompt('Type full commit hash you are releasing >',
validate=lambda n: n if len(n) == 40 else False)
if not commit_text_user_prompt:
warn('There should be 40 characters in commit hash')
elif commit_text_user_prompt not in candidate_commits:
warn(f'Your input "{commit_text_user_prompt}" does not match any commit hash:\n{commits_log}')
else:
release_commit_sha = commit_text_user_prompt
# drop all the top commits not matching the chosen one
for candidate_commit_sha in list(candidate_commits):
if candidate_commit_sha == release_commit_sha:
break
else:
candidate_commits.pop(candidate_commit_sha)
# reverse the order of commits
return list(candidate_commits.values())
def initialise_qgis_plugin_repo():
"""Initialise a QGIS plugin repo where we host test builds."""
_all()
sudo('apt-get update')
fabtools.require.deb.package('libapache2-mod-wsgi')
code_path = os.path.join(env.repo_path, env.repo_alias)
local_path = '%s/scripts/test-build-repo' % code_path
if not exists(env.plugin_repo_path):
sudo('mkdir -p %s' % env.plugin_repo_path)
sudo('chown %s.%s %s' % (env.user, env.user, env.plugin_repo_path))
run('cp %s/plugin* %s' % (local_path, env.plugin_repo_path))
run('cp %s/icon* %s' % (code_path, env.plugin_repo_path))
run('cp %(local_path)s/inasafe-test.conf.templ '
'%(local_path)s/inasafe-test.conf' % {'local_path': local_path})
sed('%s/inasafe-test.conf' % local_path, 'inasafe-test.linfiniti.com',
env.repo_site_name)
with cd('/etc/apache2/sites-available/'):
if exists('inasafe-test.conf'):
sudo('a2dissite inasafe-test.conf')
fastprint('Removing old apache2 conf')
sudo('rm inasafe-test.conf')
sudo('ln -s %s/inasafe-test.conf .' % local_path)
# Add a hosts entry for local testing - only really useful for localhost
repo_hosts = '/etc/hosts'
if not contains(repo_hosts, 'inasafe-test'):
append(repo_hosts, '127.0.0.1 %s' % env.repo_site_name, use_sudo=True)
sudo('a2ensite inasafe-test.conf')
sudo('service apache2 reload')
def list():
"""
List remote backups.
"""
require('PROJECT')
fastprint(run('%(current)s/bin/s3cmd --config=%(share)s/s3cfg.ini ls s3://metapix/backup/%(instance)s/' % env.PROJECT))
def list():
"""
List remote configurations.
"""
require('PROJECT', provided_by=['stage', 'production'])
fastprint(run('cat %(settings)s' % env.PROJECT, quiet=True))
def setup_venv(code_path, requirements_file='requirements.txt'):
"""Initialise or update the virtual environment.
It will also ensure build-essential is installed, though you may need to
install required dev library using your own script.
:param code_path: Base directory under which the venv dir should be made.
:type code_path: str
:param requirements_file: Name of the requirements file to use in the
base directory. Defaults to ``requirements.txt``.
:type requirements_file: str
To run e.g.::
fab -H 192.168.1.1:2222 setup_venv
"""
setup_env()
fastprint(
blue('Setting up virtual env in: \n%s\nusing\n%s' %
(code_path, requirements_file)))
require_packages(['python-virtualenv', 'build-essential'])
with cd(code_path):
# Ensure we have a venv set up
virtualenv('venv')
run('venv/bin/pip install -r %s' % requirements_file)
fastprint(green('Virtualenv setup completed.'))
def sync_project_to_server():
"""Synchronize project with webserver ignoring venv and sqlite db..
This is a handy way to get your secret key to the server too...
"""
base_path, code_path, git_url, repo_alias, site_name = get_vars()
update_venv(code_path)
#noinspection PyArgumentEqualDefault
rsync_project(
base_path,
delete=False,
exclude=[
'*.pyc',
'.git',
'*.dmp',
'.DS_Store',
'projecta.db',
'venv',
'django_project/static'])
update_migrations()
with cd(os.path.join(code_path, 'django_project')):
run('touch core/wsgi.py')
set_media_permissions(code_path)
set_db_permissions()
update_migrations()
collectstatic()
fastprint(blue(
'Your server is now in synchronised to your local project\n'))
def setup_venv(code_path, requirements_file='requirements.txt'):
"""Initialise or update the virtual environment.
It will also ensure build-essential is installed, though you may need to
install required dev library using your own script.
:param code_path: Base directory under which the venv dir should be made.
:type code_path: str
:param requirements_file: Name of the requirements file to use in the
base directory. Defaults to ``requirements.txt``.
:type requirements_file: str
To run e.g.::
fab -H 192.168.1.1:2222 setup_venv
"""
setup_env()
fastprint(blue('Setting up virtual env in: \n%s\nusing\n%s' % (
code_path, requirements_file)))
require_packages(['python-virtualenv', 'build-essential'])
with cd(code_path):
# Ensure we have a venv set up
virtualenv('venv')
run('venv/bin/pip install -r %s' % requirements_file)
fastprint(green('Virtualenv setup completed.'))
def install_btsync():
"""Install btsync from ppa."""
fabtools.deb.update_index(quiet=True)
fabtools.require.deb.package('software-properties-common')
fabtools.require.deb.ppa('ppa:tuxpoldo/btsync')
fabtools.require.deb.package('btsync')
fastprint(green('BTSync installed'))
def get_kernel_details():
env.output_prefix = False
print("\n++++++++++ Kernel details ++++++++++")
with hide('output', 'running', 'warnings', 'aborts',
'status'), settings(warn_only=True):
fastprint(run('uname -a'))
print("\n++++++++++++++++++++++++++++++++++++")
def setup_tilemill(proxy_url=None):
"""Set up tile mill - see http://www.mapbox.com/tilemill/ .
We use a pure node setup as described here:
https://github.com/mapbox/tilemill/issues/2103#issuecomment-37473991
in order to avoid potential conflicts with ubuntugis.
:param proxy_url: Optional parameters to specify the url to use for your
network proxy. It should be specified in the form: ``<host>:<port>``.
The same proxy will be used for both http and https urls. If ommitted
no proxy will be used.
:type proxy_url: str
"""
# Note raring seems not to be supported yet...
setup_env()
repo_alias = 'tilemill'
print env
update_git_checkout(
env.fg.workspace, 'https://github.com/mapbox/tilemill.git', repo_alias)
work_path = os.path.join(env.fg.workspace, repo_alias)
setup_node(
work_path,
node_version='0.10.26', # known good version
proxy_url=proxy_url)
with cd(work_path):
run('env/bin/npm install')
fastprint(green(
'Tilemill installed. Use the start_tilemill task to run it.'))
def get_os_details():
env.output_prefix = False
print("\n++++++++++ OS details ++++++++++++++")
with hide('output', 'running', 'warnings', 'aborts',
'status'), settings(warn_only=True):
fastprint(run('cat /etc/*release | grep ^NAME='))
print("\n++++++++++++++++++++++++++++++++++++")
def list():
"""
List remote configurations.
"""
require('PROJECT')
fastprint(run('cat %(settings)s' % env.PROJECT, quiet=True))
def setup_docs_web_site(branch='master'):
"""Initialise an InaSAFE docs site where we host docs and pdf.
:param branch: Which branch of the documentation to build.
:type branch: str
"""
build_docs()
fabtools.require.deb.package('apache2')
apache_conf_template = 'inasafe-doc.conf.templ'
if not exists(web_directory):
require.directory('mkdir -p %s/pdf' % web_directory, True, 'web')
# TODO: Fix perms below
sudo('chown -R %s.%s %s' % ('web', 'web', web_directory))
apache_path = '/etc/apache2/sites-available/'
# Clone and replace tokens in apache conf
local_dir = os.path.dirname(__file__)
local_file = os.path.abspath(os.path.join(
local_dir,
'scripts',
apache_conf_template))
context = {
'server_name': 'inasafe.org', # Web Url e.g. foo.com
'web_master': '*****@*****.**', # email of web master
'document_root': web_directory, # Content root .e.g. /var/www
}
fastprint(green('Using %s for template' % local_file))
destination = '%s/inasafe-docs.conf' % apache_path
upload_template(
local_file,
destination,
context=context,
use_sudo=True)
with cd(code_path):
# Copy built Documentation to the Webserver path
run('cp -r docs/output/html/* %s' % web_directory)
run('cp -r docs/output/pdf %s' % web_directory)
run('cp scripts/.htaccess %s' % web_directory)
run('cp scripts/directory*.html %s/en/_static/' % web_directory)
# Add a hosts entry for local testing - only really useful for localhost
hosts = '/etc/hosts'
if not contains(hosts, 'inasafe-docs'):
append(hosts, '127.0.0.1 inasafe-doc.localhost', use_sudo=True)
require.apache.enable('inasafe-docs')
require.apache.disable('default')
sudo('a2enmod rewrite')
restart('apache2')
def setup_latex():
"""Install latex and friends needed to generate sphinx PDFs."""
fastprint(blue('Setting up LaTeX\n'))
fabtools.deb.update_index(quiet=True)
fabtools.require.deb.package('texlive-latex-extra')
fabtools.require.deb.package('texinfo')
fabtools.require.deb.package('texlive-fonts-recommended')
fastprint(green('Setting up LaTeX completed\n'))
def setup_website():
"""Initialise or update the git clone.
e.g. to update the server
fab -H 10.1.1.0:8697 remote setup_website
or if you have configured env.hosts, simply
fab remote setup_website
"""
_all()
fabtools.require.deb.package('libapache2-mod-wsgi')
# Find out if the wsgi user exists and create it if needed e.g.
fabtools.require.user(
env.wsgi_user,
create_group=env.wsgi_user,
system=True,
comment='System user for running the wsgi process under')
if not exists(env.webdir):
sudo('mkdir -p %s' % env.plugin_repo_path)
sudo('chown %s.%s %s' % (env.user, env.user, env.webdir))
# Clone and replace tokens in apache conf
conf_file = ('%s/resources/server_config/apache/%s.apache.conf' %
(env.code_path, env.repo_alias))
run('cp %(conf_file)s.templ %(conf_file)s' % {'conf_file': conf_file})
replace_tokens(conf_file)
with cd('/etc/apache2/sites-available/'):
if exists('%s.apache.conf' % env.repo_alias):
sudo('a2dissite %s.apache.conf' % env.repo_alias)
fastprint('Removing old apache2 conf', False)
sudo('rm %s.apache.conf' % env.repo_alias)
sudo('ln -s %s .' % conf_file)
# Add a hosts entry for local testing - only really useful for localhost
hosts = '/etc/hosts'
if not contains(hosts, env.repo_site_name):
append(hosts, '127.0.0.1 %s' % env.repo_site_name, use_sudo=True)
if not contains(hosts, 'www.' + env.repo_site_name):
append(hosts,
'127.0.0.1 %s' % 'www.' + env.repo_site_name,
use_sudo=True)
# Make sure mod rewrite is enabled
sudo('a2enmod rewrite')
# Enable the vhost configuration
sudo('a2ensite %s.apache.conf' % env.repo_alias)
# Check if apache configs are ok - script will abort if not ok
sudo('/usr/sbin/apache2ctl configtest')
sudo('a2dissite default')
fabtools.require.service.restarted('apache2')
def setup_tilestream_daemon(
tile_dir=None, ui_port=8888, tiles_port=8888, tile_host=None):
"""Start the tilestream service - ensure it is installed first.
.. warning:: This task does not work yet.
:param tile_dir: Optional directory on the remote tile_host that holds one
or more mbtiles files to be published. If ommitted the default of
`~/Documents/MapBox/tiles` will be used by tilestream. Be sure to
escape and paths given here. e.g.::
fab vagrant setup_tilestream_daemon:tile_dir='\/vagrant'
:type tile_dir: str
:param ui_port: Port on which the tilestream ui should be available.
:type ui_port: int
:param tiles_port: Port on which tilestream tile service should be
available. You may want to use a different port here and then run a
CDN such as cloudflare in front of the tile service.
:type tiles_port: int
:param tile_host: Host name under which the service will run. Must match the
hostname to which requests are made (even if you run it behind mod
proxy). If none it will be set to the IP address of the tile_host.
:type tile_host: str
"""
setup_env()
params = _build_parameters(tile_host, tile_dir, tiles_port, ui_port)
template_file = 'tilestream.templ'
template_path = os.path.abspath(os.path.join(
os.path.dirname(__file__),
os.path.pardir,
'../fabgis_resources',
'server_config',
'tilestream',
'%s' % template_file
))
remote_path = '/tmp/%s' % template_file
fastprint(template_path, remote_path)
put(template_path, remote_path)
binary = 'index.js'
if not tile_host:
tile_host = get_ip_address()
tokens = {
'BINARY': binary,
'OPTIONS': params,
'USER': env.fg.user,
'TILES_DIR': tile_dir,
'TILES_PORT': tiles_port,
'UI_PORT': ui_port,
'HOST': tile_host}
replace_tokens(remote_path, tokens)
sudo('mv /tmp/tilestream /etc/init.d/')
sudo('chmod 755 /etc/init.d/tilestream')
sudo('/etc/init.d/tilestream start')
sudo('sysv -rc-conf --level 2345 tilestream on')
def setup_tilestream_daemon(
tile_dir=None, ui_port=8888, tiles_port=8888, tile_host=None):
"""Start the tilestream service - ensure it is installed first.
.. warning:: This task does not work yet.
:param tile_dir: Optional directory on the remote tile_host that holds one
or more mbtiles files to be published. If ommitted the default of
`~/Documents/MapBox/tiles` will be used by tilestream. Be sure to
escape and paths given here. e.g.::
fab vagrant setup_tilestream_daemon:tile_dir='\/vagrant'
:type tile_dir: str
:param ui_port: Port on which the tilestream ui should be available.
:type ui_port: int
:param tiles_port: Port on which tilestream tile service should be
available. You may want to use a different port here and then run a
CDN such as cloudflare in front of the tile service.
:type tiles_port: int
:param tile_host: Host name under which the service will run. Must match the
hostname to which requests are made (even if you run it behind mod
proxy). If none it will be set to the IP address of the tile_host.
:type tile_host: str
"""
setup_env()
params = _build_parameters(tile_host, tile_dir, tiles_port, ui_port)
template_file = 'tilestream.templ'
template_path = os.path.abspath(os.path.join(
os.path.dirname(__file__),
os.path.pardir,
'fabgis_resources',
'server_config',
'tilestream',
'%s' % template_file
))
remote_path = '/tmp/%s' % template_file
fastprint(template_path, remote_path)
put(template_path, remote_path)
binary = 'index.js'
if not tile_host:
tile_host = get_ip_address()
tokens = {
'BINARY': binary,
'OPTIONS': params,
'USER': env.fg.user,
'TILES_DIR': tile_dir,
'TILES_PORT': tiles_port,
'UI_PORT': ui_port,
'HOST': tile_host}
replace_tokens(remote_path, tokens)
sudo('mv /tmp/tilestream /etc/init.d/')
sudo('chmod 755 /etc/init.d/tilestream')
sudo('/etc/init.d/tilestream start')
sudo('sysv -rc-conf --level 2345 tilestream on')
def update_migrations():
"""Apply any pending south migrations.
"""
command = "../venv/bin/python manage.py migrate"
base_path, code_path, git_url, repo_alias, site_name = get_vars()
with cd(os.path.join(code_path, "django_project")):
run(command)
run("touch core/wsgi.py")
fastprint(green("Note: your server is now has the latest SOUTH " "migrations applied.\n"))
def list():
"""
List remote backups.
"""
require('PROJECT')
fastprint(
run('%(current)s/bin/s3cmd --config=%(share)s/s3cfg.ini ls s3://metapix/backup/%(instance)s/'
% env.PROJECT))
def setup_docs_web_proxy():
"""Set up a mod proxy based vhost to forward web traffic to internal host.
If container_id is none, it will also install docker and set up the
entire documentation web site inside that docker container.
"""
require.directory(work_dir)
with cd(work_dir):
run('echo "fabgis" > requirements.txt')
setup_venv(work_dir)
container_id_file = 'fabgis.container.id'
if not exists(container_id_file):
setup_docker()
setup_remotely()
container_id = current_docker_container()
port_mappings = get_docker_port_mappings(container_id)
http_port = port_mappings[80]
fabtools.require.deb.package('apache2')
sudo('a2enmod proxy proxy_http')
context = {
'internal_host': env.host,
'internal_port': http_port,
'server_name': 'inasafe.org'
}
apache_conf_template = 'inasafe.org.mod_proxy.conf.templ'
apache_path = '/etc/apache2/sites-available'
# Clone and replace tokens in apache conf
local_dir = os.path.dirname(__file__)
local_file = os.path.abspath(os.path.join(
local_dir,
'scripts',
apache_conf_template))
fastprint(green('Using %s for template' % local_file))
destination = '%s/inasafe.org.conf' % apache_path
upload_template(
local_file,
destination,
context=context,
use_sudo=True)
require.apache.enable('inasafe.org')
restart('apache2')
def server_to_debug_mode():
"""Put the server in debug mode (normally not recommended)."""
base_path, code_path, git_url, repo_alias, site_name = get_vars()
config_file = os.path.join(code_path, "django_project", "core", "settings", "project.py")
sed(config_file, "DEBUG = TEMPLATE_DEBUG = False", "DEBUG = TEMPLATE_DEBUG = True")
with cd(os.path.join(code_path, "django_project")):
run("touch core/wsgi.py")
set_db_permissions()
collectstatic()
fastprint(red("Warning: your server is now in DEBUG mode!\n"))
def server_to_production_mode():
"""Put the server in production mode (recommended)."""
base_path, code_path, git_url, repo_alias, site_name = get_vars()
config_file = os.path.join(code_path, "django_project", "core", "settings", "project.py")
sed(config_file, "DEBUG = TEMPLATE_DEBUG = True", "DEBUG = TEMPLATE_DEBUG = False")
with cd(os.path.join(code_path, "django_project")):
run("touch core/wsgi.py")
set_db_permissions()
collectstatic()
fastprint(blue("Note: your server is now in PRODUCTION mode!\n"))
def install_oracle_jdk():
"""Install the official oracle jdk."""
fastprint(yellow('Setting up oracle java on host: %s' % env.host))
require_ppa('ppa:webupd8team/java')
apt_get_update()
require_package('software-properties-common')
sudo('echo oracle-java7-installer shared/accepted-oracle-license-v1-1 '
'select true | debconf-set-selections')
require_package('oracle-java7-installer')
fastprint(green('Installation of oracle java completed ok!'))
def update_migrations():
"""Apply any pending south migrations.
"""
command = '../venv/bin/python manage.py migrate'
base_path, code_path, git_url, repo_alias, site_name = get_vars()
with cd(os.path.join(code_path, 'django_project')):
run(command)
run('touch core/wsgi.py')
fastprint(green('Note: your server is now has the latest SOUTH '
'migrations applied.\n'))
def select_card():
if odb.arg.ofl is not None:
return odb.arg.ofl
keywordsSet = set(["MMC", "mmc", "card", "SDHC", "SD"])
print("Please insert SD card now.")
with quiet():
str_time = str(local("sudo dmesg|tail -n 1",
capture=True)).strip().split("]")[0][1:]
print(str_time)
lastTimestamp = float(str_time)
# lastTimestamp = str(local("sudo dmesg|tail -n 1",capture=True)).strip()
print(lastTimestamp)
sdPath = ""
while sdPath == "":
fastprint(".")
time.sleep(1)
with quiet():
dmsgTail = local("sudo dmesg|tail", capture=True).split("\n")
# print( dmsgTail)
possibleSDCard = False
detectedActivity = False
for i, row in enumerate(dmsgTail):
ts = float(str(row).strip().split("]")[0][1:])
if ts > lastTimestamp:
rowList = row.replace(":", "").split()
rowSet = set(rowList)
if not detectedActivity:
detectedActivity = True
print("Detected activity.")
with quiet():
print(
yellow(local("sudo dmesg|tail -n 4",
capture=True)))
if not possibleSDCard:
if len(rowSet.intersection(
keywordsSet)) == 0 or "removed" in rowSet:
lastTimestamp = ts
continue
TEMPsdPath = pjoin("/dev", rowList[1].strip())
if os.path.exists(TEMPsdPath):
print("Detected SD card.")
sdPath = TEMPsdPath
break
else:
possibleSDCard = True
odb.arg.ofl = sdPath
return odb.arg.ofl
def setup_apache(site_name, web_root, template_dir=None, **kwargs):
"""Set up the apache server for this site.
:param site_name: Name of the site e.g. cam.linfiniti.com. Should be a
single word with only alpha characters in it.
:type site_name: str
:param web_root: Directory where the content lives.
:type web_root: str
:param template_dir: Directory where the template files live. If none
will default to ``resources/server_config/apache``. Must be a
relative path to the fabfile you are running.
:type template_dir: str
:param kwargs: Any extra keyword arguments that should be appended to the
token list that will be used when rendering the apache config template.
Use this to pass in sensitive data such as passwords.
:type kwargs: dict
:returns: Path to the apache conf file.
:rtype: str
"""
setup_env()
# Ensure we have a mailserver setup for our domain
# Note that you may have problems if you intend to run more than one
# site from the same server
require.postfix.server(site_name)
# Clone and replace tokens in apache conf
if template_dir is None:
template_dir = 'resources/server_config/apache/'
filename = '%s.apache.conf.templ' % site_name
template_path = os.path.join(template_dir, filename)
fastprint(green('Using %s for template' % template_path))
context = {
'escaped_server_name': site_name.replace('.', '\.'),
'server_name': site_name,
'web_root': web_root,
'site_name': site_name
}
context.update(kwargs) # merge in any params passed in to this function
destination = '/etc/apache2/sites-available/%s.apache.conf' % site_name
fastprint(context)
upload_template(template_path, destination, context=context, use_sudo=True)
require.deb.package('apache2')
sudo('a2dissite default')
sudo('a2ensite %s.apache.conf' % site_name)
# Check if apache configs are ok - script will abort if not ok
sudo('/usr/sbin/apache2ctl configtest')
require.service.restarted('apache2')
return destination
def setup_docs_web_site(branch='master'):
"""Initialise an InaSAFE docs site where we host docs and pdf.
:param branch: Which branch of the documentation to build.
:type branch: str
"""
build_docs()
fabtools.require.deb.package('apache2')
apache_conf_template = 'inasafe-doc.conf.templ'
if not exists(web_directory):
require.directory('mkdir -p %s/pdf' % web_directory, True, 'web')
# TODO: Fix perms below
sudo('chown -R %s.%s %s' % ('web', 'web', web_directory))
apache_path = '/etc/apache2/sites-available/'
# Clone and replace tokens in apache conf
local_dir = os.path.dirname(__file__)
local_file = os.path.abspath(
os.path.join(local_dir, 'scripts', apache_conf_template))
context = {
'server_name': 'inasafe.org', # Web Url e.g. foo.com
'web_master': '*****@*****.**', # email of web master
'document_root': web_directory, # Content root .e.g. /var/www
}
fastprint(green('Using %s for template' % local_file))
destination = '%s/inasafe-docs.conf' % apache_path
upload_template(local_file, destination, context=context, use_sudo=True)
with cd(code_path):
# Copy built Documentation to the Webserver path
run('cp -r docs/output/html/* %s' % web_directory)
run('cp -r docs/output/pdf %s' % web_directory)
run('cp scripts/.htaccess %s' % web_directory)
run('cp scripts/directory*.html %s/en/_static/' % web_directory)
# Add a hosts entry for local testing - only really useful for localhost
hosts = '/etc/hosts'
if not contains(hosts, 'inasafe-docs'):
append(hosts, '127.0.0.1 inasafe-doc.localhost', use_sudo=True)
require.apache.enable('inasafe-docs')
require.apache.disable('default')
sudo('a2enmod rewrite')
restart('apache2')
def install_oracle_jdk():
"""Install the official oracle jdk."""
fastprint(yellow('Setting up oracle java on host: %s' % env.host))
require_ppa('ppa:webupd8team/java')
apt_get_update()
require_package('software-properties-common')
sudo(
'echo oracle-java7-installer shared/accepted-oracle-license-v1-1 '
'select true | debconf-set-selections')
require_package('oracle-java7-installer')
fastprint(green('Installation of oracle java completed ok!'))
def setup_inasafe():
"""Setup requirements for InaSAFE."""
fastprint(blue('Setting up InaSAFE dependencies\n'))
fabtools.require.deb.package('pep8')
fabtools.require.deb.package('pylint')
fabtools.require.deb.package('python-nose')
fabtools.require.deb.package('python-nosexcover')
fabtools.require.deb.package('python-pip')
fabtools.require.deb.package('python-numpy')
fabtools.require.deb.package('python-qt4')
fabtools.require.deb.package('python-nose')
fastprint(green('Setting up InaSAFE dependencies completed.\n'))
def setup_docs_web_proxy():
"""Set up a mod proxy based vhost to forward web traffic to internal host.
If container_id is none, it will also install docker and set up the
entire documentation web site inside that docker container.
"""
require.directory(work_dir)
with cd(work_dir):
run('echo "fabgis" > requirements.txt')
setup_venv(work_dir)
container_id_file = 'fabgis.container.id'
if not exists(container_id_file):
setup_docker()
setup_remotely()
container_id = current_docker_container()
port_mappings = get_docker_port_mappings(container_id)
http_port = port_mappings[80]
fabtools.require.deb.package('apache2')
sudo('a2enmod proxy proxy_http')
context = {
'internal_host': env.host,
'internal_port': http_port,
'server_name': 'inasafe.org'
}
apache_conf_template = 'inasafe.org.mod_proxy.conf.templ'
apache_path = '/etc/apache2/sites-available'
# Clone and replace tokens in apache conf
local_dir = os.path.dirname(__file__)
local_file = os.path.abspath(
os.path.join(local_dir, 'scripts', apache_conf_template))
fastprint(green('Using %s for template' % local_file))
destination = '%s/inasafe.org.conf' % apache_path
upload_template(local_file,
destination,
context=context,
use_sudo=True)
require.apache.enable('inasafe.org')
restart('apache2')
def server_to_production_mode():
"""Put the server in production mode (recommended)."""
base_path, code_path, git_url, repo_alias, site_name = get_vars()
config_file = os.path.join(
code_path, 'django_project', 'core', 'settings', 'project.py')
sed(
config_file,
'DEBUG = TEMPLATE_DEBUG = True',
'DEBUG = TEMPLATE_DEBUG = False')
with cd(os.path.join(code_path, 'django_project')):
run('touch core/wsgi.py')
set_db_permissions()
collectstatic()
fastprint(blue('Note: your server is now in PRODUCTION mode!\n'))
def server_to_production_mode():
"""Put the server in production mode (recommended)."""
base_path, code_path, git_url, repo_alias, site_name = get_vars()
config_file = os.path.join(
code_path, 'django_project', 'core', 'settings', 'project.py')
sed(
config_file,
'DEBUG = TEMPLATE_DEBUG = True',
'DEBUG = TEMPLATE_DEBUG = False')
with cd(os.path.join(code_path, 'django_project')):
run('touch core/wsgi.py')
set_db_permissions()
collectstatic()
fastprint(blue('Note: your server is now in PRODUCTION mode!\n'))
def setup_mapserver():
"""Install UMN Mapserver from apt and add 900913 CRS to proj epsg file."""
fastprint(cyan('Setting up UMN Mapserver\n'))
# Clone and replace tokens in mapserver map file
# Clone and replace tokens in mapserver conf
fabtools.require.deb.package('cgi-mapserver')
# We also need to append 900913 epsg code to the proj epsg list
epsg_path = '/usr/share/proj/epsg'
epsg_code = (
'<900913> +proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_'
'0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs')
epsg_id = '900913'
if not contains(epsg_path, epsg_id):
append(epsg_path, epsg_code, use_sudo=True)
def start_tilemill():
"""Start the tilemill service - ensure it is installed first."""
# sudo('start tilemill')
# Note port forward seems not to work well
# Using this config on the vhost:
# {
# 'listenHost': '0.0.0.0',
# 'coreUrl': '192.168.1.115:20009',
# 'tileUrl': '192.168.1.115:20008',
# 'files': '/usr/share/mapbox',
# 'server': true
# }
# Worked, accessible from http://192.168.1.115:20009/
# More reliable way (blocking process, ctl-c to kill)
host_ip = get_ip_address()
fastprint(host_ip)
command = (
"/usr/bin/nodejs /usr/share/tilemill/index.js "
"--server=true "
"--listenHost=0.0.0.0 "
"--coreUrl=%s:20009 "
"--tileUrl=%s:20008" % (host_ip, host_ip)
)
run(command)
fastprint("Tilemill is running - point your browser at:")
fastprint("http://%s:20009" % host_ip)
fastprint("to use it.")
def server_to_debug_mode():
"""Put the server in debug mode (normally not recommended)."""
base_path, code_path, git_url, repo_alias, site_name = get_vars()
config_file = os.path.join(
code_path, 'django_project', 'core', 'settings', 'project.py')
sed(
config_file,
'DEBUG = TEMPLATE_DEBUG = False',
'DEBUG = TEMPLATE_DEBUG = True')
with cd(os.path.join(code_path, 'django_project')):
run('touch core/wsgi.py')
set_db_permissions()
collectstatic()
fastprint(red('Warning: your server is now in DEBUG mode!\n'))
def setup_docker_user(user=None):
"""Setup a user with rights to run docker after their next relogin.
:param user: The user that should be granted docker rights. If none,
env.user will be assumed.
:type user: str
"""
setup_docker_group()
if user is None:
user = env.user
sudo('usermod -a -G docker %s' % user)
fastprint(
red('User %s will be able to run docker without sudo on their '
'next log in.' % user))
def setup_docker_user(user=None):
"""Setup a user with rights to run docker after their next relogin.
:param user: The user that should be granted docker rights. If none,
env.user will be assumed.
:type user: str
"""
setup_docker_group()
if user is None:
user = env.user
sudo('usermod -a -G docker %s' % user)
fastprint(red(
'User %s will be able to run docker without sudo on their '
'next log in.' % user))
def create_user(user, password=None):
"""Create a user on the remote system matching the user running this task.
:param user: User name for the new user.
:type user: str
:param password: Password for new user - will prompt interactively if None.
:type password: str
"""
if password is None:
fastprint(red('Please enter a password for the new web user.\n'))
password = getpass()
fabtools.require.users.user(user, password=password)
fabtools.require.users.sudoer(user)
def create_user(user, password=None):
"""Create a user on the remote system matching the user running this task.
:param user: User name for the new user.
:type user: str
:param password: Password for new user - will prompt interactively if None.
:type password: str
"""
if password is None:
fastprint(red('Please enter a password for the new web user.\n'))
password = getpass()
fabtools.require.users.user(user, password=password)
fabtools.require.users.sudoer(user)
def server_to_debug_mode():
"""Put the server in debug mode (normally not recommended)."""
base_path, code_path, git_url, repo_alias, site_name = get_vars()
config_file = os.path.join(
code_path, 'django_project', 'core', 'settings', 'project.py')
sed(
config_file,
'DEBUG = TEMPLATE_DEBUG = False',
'DEBUG = TEMPLATE_DEBUG = True')
with cd(os.path.join(code_path, 'django_project')):
run('touch core/wsgi.py')
set_db_permissions()
collectstatic()
fastprint(red('Warning: your server is now in DEBUG mode!\n'))
def start_tilemill():
"""Start the tilemill service - ensure it is installed first."""
#sudo('start tilemill')
# Note port forward seems not to work well
# Using this config on the vhost:
#{
# 'listenHost': '0.0.0.0',
# 'coreUrl': '192.168.1.115:20009',
# 'tileUrl': '192.168.1.115:20008',
# 'files': '/usr/share/mapbox',
# 'server': true
#}
# Worked, accessible from http://192.168.1.115:20009/
# More reliable way (blocking process, ctl-c to kill)
host_ip = get_ip_address()
fastprint(host_ip)
command = (
'/usr/bin/nodejs /usr/share/tilemill/index.js '
'--server=true '
'--listenHost=0.0.0.0 '
'--coreUrl=%s:20009 '
'--tileUrl=%s:20008' % (host_ip, host_ip))
run(command)
fastprint('Tilemill is running - point your browser at:')
fastprint('http://%s:20009' % host_ip)
fastprint('to use it.')
def get_private():
"""Copy the private.py with site specific settings to the server."""
base_path, code_path, git_url, repo_alias, site_name = get_vars()
local_path = os.path.join("django_project", "core", "settings", "private.py")
remote_path = os.path.join(code_path, local_path)
if not exists(remote_path):
fastprint(red("Please create your remove copy of private.py\n"))
fastprint(red("See README.md for more instructions."))
raise Exception("Could not get remote private settings")
get(remote_path=remote_path, local_path=local_path)
def setup_tilemill():
"""Set up tile mill - see http://www.mapbox.com/tilemill/ ."""
# Note raring seems not to be supported yet...
setup_env()
add_developmentseed_ppa()
fabtools.require.deb.package('tilemill')
fabtools.require.deb.package('libmapnik')
# TODO: switch to using nodeenv
# SEE: https://github.com/ekalinin/nodeenv
fabtools.require.deb.package('nodejs')
fastprint('Now you can log in and use tilemill like this:')
fastprint('vagrant ssh -- -X')
fastprint('/usr/bin/nodejs /usr/share/tilemill/index.js')
fastprint('Or use the start tilemill task and open your')
fastprint('browser at the url provided.')
def put_private():
"""Copy the private.py with site specific settings to the server."""
base_path, code_path, git_url, repo_alias, site_name = get_vars()
local_path = os.path.join(
'django_project', 'core', 'settings', 'private.py')
if not os.path.exists(local_path):
fastprint(red('Please create your local copy of private.py\n'))
fastprint(red('See README.md for more instructions.'))
raise Exception('Could not copy local private settings to server')
remote_path = os.path.join(code_path, local_path)
put(local_path=local_path, remote_path=remote_path)
def setup_sphinx():
"""Install sphinx from pip.
We prefer packages from pip as ubuntu packages are usually old.
To build the Documentation we also need to check and update the
subjacent docutils installation"""
fastprint(blue('Setting up Sphinx\n'))
if is_installed('docutils-common'):
sudo('apt-get remove docutils-common')
if is_installed('docutils-doc'):
sudo('apt-get remove docutils-doc')
if is_installed('python-docutils'):
sudo('apt-get remove python-docutils')
sudo('pip install --upgrade docutils==0.10')
sudo('pip install sphinx')
fastprint(green('Setting up Sphinx completed\n'))
def setup_tilestream():
"""Set up tile stream - see https://github.com/mapbox/tilestream.
This one deserves a little explanation:
Tilestream is a nodejs application. Node seems to be pretty unparticular
about maintaining api compatibility between releases so if you grab one
from e.g. apt, chances are it won't work with tilestream.
To address this, we use the nodeenv virtualisation environment (somewhat
equivalent to using python virtualenv) to ensure that we have the
expected version of tilestream. e.g.::
nodeenv env --node=0.8.15
"""
setup_env()
require.deb.package('curl')
require.deb.package('build-essential')
require.deb.package('libssl-dev')
require.deb.package('libsqlite3-0')
require.deb.package('libsqlite3-dev')
require.deb.package('git-core')
require.deb.package('nodejs nodejs-dev npm')
require.deb.package('python-pip')
sudo('pip install nodeenv')
dev_dir = '/home/%s/dev/javascript' % env.fg.user
fastprint('making directory %s' % dev_dir)
require.directory(dev_dir)
tile_stream_dir = os.path.join(dev_dir, 'tilestream')
fastprint('checkout out tilestream to %s' % tile_stream_dir)
if not exists(tile_stream_dir):
with cd(dev_dir):
run('git clone http://github.com/mapbox/tilestream.git')
with cd(tile_stream_dir):
if not exists(os.path.join(tile_stream_dir, 'env')):
run('nodeenv env --node=0.8.15')
# If doing this interactively from a shell you would first do:
# . env/bin/activate
# npm install
# From our scripted environment (where we cant activate the venv):
run('env/bin/npm install')
def get_docker_port_mappings(container_id):
"""Given the id of a container, get the ports mapped for that container.
This is basically a wrapper for docker ps that parses the output and
returns the ports that are mapped as a dict where the keys are the
container internal ports and the values are the container external ports.
:param container_id: ID for the container to obtain port mappings for.
:type container_id: str
e.g. for this container::
d3caf337bfc1 dhrp/sshd:latest /usr/sbin/sshd -D 8 minutes \
ago Up 8 minutes 49171->22, 49172->8000
The following dictionary would be returned::
{
22: 49171,
8000, 49172
}
"""
ports = sudo(
'docker ps | grep %s | awk '
'\'{ s = ""; for (i = 11; i <= NF; i++) s = s $i " "; print s }\''
% container_id)
#fastprint('Ports: %s\n' % str(ports))
if ', ' in ports:
tokens = ports.split(', ')
else:
tokens = [ports]
mappings = {}
#fastprint('Tokens: %s\n' % str(tokens))
for token in tokens:
parts = token.split('->')
#fastprint('Parts: %s\n' % str(parts))
host_port = int(parts[0])
container_port = int(parts[1])
mappings[container_port] = host_port
fastprint('Mappings: %s\n' % str(mappings))
return mappings
def freshen():
"""Freshen the server with latest git copy and touch wsgi.
.. note:: Preferred normal way of doing this is rather to use the
sync_project_to_server task and not to checkout from git.
"""
base_path, code_path, git_url, repo_alias, site_name = get_vars()
git_url = 'http://github.com/timlinux/osm-reporter.git'
update_git_checkout(base_path, git_url, repo_alias)
with cd(code_path):
run('touch apache/osm-reporter.wsgi')
fastprint('*******************************************\n')
fastprint(red(' Don\'t forget set ALLOWED_HOSTS in \n'))
fastprint(' django_project/core/settings/prod.py\n')
fastprint(' to the domain name for the site.\n')
fastprint('*******************************************\n')
