def handle(self, *args, **options):
context = {
'username': options.get('username'),
'manage_file': os.path.join(get_site_root(), 'manage.py'),
}
exists = "crontab -u %(username)s -l | grep ' monitorlocalsystem' &> /dev/null"
exists = run(exists % context, display=False, err_codes=[0,1]).return_code
if exists != 0:
cmd = (
'{ crontab -l -u %(username)s;'
' echo "*/5 * * * * python %(manage_file)s monitorlocalsystem --email --quiet"; '
'} | crontab -u %(username)s -') % context
cmd = run(cmd)
print cmd.return_code
else:
self.stdout.write('Done nothing, crontab was already installed')
def handle(self, *args, **options):
context = {
'username': options.get('username'),
'manage_file': os.path.join(get_site_root(), 'manage.py'),
}
exists = "crontab -u %(username)s -l | grep ' monitorlocalsystem' &> /dev/null"
exists = run(exists % context, display=False,
err_codes=[0, 1]).return_code
if exists != 0:
cmd = (
'{ crontab -l -u %(username)s;'
' echo "*/5 * * * * python %(manage_file)s monitorlocalsystem --email --quiet"; '
'} | crontab -u %(username)s -') % context
cmd = run(cmd)
print cmd.return_code
else:
self.stdout.write('Done nothing, crontab was already installed')
import os
from django import forms
from controller.utils.paths import get_site_root
from controller.utils.system import run
from api import serializers
from firmware.image import Image
from firmware.plugins import FirmwarePlugin
from firmware.settings import FIRMWARE_PLUGINS_USB_IMAGE
usb_image = FIRMWARE_PLUGINS_USB_IMAGE % {'site_root': get_site_root()}
class USBImagePlugin(FirmwarePlugin):
verbose_name = 'USB image'
description = (
'Optionally puts the firmware image into confine-install USB image.\n'
'The base image can be downloaded from http://media.confine-project.eu/'
'confine-install/confine-install.img.gz and stored in %s.' % usb_image)
enabled_by_default = True
def get_form(self):
class USBImageForm(forms.Form):
usb_image = forms.BooleanField(
label='USB Image',
required=False,
help_text=
'Select this option if you want to install the node image '
'from a USB stick. This option requires a node internal hard drive.'
def handle(self, *args, **options):
interactive = options.get('interactive')
# Avoid import errors
from nodes.models import Server
server = Server.objects.first()
context = {
'project_name': get_project_name(),
'project_root': get_project_root(),
'site_root': get_site_root(),
'media_root': settings.MEDIA_ROOT,
'static_root': settings.STATIC_ROOT,
'cert_path': ca.cert_path,
'cert_key_path': ca.priv_key_path,
'mgmt_addr': str(server.mgmt_net.addr),
'user': options.get('user'),
'group': options.get('group') or options.get('user'),
'home': expanduser("~%s" % options.get('user')),
'processes': int(options.get('processes')),
}
nginx_conf = (
'server {\n'
' listen 80;\n'
' listen [::]:80 ipv6only=on;\n'
' rewrite ^/$ /admin;\n'
' client_max_body_size 500m;\n'
' location / {\n'
' uwsgi_pass unix:///var/run/uwsgi/app/%(project_name)s/socket;\n'
' include uwsgi_params;\n'
' }\n'
' location /media {\n'
' alias %(media_root)s;\n'
' expires 30d;\n'
' }\n'
' location /static {\n'
' alias %(static_root)s;\n'
' expires 30d;\n'
' }\n'
'}\n'
'\n'
'server {\n'
' listen [%(mgmt_addr)s]:443 default_server ssl;\n'
' ssl_certificate %(cert_path)s;\n'
' ssl_certificate_key %(cert_key_path)s;\n'
' rewrite ^/$ /admin;\n'
' client_max_body_size 50m;\n'
' location / {\n'
' uwsgi_pass unix:///var/run/uwsgi/app/%(project_name)s/socket;\n'
' include uwsgi_params;\n'
' }\n'
' location /media {\n'
' alias %(media_root)s;\n'
' expires 30d;\n'
' }\n'
' location /static {\n'
' alias %(static_root)s;\n'
' expires 30d;\n'
' }\n'
'}\n') % context
uwsgi_conf = (
'[uwsgi]\n'
'plugins = python\n'
'chdir = %(site_root)s\n'
'module = %(project_name)s.wsgi\n'
'master = true\n'
'processes = %(processes)d\n'
'chmod-socket = 664\n'
'stats = /run/uwsgi/%%(deb-confnamespace)/%%(deb-confname)/statsocket\n'
'vacuum = true\n'
'uid = %(user)s\n'
'gid = %(group)s\n'
'env = HOME=%(home)s\n') % context
nginx = {
'file': '/etc/nginx/conf.d/%(project_name)s.conf' % context,
'conf': nginx_conf
}
uwsgi = {
'file': '/etc/uwsgi/apps-available/%(project_name)s.ini' % context,
'conf': uwsgi_conf
}
for extra_context in (nginx, uwsgi):
context.update(extra_context)
diff = run("echo '%(conf)s'|diff - %(file)s" % context,
err_codes=[0, 1, 2])
if diff.return_code == 2:
# File does not exist
run("echo '%(conf)s' > %(file)s" % context)
elif diff.return_code == 1:
# File is different, save the old one
if interactive:
msg = (
"\n\nFile %(file)s be updated, do you like to override "
"it? (yes/no): " % context)
confirm = input(msg)
while 1:
if confirm not in ('yes', 'no'):
confirm = input(
'Please enter either "yes" or "no": ')
continue
if confirm == 'no':
return
break
run("cp %(file)s %(file)s.save" % context)
run("echo '%(conf)s' > %(file)s" % context)
self.stdout.write(
"\033[1;31mA new version of %(file)s has been installed.\n "
"The old version has been placed at %(file)s.save\033[m" %
context)
run('ln -s /etc/uwsgi/apps-available/%(project_name)s.ini /etc/uwsgi/apps-enabled/'
% context,
err_codes=[0, 1])
# Disable default site
run('rm -f /etc/nginx/sites-enabled/default')
# Give read permissions to cert key file
run('chmod g+r %(cert_key_path)s' % context)
# nginx should start after tincd
current = "\$local_fs \$remote_fs \$network \$syslog"
run('sed -i "s/ %s$/ %s \$named/g" /etc/init.d/nginx' %
(current, current))
rotate = (
'/var/log/nginx/*.log {\n'
' daily\n'
' missingok\n'
' rotate 30\n'
' compress\n'
' delaycompress\n'
' notifempty\n'
' create 640 root adm\n'
' sharedscripts\n'
' postrotate\n'
' [ ! -f /var/run/nginx.pid ] || kill -USR1 `cat /var/run/nginx.pid`\n'
' endscript\n'
'}\n')
run("echo '%s' > /etc/logrotate.d/nginx" % rotate)
# Allow nginx to write to uwsgi socket
run('adduser www-data %(group)s' % context)
# Restart nginx to apply new configuration and also uwsgi
# to force reloading permissions (user added to a group)
run('service nginx restart')
run('service uwsgi restart')
import os
from django import forms
from controller.utils.paths import get_site_root
from controller.utils.system import run
from api import serializers
from firmware.image import Image
from firmware.plugins import FirmwarePlugin
from firmware.settings import FIRMWARE_PLUGINS_USB_IMAGE
usb_image = FIRMWARE_PLUGINS_USB_IMAGE % {'site_root': get_site_root()}
class USBImagePlugin(FirmwarePlugin):
verbose_name = 'USB image'
description = ('Optionally puts the firmware image into confine-install USB image.\n'
'The base image can be downloaded from http://media.confine-project.eu/'
'confine-install/confine-install.img.gz and stored in %s.' % usb_image)
enabled_by_default = True
def get_form(self):
class USBImageForm(forms.Form):
usb_image = forms.BooleanField(label='USB Image', required=False,
help_text='Select this option if you want to install the node image '
'from a USB stick. This option requires a node internal hard drive.')
return USBImageForm
def get_serializer(self):
def cert_path(self):
return settings.PKI_CA_CERT_PATH % { 'site_root': get_site_root() }
def priv_key_path(self):
return settings.PKI_CA_PRIV_KEY_PATH % {'site_root': get_site_root()}
def handle(self, *args, **options):
if options.get('username'):
warnings.warn('"username" has been deprecated in favor of "user"',
DeprecationWarning)
if options.get('user') and options.get('username'):
raise CommandError(
"Only one of this options should be provided: --user OR --username"
)
user = options.get('user') or options.get('username') or 'confine'
context = {
'site_root': get_site_root(),
'user': user,
'group': options.get('group') or user,
'bin_path': path.join(get_controller_root(), 'bin'),
'processes': options.get('processes'),
'greenlets': options.get('greenlets')
}
celery_config = (
'# Name of nodes to start, here we have a single node\n'
'CELERYD_NODES="w1 w2"\n'
'\n'
'# Where to chdir at start.\n'
'CELERYD_CHDIR="%(site_root)s"\n'
'\n'
'# How to call "manage.py celeryd_multi"\n'
'CELERYD_MULTI="$CELERYD_CHDIR/manage.py celeryd_multi"\n'
'\n'
'# Extra arguments to celeryd\n'
'CELERYD_OPTS="-P:w1 processes -c:w1 %(processes)s -Q:w1 celery \\\n'
' -P:w2 gevent -c:w2 %(greenlets)s -Q:w2 gevent --time-limit=300"\n'
'\n'
'# Name of the celery config module.\n'
'CELERY_CONFIG_MODULE="celeryconfig"\n'
'\n'
'# %%n will be replaced with the nodename.\n'
'CELERYD_LOG_FILE="/var/log/celery/%%n.log"\n'
'CELERYD_PID_FILE="/var/run/celery/%%n.pid"\n'
'CELERY_CREATE_DIRS=1\n'
'\n'
'# Full path to the celeryd logfile.\n'
'CELERYEV_LOG_FILE="/var/log/celery/celeryev.log"\n'
'CELERYEV_PID_FILE="/var/run/celery/celeryev.pid"\n'
'\n'
'# Workers should run as an unprivileged user.\n'
'CELERYD_USER="******"\n'
'CELERYD_GROUP="%(group)s"\n'
'\n'
'# Persistent revokes\n'
'CELERYD_STATE_DB="$CELERYD_CHDIR/persistent_revokes"\n'
'\n'
'# Celeryev\n'
'CELERYEV="$CELERYD_CHDIR/manage.py"\n'
'CELERYEV_CAM="djcelery.snapshot.Camera"\n'
'CELERYEV_USER="******"\n'
'CELERYEV_GROUP="$CELERYD_GROUP"\n'
'CELERYEV_OPTS="celeryev"\n'
'\n'
'# Celerybeat\n'
'CELERYBEAT="${CELERYD_CHDIR}/manage.py celerybeat"\n'
'CELERYBEAT_USER="******"\n'
'CELERYBEAT_GROUP="$CELERYD_GROUP"\n'
'CELERYBEAT_CHDIR="$CELERYD_CHDIR"\n'
'CELERYBEAT_OPTS="--schedule=/var/run/celerybeat-schedule"\n' %
context)
run("echo '%s' > /etc/default/celeryd" % celery_config)
# https://raw.github.com/celery/celery/master/extra/generic-init.d/
for script in ['celeryevcam', 'celeryd', 'celerybeat']:
context['script'] = script
run('cp %(bin_path)s/%(script)s /etc/init.d/%(script)s' % context)
run('chmod +x /etc/init.d/%(script)s' % context)
run('update-rc.d %(script)s defaults' % context)
rotate = ('/var/log/celery/*.log {\n'
' weekly\n'
' missingok\n'
' rotate 10\n'
' compress\n'
' delaycompress\n'
' notifempty\n'
' copytruncate\n'
'}')
run("echo '%s' > /etc/logrotate.d/celeryd" % rotate)
def handle(self, *args, **options):
# Warn about deprecated options
if options.get('local'):
self.stderr.write("Warning: 'local' option is deprecated and will be ignored.\n")
version = options.get('version')
upgrade_notes = []
if version:
try:
major, major2, minor = decode_version(version)
except ValueError as e:
raise CommandError(e)
# Represent version as two digits per number: 1.2.2 -> 10202
version = int(str(major) + "%02d" % int(major2) + "%02d" % int(minor))
# Pre-upgrade operations (version specific)
if version < 835:
# prevent schema migrations from failing
if is_installed('firmware'):
from firmware.models import Build
Build.objects.filter(base_image=None).update(base_image='')
if version <= 902:
if is_installed('maintenance'):
# Apply losed migrations
from south.models import MigrationHistory
migrated = MigrationHistory.objects.filter(app_name='maintenance').exists()
if not migrated:
run('python manage.py migrate maintenance 0001 --fake')
if version < 1002:
# Update monitor settings (fix typo and add DiskFreeMonitor)
context = {
'settings': run("find . -type f -name 'settings.py'|grep -v 'vct/'")
}
# Try automaticate update (making a backup)
if context['settings']:
run("cp %(settings)s %(settings)s.upgrade.bak" % context)
# fix NumProcessesMonitor typo
run("sed -i 's/NumPocessesMonitor/NumProcessesMonitor/g' "
"%(settings)s" % context)
# append disk monitor (if needed)
# this is a rude check (but runned in a conservative way)
if 'DiskFreeMonitor' not in open(context['settings']).read():
run("sed -i '/MONITOR_MONITORS = ($/ a\ "
" (\"monitor.monitors.DiskFreeMonitor\",),' "
"%(settings)s" % context)
# warn the user about settings changes
autoupdate_status = 'OK' if context['settings'] else 'FAIL'
upgrade_notes.append('The monitor application has changed and .'
'some settings updates are required:\n'
' - Fix typo on NumProcessesMonitor (missing "r")\n'
' - Enable disk monitor\n'
' Please read the monitor app doc (MONITOR_MONITORS setting)\n'
'AUTOUPDATE: %s' % autoupdate_status)
if version <= 1102:
# Handle InconsistentMigrationHistory on tinc app
# * v0.11.2 tinc includes 0022, 0028..0030
# * v0.11.3 tinc adds 0023..0027
# We can force south to merge migrations because
# are implemented to be runned without dependencies
run('python manage.py migrate tinc 0030 --merge --noinput')
if not options.get('specifics_only'):
# Common stuff
development = options.get('development')
controller_admin = os.path.join(os.path.dirname(__file__), '../../bin/')
controller_admin = os.path.join(controller_admin, 'controller-admin.sh')
run('chmod +x %s' % controller_admin)
extra = '--development' if development else ''
if options.get('proxy'):
extra += ' --proxy %s' % options.get('proxy')
run("%s install_requirements " % controller_admin + extra)
run("python manage.py collectstatic --noinput")
run("python manage.py syncdb --noinput")
run("python manage.py migrate --noinput")
if is_installed('firmware'):
run("python manage.py syncfirmwareplugins")
if is_installed('notifications'):
run("python manage.py syncnotifications")
if is_installed('resources'):
run("python manage.py syncresources")
if options.get('restart'):
run("python manage.py restartservices")
if not version:
self.stderr.write('\nNext time you migth want to provide a --from argument '
'in order to run version specific upgrade operations\n')
return
# Post-upgrade operations (version specific)
if version <= 629:
# Clean existing sessions because of change on auth backend
run('echo "delete from django_session;" | python manage.py dbshell')
if version < 801:
deprecate_periodic_tasks(('state.ping',))
if version < 809:
# Add PKI directories
from pki import ca
from controller.utils.paths import get_site_root
site_root = get_site_root()
username = run("stat -c %%U %s" % site_root)
get_dir = lambda f: os.path.dirname(getattr(ca, f+'_path'))
for d in set( get_dir(f) for f in ['priv_key', 'pub_key', 'cert'] ):
run('mkdir -p %s' % d)
run('chown %s %s' % (username, d))
upgrade_notes.append('HTTPS certificate support for the management '
'network has been introduced in version 0.8.9.\n'
'In order to use it you sould run:\n'
' > python manage.py setuppki\n'
' > sudo python manage.py setupapache\n')
if version < 838:
# Purge communitynetworks.periodic_cache_node_db
from djcelery.models import PeriodicTask
PeriodicTask.objects.filter(name='communitynetworks.periodic_cache_node_db').delete()
run('rabbitmqctl stop_app')
run('rabbitmqctl reset')
run('rabbitmqctl start_app')
run('service celeryd restart')
upgrade_notes.append('New Celeryd init.d configuration has been '
'introduced in 0.8.38.\nIt is strongly recommended to upgrade by\n'
' > sudo python manage.py setupceleryd\n')
# Deprecate x86 and amd64 architectures
from nodes.models import Node
Node.objects.filter(arch='x86').update(arch='i686')
Node.objects.filter(arch='amd64').update(arch='x86_64')
upgrade_notes.append('In order to support Base authentication while downloading '
'firmwares you should add "WSGIPassAuthorization On" on your apache config.\n'
'Alternatively you can perform this operation with the following command\n'
' > sudo python manage.py setupapache\n'
' > /etc/init.d/apache2 reload\n')
if version < 900:
upgrade_notes.append('Apache configuration is now placed under '
'/etc/apache2/conf.d/<project_name>.conf. It is convenient for you '
'to migrate your current configuration located on /etc/apache2/httpd.conf '
'to this new location.\n')
upgrade_notes.append('Celery workers configuration has been updated. '
'Please update it by running:\n'
' > sudo python manage.py setupceleryd\n')
if version < 905:
# TODO find the root cause of this
# maybe is shit imported on settings that import settings like add_app
# Prevent crazy import erros to appear :S
from django.utils import translation
translation.activate('en-us')
# Change template types for more generic ones
from slices.models import Template
from slices.settings import SLICES_TEMPLATE_TYPES
template_types = [ t[0] for t in SLICES_TEMPLATE_TYPES ]
if 'debian' in template_types:
Template.objects.filter(type='debian6').update(type='debian')
if 'openwrt' in template_types:
Template.objects.filter(type='openwrt-backfire').update(type='openwrt')
if version < 906:
deprecate_periodic_tasks(('state.nodestate', 'state.sliverstate'))
if version <= 907:
# Generate sha256
from slices.models import Template
for template in Template.objects.all():
template.save()
upgrade_notes.append("It is extremly recommended to update your database "
"settings to enable atomic request behaviour:\n"
" https://docs.djangoproject.com/en/dev/topics/db/transactions/#tying-transactions-to-http-requests\n"
"Just add:\n"
" 'ATOMIC_REQUESTS': True,\n"
"into DATABASES setting within <project_name>/<project_name>/settings.py")
if version <= 1003:
# Update firmware configuration after Island refactor (#264)
from firmware.models import ConfigFile
try:
cfg_file = ConfigFile.objects.get(path__contains="node.tinc.connect_to")
except (ConfigFile.DoesNotExist, ConfigFile.MultipleObjectsReturned):
# Warn the user that needs to perform manual update
msg = "Firmware configuration update has failed. "
else:
cfg_file.content = cfg_file.content.replace("node.tinc.island", "node.island")
cfg_file.save()
msg = "Firmware configuration updated successfully. Updated ConfigFile ID: %i." % cfg_file.pk
upgrade_notes.append("%s\nPlease check version 0.10.4 release notes:\n"
"https://wiki.confine-project.eu/soft:server-release-notes#section0104" % msg)
if version < 1103:
# Update mgmt network Server APIs certificate
# perform raw SQL querie because models hasn't been
# reloaded yet and cannot access e.g. server.api
from django.db import connection
from nodes.models import Server
from pki import ca
server_id = Server.objects.order_by('id').first().pk
try:
cert = ca.get_cert().as_pem()
except IOError:
msg = ("Failed to update Server APIs certificate. Missing "
"server certificate '%s'.\n"
"Run 'python manage.py setuppki --help'" % ca.cert_path)
upgrade_notes.append(msg)
else:
update_sql = ('UPDATE "nodes_serverapi" SET cert = %s '
'WHERE "nodes_serverapi"."server_id" = %s')
cursor = connection.cursor()
cursor.execute(update_sql, [cert, server_id])
del cursor
upgrade_notes.append("Updated Server APIs certificate.")
if upgrade_notes and options.get('print_upgrade_notes'):
self.stdout.write('\n\033[1m\n'
' ===================\n'
' ** UPGRADE NOTES **\n'
' ===================\n\n' +
'\n'.join(upgrade_notes) + '\033[m\n')
def handle(self, *args, **options):
interactive = options.get('interactive')
# Avoid import errors
from nodes.models import Server
server = Server.objects.first()
context = {
'project_name': get_project_name(),
'project_root': get_project_root(),
'site_root': get_site_root(),
'media_root': settings.MEDIA_ROOT,
'static_root': settings.STATIC_ROOT,
'cert_path': ca.cert_path,
'cert_key_path': ca.priv_key_path,
'mgmt_addr': str(server.mgmt_net.addr),
'user': options.get('user'),
'group': options.get('group') or options.get('user'),
'processes': int(options.get('processes')),
'threads': int(options.get('threads'))
}
apache_conf = (
'WSGIDaemonProcess %(project_name)s user=%(user)s group=%(group)s processes=%(processes)d \\\n'
' threads=%(threads)d python-path=%(site_root)s \\\n'
' display-name=%%{GROUP}\n'
'WSGIProcessGroup %(project_name)s\n'
'WSGIScriptAlias / %(project_root)s/wsgi.py\n'
'WSGIPassAuthorization On\n\n'
'<Directory %(project_root)s>\n'
' <Files wsgi.py>\n'
' Order deny,allow\n'
' Allow from all\n'
' </Files>\n'
'</Directory>\n\n'
'Alias /media/ %(media_root)s/\n'
'Alias /static/ %(static_root)s/\n'
'<Directory %(static_root)s>\n'
' ExpiresActive On\n'
' ExpiresByType image/gif A1209600\n'
' ExpiresByType image/jpeg A1209600\n'
' ExpiresByType image/png A1209600\n'
' ExpiresByType text/css A1209600\n'
' ExpiresByType text/javascript A1209600\n'
' ExpiresByType application/x-javascript A1209600\n'
' <FilesMatch "\.(css|js|gz|png|gif|jpe?g|flv|swf|ico|pdf|txt|html|htm)$">\n'
' ContentDigest On\n'
' FileETag MTime Size\n'
' </FilesMatch>\n'
'</Directory>\n\n'
'RedirectMatch ^/$ /admin\n\n'
'<VirtualHost [%(mgmt_addr)s]:443>\n'
' ServerName localhost\n'
' SSLEngine on\n'
' SSLCertificateFile %(cert_path)s\n'
' SSLCertificateKeyFile %(cert_key_path)s\n'
' SSLCACertificateFile %(cert_path)s\n'
' SSLVerifyClient None\n'
'</VirtualHost>' % context
)
context.update({
'apache_conf': apache_conf,
'apache_conf_file': '/etc/apache2/conf.d/%(project_name)s.conf' % context
})
# Apache 2.4 compatibility - feature #684
run('mkdir -p /etc/apache2/conf.d')
run('mkdir -p /etc/apache2/conf-available')
compat_conf = (
'# This enables configuration files in the legacy Apache directory.\n'
'IncludeOptional conf.d/*.conf\n'
'\n'
'# These correct access control to Controller files for Apache 2.4.\n'
'<Directory %(project_root)s>\n'
' <Files wsgi.py>\n'
' Require all granted\n'
' </Files>\n'
'</Directory>\n'
'<Location /media>\n'
' Require all granted\n'
'</Location>\n'
'<Location /static>\n'
' Require all granted\n'
'</Location>' % context
)
context.update({
'compat_conf': compat_conf,
'compat_conf_file': '/etc/apache2/conf-available/local-%(project_name)s-compat.conf' % context
})
run("echo '%(compat_conf)s' > %(compat_conf_file)s" % context)
# Store apache2 configuration keeping existing one (if any).
diff = run("echo '%(apache_conf)s'|diff - %(apache_conf_file)s" % context, err_codes=[0,1,2])
if diff.return_code == 2:
# File does not exist
run("echo '%(apache_conf)s' > %(apache_conf_file)s" % context)
elif diff.return_code == 1:
# File is different, save the old one
if interactive:
msg = ("\n\nFile %(apache_conf_file)s should be updated, do "
"you like to override it? (yes/no): " % context)
confirm = input(msg)
while 1:
if confirm not in ('yes', 'no'):
confirm = input('Please enter either "yes" or "no": ')
continue
if confirm == 'no':
return
break
run("cp %(apache_conf_file)s %(apache_conf_file)s.\$save" % context)
run("echo '%(apache_conf)s' > %(apache_conf_file)s" % context)
self.stdout.write("\033[1;31mA new version of %(apache_conf_file)s "
"has been installed.\n The old version has been placed at "
"%(apache_conf_file)s.$save\033[m" % context)
run('a2enmod wsgi')
run('a2enmod expires')
run('a2enmod deflate')
run('a2enmod ssl')
# catch 127 error 'command not found' for apache 2.2 installations
run('a2enconf local-%(project_name)s-compat' % context, err_codes=[0, 127])
# Give read permissions to cert key file
run('chmod g+r %(cert_key_path)s' % context)
def handle(self, *args, **options):
# Warn about deprecated options
if options.get('local'):
self.stderr.write(
"Warning: 'local' option is deprecated and will be ignored.\n")
version = options.get('version')
upgrade_notes = []
if version:
try:
major, major2, minor = decode_version(version)
except ValueError as e:
raise CommandError(e)
# Represent version as two digits per number: 1.2.2 -> 10202
version = int(
str(major) + "%02d" % int(major2) + "%02d" % int(minor))
# Pre-upgrade operations (version specific)
if version < 835:
# prevent schema migrations from failing
if is_installed('firmware'):
from firmware.models import Build
Build.objects.filter(base_image=None).update(base_image='')
if version <= 902:
if is_installed('maintenance'):
# Apply losed migrations
from south.models import MigrationHistory
migrated = MigrationHistory.objects.filter(
app_name='maintenance').exists()
if not migrated:
run('python manage.py migrate maintenance 0001 --fake')
if version < 1002:
# Update monitor settings (fix typo and add DiskFreeMonitor)
context = {
'settings':
run("find . -type f -name 'settings.py'|grep -v 'vct/'")
}
# Try automaticate update (making a backup)
if context['settings']:
run("cp %(settings)s %(settings)s.upgrade.bak" % context)
# fix NumProcessesMonitor typo
run("sed -i 's/NumPocessesMonitor/NumProcessesMonitor/g' "
"%(settings)s" % context)
# append disk monitor (if needed)
# this is a rude check (but runned in a conservative way)
if 'DiskFreeMonitor' not in open(
context['settings']).read():
run("sed -i '/MONITOR_MONITORS = ($/ a\ "
" (\"monitor.monitors.DiskFreeMonitor\",),' "
"%(settings)s" % context)
# warn the user about settings changes
autoupdate_status = 'OK' if context['settings'] else 'FAIL'
upgrade_notes.append(
'The monitor application has changed and .'
'some settings updates are required:\n'
' - Fix typo on NumProcessesMonitor (missing "r")\n'
' - Enable disk monitor\n'
' Please read the monitor app doc (MONITOR_MONITORS setting)\n'
'AUTOUPDATE: %s' % autoupdate_status)
if version <= 1102:
# Handle InconsistentMigrationHistory on tinc app
# * v0.11.2 tinc includes 0022, 0028..0030
# * v0.11.3 tinc adds 0023..0027
# We can force south to merge migrations because
# are implemented to be runned without dependencies
run('python manage.py migrate tinc 0030 --merge --noinput')
if not options.get('specifics_only'):
# Common stuff
development = options.get('development')
controller_admin = os.path.join(os.path.dirname(__file__),
'../../bin/')
controller_admin = os.path.join(controller_admin,
'controller-admin.sh')
run('chmod +x %s' % controller_admin)
extra = '--development' if development else ''
if options.get('proxy'):
extra += ' --proxy %s' % options.get('proxy')
run("%s install_requirements " % controller_admin + extra)
run("python manage.py collectstatic --noinput")
run("python manage.py syncdb --noinput")
run("python manage.py migrate --noinput")
if is_installed('firmware'):
run("python manage.py syncfirmwareplugins")
if is_installed('notifications'):
run("python manage.py syncnotifications")
if is_installed('resources'):
run("python manage.py syncresources")
if options.get('restart'):
run("python manage.py restartservices")
if not version:
self.stderr.write(
'\nNext time you migth want to provide a --from argument '
'in order to run version specific upgrade operations\n')
return
# Post-upgrade operations (version specific)
if version <= 629:
# Clean existing sessions because of change on auth backend
run('echo "delete from django_session;" | python manage.py dbshell'
)
if version < 801:
deprecate_periodic_tasks(('state.ping', ))
if version < 809:
# Add PKI directories
from pki import ca
from controller.utils.paths import get_site_root
site_root = get_site_root()
username = run("stat -c %%U %s" % site_root)
get_dir = lambda f: os.path.dirname(getattr(ca, f + '_path'))
for d in set(get_dir(f) for f in ['priv_key', 'pub_key', 'cert']):
run('mkdir -p %s' % d)
run('chown %s %s' % (username, d))
upgrade_notes.append(
'HTTPS certificate support for the management '
'network has been introduced in version 0.8.9.\n'
'In order to use it you sould run:\n'
' > python manage.py setuppki\n'
' > sudo python manage.py setupapache\n')
if version < 838:
# Purge communitynetworks.periodic_cache_node_db
from djcelery.models import PeriodicTask
PeriodicTask.objects.filter(
name='communitynetworks.periodic_cache_node_db').delete()
run('rabbitmqctl stop_app')
run('rabbitmqctl reset')
run('rabbitmqctl start_app')
run('service celeryd restart')
upgrade_notes.append(
'New Celeryd init.d configuration has been '
'introduced in 0.8.38.\nIt is strongly recommended to upgrade by\n'
' > sudo python manage.py setupceleryd\n')
# Deprecate x86 and amd64 architectures
from nodes.models import Node
Node.objects.filter(arch='x86').update(arch='i686')
Node.objects.filter(arch='amd64').update(arch='x86_64')
upgrade_notes.append(
'In order to support Base authentication while downloading '
'firmwares you should add "WSGIPassAuthorization On" on your apache config.\n'
'Alternatively you can perform this operation with the following command\n'
' > sudo python manage.py setupapache\n'
' > /etc/init.d/apache2 reload\n')
if version < 900:
upgrade_notes.append(
'Apache configuration is now placed under '
'/etc/apache2/conf.d/<project_name>.conf. It is convenient for you '
'to migrate your current configuration located on /etc/apache2/httpd.conf '
'to this new location.\n')
upgrade_notes.append(
'Celery workers configuration has been updated. '
'Please update it by running:\n'
' > sudo python manage.py setupceleryd\n')
if version < 905:
# TODO find the root cause of this
# maybe is shit imported on settings that import settings like add_app
# Prevent crazy import erros to appear :S
from django.utils import translation
translation.activate('en-us')
# Change template types for more generic ones
from slices.models import Template
from slices.settings import SLICES_TEMPLATE_TYPES
template_types = [t[0] for t in SLICES_TEMPLATE_TYPES]
if 'debian' in template_types:
Template.objects.filter(type='debian6').update(type='debian')
if 'openwrt' in template_types:
Template.objects.filter(type='openwrt-backfire').update(
type='openwrt')
if version < 906:
deprecate_periodic_tasks(('state.nodestate', 'state.sliverstate'))
if version <= 907:
# Generate sha256
from slices.models import Template
for template in Template.objects.all():
template.save()
upgrade_notes.append(
"It is extremly recommended to update your database "
"settings to enable atomic request behaviour:\n"
" https://docs.djangoproject.com/en/dev/topics/db/transactions/#tying-transactions-to-http-requests\n"
"Just add:\n"
" 'ATOMIC_REQUESTS': True,\n"
"into DATABASES setting within <project_name>/<project_name>/settings.py"
)
if version <= 1003:
# Update firmware configuration after Island refactor (#264)
from firmware.models import ConfigFile
try:
cfg_file = ConfigFile.objects.get(
path__contains="node.tinc.connect_to")
except (ConfigFile.DoesNotExist,
ConfigFile.MultipleObjectsReturned):
# Warn the user that needs to perform manual update
msg = "Firmware configuration update has failed. "
else:
cfg_file.content = cfg_file.content.replace(
"node.tinc.island", "node.island")
cfg_file.save()
msg = "Firmware configuration updated successfully. Updated ConfigFile ID: %i." % cfg_file.pk
upgrade_notes.append(
"%s\nPlease check version 0.10.4 release notes:\n"
"https://wiki.confine-project.eu/soft:server-release-notes#section0104"
% msg)
if version < 1103:
# Update mgmt network Server APIs certificate
# perform raw SQL querie because models hasn't been
# reloaded yet and cannot access e.g. server.api
from django.db import connection
from nodes.models import Server
from pki import ca
server_id = Server.objects.order_by('id').first().pk
try:
cert = ca.get_cert().as_pem()
except IOError:
msg = ("Failed to update Server APIs certificate. Missing "
"server certificate '%s'.\n"
"Run 'python manage.py setuppki --help'" % ca.cert_path)
upgrade_notes.append(msg)
else:
update_sql = ('UPDATE "nodes_serverapi" SET cert = %s '
'WHERE "nodes_serverapi"."server_id" = %s')
cursor = connection.cursor()
cursor.execute(update_sql, [cert, server_id])
del cursor
upgrade_notes.append("Updated Server APIs certificate.")
if upgrade_notes and options.get('print_upgrade_notes'):
self.stdout.write('\n\033[1m\n'
' ===================\n'
' ** UPGRADE NOTES **\n'
' ===================\n\n' +
'\n'.join(upgrade_notes) + '\033[m\n')
from django.conf import settings
from controller.utils.paths import get_site_root
MAINTENANCE_KEY_PATH = getattr(settings, 'MAINTENANCE_KEY_PATH',
'%(site_root)s/pki/maintenance_rsa' % { 'site_root': get_site_root() })
MAINTENANCE_PUB_KEY_PATH = getattr(settings, 'MAINTENANCE_PUB_KEY_PATH',
'%(site_root)s/pki/maintenance_rsa.pub' % { 'site_root': get_site_root() })
def cert_path(self):
return settings.PKI_CA_CERT_PATH % {'site_root': get_site_root()}
def pub_key_path(self):
return settings.PKI_CA_PUB_KEY_PATH % {'site_root': get_site_root()}
def priv_key_path(self):
return settings.PKI_CA_PRIV_KEY_PATH % { 'site_root': get_site_root() }
def handle(self, *args, **options):
interactive = options.get('interactive')
# Avoid import errors
from nodes.models import Server
server = Server.objects.first()
context = {
'project_name': get_project_name(),
'project_root': get_project_root(),
'site_root': get_site_root(),
'media_root': settings.MEDIA_ROOT,
'static_root': settings.STATIC_ROOT,
'cert_path': ca.cert_path,
'cert_key_path': ca.priv_key_path,
'mgmt_addr': str(server.mgmt_net.addr),
'user': options.get('user'),
'group': options.get('group') or options.get('user'),
'home': expanduser("~%s" % options.get('user')),
'processes': int(options.get('processes')),}
nginx_conf = (
'server {\n'
' listen 80;\n'
' listen [::]:80 ipv6only=on;\n'
' rewrite ^/$ /admin;\n'
' client_max_body_size 500m;\n'
' location / {\n'
' uwsgi_pass unix:///var/run/uwsgi/app/%(project_name)s/socket;\n'
' include uwsgi_params;\n'
' }\n'
' location /media {\n'
' alias %(media_root)s;\n'
' expires 30d;\n'
' }\n'
' location /static {\n'
' alias %(static_root)s;\n'
' expires 30d;\n'
' }\n'
'}\n'
'\n'
'server {\n'
' listen [%(mgmt_addr)s]:443 default_server ssl;\n'
' ssl_certificate %(cert_path)s;\n'
' ssl_certificate_key %(cert_key_path)s;\n'
' rewrite ^/$ /admin;\n'
' client_max_body_size 50m;\n'
' location / {\n'
' uwsgi_pass unix:///var/run/uwsgi/app/%(project_name)s/socket;\n'
' include uwsgi_params;\n'
' }\n'
' location /media {\n'
' alias %(media_root)s;\n'
' expires 30d;\n'
' }\n'
' location /static {\n'
' alias %(static_root)s;\n'
' expires 30d;\n'
' }\n'
'}\n') % context
uwsgi_conf = (
'[uwsgi]\n'
'plugins = python\n'
'chdir = %(site_root)s\n'
'module = %(project_name)s.wsgi\n'
'master = true\n'
'processes = %(processes)d\n'
'chmod-socket = 664\n'
'stats = /run/uwsgi/%%(deb-confnamespace)/%%(deb-confname)/statsocket\n'
'vacuum = true\n'
'uid = %(user)s\n'
'gid = %(group)s\n'
'env = HOME=%(home)s\n') % context
nginx = {
'file': '/etc/nginx/conf.d/%(project_name)s.conf' % context,
'conf': nginx_conf }
uwsgi = {
'file': '/etc/uwsgi/apps-available/%(project_name)s.ini' % context,
'conf': uwsgi_conf }
for extra_context in (nginx, uwsgi):
context.update(extra_context)
diff = run("echo '%(conf)s'|diff - %(file)s" % context, err_codes=[0,1,2])
if diff.return_code == 2:
# File does not exist
run("echo '%(conf)s' > %(file)s" % context)
elif diff.return_code == 1:
# File is different, save the old one
if interactive:
msg = ("\n\nFile %(file)s be updated, do you like to override "
"it? (yes/no): " % context)
confirm = input(msg)
while 1:
if confirm not in ('yes', 'no'):
confirm = input('Please enter either "yes" or "no": ')
continue
if confirm == 'no':
return
break
run("cp %(file)s %(file)s.save" % context)
run("echo '%(conf)s' > %(file)s" % context)
self.stdout.write("\033[1;31mA new version of %(file)s has been installed.\n "
"The old version has been placed at %(file)s.save\033[m" % context)
run('ln -s /etc/uwsgi/apps-available/%(project_name)s.ini /etc/uwsgi/apps-enabled/' % context, err_codes=[0,1])
# Give read permissions to cert key file
run('chmod g+r %(cert_key_path)s' % context)
# nginx should start after tincd
current = "\$local_fs \$remote_fs \$network \$syslog"
run('sed -i "s/ %s$/ %s \$named/g" /etc/init.d/nginx' % (current, current))
rotate = (
'/var/log/nginx/*.log {\n'
' daily\n'
' missingok\n'
' rotate 30\n'
' compress\n'
' delaycompress\n'
' notifempty\n'
' create 640 root adm\n'
' sharedscripts\n'
' postrotate\n'
' [ ! -f /var/run/nginx.pid ] || kill -USR1 `cat /var/run/nginx.pid`\n'
' endscript\n'
'}\n')
run("echo '%s' > /etc/logrotate.d/nginx" % rotate)
# Allow nginx to write to uwsgi socket
run('adduser www-data %(group)s' % context)
def pub_key_path(self):
return settings.PKI_CA_PUB_KEY_PATH % { 'site_root': get_site_root() }
def handle(self, *args, **options):
if options.get('username'):
warnings.warn('"username" has been deprecated in favor of "user"', DeprecationWarning)
if options.get('user') and options.get('username'):
raise CommandError("Only one of this options should be provided: --user OR --username")
user = options.get('user') or options.get('username') or 'confine'
context = {'site_root': get_site_root(),
'user': user,
'group': options.get('group') or user,
'bin_path': path.join(get_controller_root(), 'bin'),
'processes': options.get('processes'),
'greenlets': options.get('greenlets') }
celery_config = (
'# Name of nodes to start, here we have a single node\n'
'CELERYD_NODES="w1 w2"\n'
'\n'
'# Where to chdir at start.\n'
'CELERYD_CHDIR="%(site_root)s"\n'
'\n'
'# How to call "manage.py celeryd_multi"\n'
'CELERYD_MULTI="$CELERYD_CHDIR/manage.py celeryd_multi"\n'
'\n'
'# Extra arguments to celeryd\n'
'CELERYD_OPTS="-P:w1 processes -c:w1 %(processes)s -Q:w1 celery \\\n'
' -P:w2 gevent -c:w2 %(greenlets)s -Q:w2 gevent --time-limit=300"\n'
'\n'
'# Name of the celery config module.\n'
'CELERY_CONFIG_MODULE="celeryconfig"\n'
'\n'
'# %%n will be replaced with the nodename.\n'
'CELERYD_LOG_FILE="/var/log/celery/%%n.log"\n'
'CELERYD_PID_FILE="/var/run/celery/%%n.pid"\n'
'CELERY_CREATE_DIRS=1\n'
'\n'
'# Full path to the celeryd logfile.\n'
'CELERYEV_LOG_FILE="/var/log/celery/celeryev.log"\n'
'CELERYEV_PID_FILE="/var/run/celery/celeryev.pid"\n'
'\n'
'# Workers should run as an unprivileged user.\n'
'CELERYD_USER="******"\n'
'CELERYD_GROUP="%(group)s"\n'
'\n'
'# Persistent revokes\n'
'CELERYD_STATE_DB="$CELERYD_CHDIR/persistent_revokes"\n'
'\n'
'# Celeryev\n'
'CELERYEV="$CELERYD_CHDIR/manage.py"\n'
'CELERYEV_CAM="djcelery.snapshot.Camera"\n'
'CELERYEV_USER="******"\n'
'CELERYEV_GROUP="$CELERYD_GROUP"\n'
'CELERYEV_OPTS="celeryev"\n'
'\n'
'# Celerybeat\n'
'CELERYBEAT="${CELERYD_CHDIR}/manage.py celerybeat"\n'
'CELERYBEAT_USER="******"\n'
'CELERYBEAT_GROUP="$CELERYD_GROUP"\n'
'CELERYBEAT_CHDIR="$CELERYD_CHDIR"\n'
'CELERYBEAT_OPTS="--schedule=/var/run/celerybeat-schedule"\n' % context)
run("echo '%s' > /etc/default/celeryd" % celery_config)
# https://raw.github.com/celery/celery/master/extra/generic-init.d/
for script in ['celeryevcam', 'celeryd', 'celerybeat']:
context['script'] = script
run('cp %(bin_path)s/%(script)s /etc/init.d/%(script)s' % context)
run('chmod +x /etc/init.d/%(script)s' % context)
run('update-rc.d %(script)s defaults' % context)
rotate = ('/var/log/celery/*.log {\n'
' weekly\n'
' missingok\n'
' rotate 10\n'
' compress\n'
' delaycompress\n'
' notifempty\n'
' copytruncate\n'
'}')
run("echo '%s' > /etc/logrotate.d/celeryd" % rotate)
def handle(self, *args, **options):
interactive = options.get('interactive')
# Avoid import errors
from nodes.models import Server
server = Server.objects.first()
context = {
'project_name': get_project_name(),
'project_root': get_project_root(),
'site_root': get_site_root(),
# TODO end with single /
'media_root': settings.MEDIA_ROOT,
'static_root': settings.STATIC_ROOT,
'cert_path': ca.cert_path,
'cert_key_path': ca.priv_key_path,
'mgmt_addr': str(server.mgmt_net.addr),
'user': options.get('user'),
'group': options.get('group') or options.get('user'),
'processes': int(options.get('processes')),
'threads': int(options.get('threads')) }
apache_conf = (
'WSGIDaemonProcess %(project_name)s user=%(user)s group=%(group)s processes=%(processes)d \\\n'
' threads=%(threads)d python-path=%(site_root)s \\\n'
' display-name=%%{GROUP}\n'
'WSGIProcessGroup %(project_name)s\n'
'WSGIScriptAlias / %(project_root)s/wsgi.py\n'
'WSGIPassAuthorization On\n\n'
'<Directory %(project_root)s>\n'
' Require all granted\n'
' <Files wsgi.py>\n'
' Order deny,allow\n'
' Allow from all\n'
' </Files>\n'
'</Directory>\n\n'
'Alias /media/ %(media_root)s/\n'
'Alias /static/ %(static_root)s/\n'
'<Directory %(static_root)s>\n'
' Require all granted\n'
' ExpiresActive On\n'
' ExpiresByType image/gif A1209600\n'
' ExpiresByType image/jpeg A1209600\n'
' ExpiresByType image/png A1209600\n'
' ExpiresByType text/css A1209600\n'
' ExpiresByType text/javascript A1209600\n'
' ExpiresByType application/x-javascript A1209600\n'
' <FilesMatch "\.(css|js|gz|png|gif|jpe?g|flv|swf|ico|pdf|txt|html|htm)$">\n'
' ContentDigest On\n'
' FileETag MTime Size\n'
' </FilesMatch>\n'
'</Directory>\n\n'
'RedirectMatch ^/$ /admin\n\n'
'<VirtualHost [%(mgmt_addr)s]:443>\n'
' ServerName localhost\n'
' SSLEngine on\n'
' SSLCertificateFile %(cert_path)s\n'
' SSLCertificateKeyFile %(cert_key_path)s\n'
' SSLCACertificateFile %(cert_path)s\n'
' SSLVerifyClient None\n'
'</VirtualHost>' % context)
context.update({
'apache_conf': apache_conf,
'apache_conf_file': '/etc/apache2/sites-enabled/%(project_name)s.conf' % context})
diff = run("echo '%(apache_conf)s'|diff - %(apache_conf_file)s" % context, err_codes=[0,1,2])
if diff.return_code == 2:
# File does not exist
run("echo '%(apache_conf)s' > %(apache_conf_file)s" % context)
elif diff.return_code == 1:
# File is different, save the old one
if interactive:
msg = ("\n\nFile %(apache_conf_file)s should be updated, do "
"you like to override it? (yes/no): " % context)
confirm = input(msg)
while 1:
if confirm not in ('yes', 'no'):
confirm = input('Please enter either "yes" or "no": ')
continue
if confirm == 'no':
return
break
run("cp %(apache_conf_file)s %(apache_conf_file)s.\$save" % context)
run("echo '%(apache_conf)s' > %(apache_conf_file)s" % context)
self.stdout.write("\033[1;31mA new version of %(apache_conf_file)s "
"has been installed.\n The old version has been placed at "
"%(apache_conf_file)s.$save\033[m" % context)
# include_httpd = run("grep '^\s*Include\s\s*httpd.conf\s*' /etc/apache2/apache2.conf", err_codes=[0,1])
# if include_httpd.return_code == 1:
# run("echo 'Include httpd.conf' >> /etc/apache2/apache2.conf")
# run('a2ensite %s' % project_name)
run('a2enmod wsgi')
run('a2enmod expires')
run('a2enmod deflate')
run('a2enmod ssl')
# Give upload file permissions to apache
# username = run("stat -c %%U %(project_root)s" % context)
# run('adduser www-data %s' % username)
# run('chmod g+w %(media_root)s' % context)
# Give read permissions to cert key file
run('chmod g+r %(cert_key_path)s' % context)