def pem2der_string(pem_string):
"""
Converts private key or certificate from ascii-armored
PEM format to binary DER format.
Based on openssl as described in OpenWrt Wiki:
http://wiki.openwrt.org/doc/howto/certificates.overview
"""
# FIXME: temporal files can be avoid using load_cert_string?
# write PEM content to a temporal file
pem_file = tempfile.NamedTemporaryFile()
pem_file.write(pem_string)
pem_file.seek(0)
# use another temporal file to write binary DER
der_file = tempfile.NamedTemporaryFile()
if 'CERTIFICATE' in pem_string:
run('openssl x509 -in %s -outform DER -out %s' %
(pem_file.name, der_file.name))
elif 'RSA PRIVATE KEY' in pem_string:
run('openssl rsa -in %s -outform DER -out %s' %
(pem_file.name, der_file.name))
else:
raise ValueError('Invalid literal for pem2der_string provided.')
return der_file.read()
def pre_umount(self, image, build, *args, **kwargs):
""" Configuring image password """
password = kwargs.get('password')
# days since Jan 1, 1970 from now
epoch = datetime.utcfromtimestamp(0)
today = datetime.today()
d = today - epoch
context = {
'image': image.mnt
}
# remove old uci-defaults script that generates
run('rm -f %(image)s/etc/uci-defaults/confine-passwd.sh' % context)
# generate root line for shadow file
if password:
line_context = {
'pwd': password,
'user': '******',
'last_changed': d.days,
'min_days': 0,
'max_days': 99999,
'warn_days': 7
}
line = '%(user)s:%(pwd)s:%(last_changed)i:%(min_days)i:'\
'%(max_days)i:%(warn_days)i:::' % line_context
context.update({
'line': line.replace('/', '\/'),
'shadow': os.path.join(image.mnt, 'etc/shadow'),
})
run("sed -i 's/^root:.*/%(line)s/' %(shadow)s" % context)
def pre_umount(self, image, build, *args, **kwargs):
""" Configuring image password """
password = kwargs.get('password')
# days since Jan 1, 1970 from now
epoch = datetime.utcfromtimestamp(0)
today = datetime.today()
d = today - epoch
context = {'image': image.mnt}
# remove old uci-defaults script that generates
run('rm -f %(image)s/etc/uci-defaults/confine-passwd.sh' % context)
# generate root line for shadow file
if password:
line_context = {
'pwd': password,
'user': '******',
'last_changed': d.days,
'min_days': 0,
'max_days': 99999,
'warn_days': 7
}
line = '%(user)s:%(pwd)s:%(last_changed)i:%(min_days)i:'\
'%(max_days)i:%(warn_days)i:::' % line_context
context.update({
'line': line.replace('/', '\/'),
'shadow': os.path.join(image.mnt, 'etc/shadow'),
})
run("sed -i 's/^root:.*/%(line)s/' %(shadow)s" % context)
def handle(self, *args, **options):
context = {
'key': ''.join([choice('abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)') for i in range(50)]),
'settings': os.path.join(get_project_root(), 'settings.py')}
if run("grep 'SECRET_KEY' %(settings)s" % context, err_codes=[0,1]).return_code == 0:
run("sed -i \"s/SECRET_KEY = '\w*'/SECRET_KEY = '%(key)s'/\" %(settings)s" % context)
else:
run("echo \"SECRET_KEY = '%(key)s'\" >> %(settings)s" % context)
def execute(self):
ps = run('ps -A -o pid,cmd')
value = {}
for name, regex, __, __ in self.processes:
processes = re.findall('\n\s*([0-9]+) '+regex, ps.stdout)
value[name] = 0
for pid in processes:
pid = int(pid)
mem = run(self.cmd.format(pid)).stdout
value[name] += int(mem) if mem else 0
return value, []
def execute(self):
ps = run('ps -A -o pid,cmd')
value = {}
for name, regex, __, __ in self.processes:
processes = re.findall('\n\s*([0-9]+) ' + regex, ps.stdout)
value[name] = 0
for pid in processes:
pid = int(pid)
mem = run(self.cmd.format(pid)).stdout
value[name] += int(mem) if mem else 0
return value, []
def deprecate_periodic_tasks(names):
from djcelery.models import PeriodicTask
for name in names:
PeriodicTask.objects.filter(name=name).delete()
run('rabbitmqctl stop_app')
run('rabbitmqctl reset')
run('rabbitmqctl start_app')
run('service celeryd restart')
def deprecate_periodic_tasks(names):
from djcelery.models import PeriodicTask
for name in names:
PeriodicTask.objects.filter(name=name).delete()
run('rabbitmqctl stop_app')
run('rabbitmqctl reset')
run('rabbitmqctl start_app')
run('service celeryd restart')
def execute(self):
ps = run('ps -A -o pid,ppid,cmd')
ticks = {}
for name, regex, __, __ in self.processes:
processes = re.findall('\n\s*([0-9]+)\s*([0-9]+) '+regex, ps.stdout)
pstree = PsTree()
for pid, ppid in processes:
pid = int(pid)
ppid = int(ppid)
stat = run(self.cmd.format(pid)).stdout.split(' ')
correct = (len(stat) == 2)
worked = int(stat[0]) if correct else 0
waited = int(stat[1]) if correct else 0
pstree.insert(pid, ppid, worked, waited)
ticks[name] = pstree.total_ticks()
return ticks, []
def pre_umount(self, image, build, *args, **kwargs):
""" Creating ssh authorized keys file """
admins_keys = kwargs.get('admins_keys', '')
additional_keys = kwargs.get('additional_keys', '')
auth_keys = ("# Group and node admins' keys:\n%s\n"
"# Additional keys:\n%s\n"
"# Other keys:\n" % (admins_keys, additional_keys))
context = {
'auth_keys': auth_keys,
'auth_keys_path': os.path.join(image.mnt, 'etc/dropbear/authorized_keys')
}
run('echo "%(auth_keys)s" > %(auth_keys_path)s' % context)
os.chown(context['auth_keys_path'], 0, 0)
key_stat = os.stat(context['auth_keys_path'])
assert 0 == key_stat.st_uid == key_stat.st_gid, "Failing when changing ownership!"
run('chmod 0600 %(auth_keys_path)s' % context)
def forwards(self, orm):
"""Create default NodeApi and ServerApi for existing objects."""
# Note: Don't use "from appname.models import ModelName".
# Use orm.ModelName to refer to models in this application,
# and orm['appname.ModelName'] for models in other applications.
# Create default NodeApi
for node in orm.Node.objects.all():
mgmt_addr = node_mgmt_address(node)
url = NODES_NODE_API_BASE_URI_DEFAULT % {'mgmt_addr': mgmt_addr}
orm.NodeApi.objects.create(node=node, base_uri=url, cert=node.cert)
# Create two ServerApi for server (one for REGISTRY and another for CONTROLLER)
if not orm.Server.objects.exists():
# Create the main server
description = run('hostname', display=False).stdout
server = orm.Server.objects.create(description=description)
for server in orm.Server.objects.all():
mgmt_addr = server_mgmt_address(server)
url = NODES_SERVER_API_BASE_URI_DEFAULT % {'mgmt_addr': mgmt_addr}
orm.ServerApi.objects.create(server=server,
base_uri=url,
type=ServerApi.REGISTRY)
orm.ServerApi.objects.create(server=server,
base_uri=url,
type=ServerApi.CONTROLLER)
def handle(self, *args, **options):
context = {
'username': options.get('username'),
'manage_file': os.path.join(get_site_root(), 'manage.py'),
}
exists = "crontab -u %(username)s -l | grep ' monitorlocalsystem' &> /dev/null"
exists = run(exists % context, display=False, err_codes=[0,1]).return_code
if exists != 0:
cmd = (
'{ crontab -l -u %(username)s;'
' echo "*/5 * * * * python %(manage_file)s monitorlocalsystem --email --quiet"; '
'} | crontab -u %(username)s -') % context
cmd = run(cmd)
print cmd.return_code
else:
self.stdout.write('Done nothing, crontab was already installed')
def post_umount(self, image, build, *args, **kwargs):
""" Creating confine-install USB image """
if kwargs.get('usb_image', False):
install = Image(usb_image)
try:
install.prepare()
install.gunzip()
install.mount()
path = os.path.join(install.mnt, 'confine/*img.gz')
dst = run('ls %s' % path).stdout
image.gzip()
run('mv %s %s' % (image.file, dst))
install.umount()
except:
install.clean()
image.file = install.file
image.tmp = install.tmp
def get_existing_pip_installation():
""" returns current pip installation path """
if run("pip freeze|grep confine-controller", err_codes=[0,1]).return_code == 0:
for lib_path in get_python_lib(), get_python_lib(prefix="/usr/local"):
existing_path = os.path.abspath(os.path.join(lib_path, "controller"))
if os.path.exists(existing_path):
return existing_path
return None
def post_umount(self, image, build, *args, **kwargs):
""" Creating confine-install USB image """
if kwargs.get('usb_image', False):
install = Image(usb_image)
try:
install.prepare()
install.gunzip()
install.mount()
path = os.path.join(install.mnt, 'confine/*img.gz')
dst = run('ls %s' % path).stdout
image.gzip()
run('mv %s %s' % (image.file, dst))
install.umount()
except:
install.clean()
image.file = install.file
image.tmp = install.tmp
def execute(self):
ps = run('ps -A -o pid,ppid,cmd')
ticks = {}
for name, regex, __, __ in self.processes:
processes = re.findall('\n\s*([0-9]+)\s*([0-9]+) ' + regex,
ps.stdout)
pstree = PsTree()
for pid, ppid in processes:
pid = int(pid)
ppid = int(ppid)
stat = run(self.cmd.format(pid)).stdout.split(' ')
correct = (len(stat) == 2)
worked = int(stat[0]) if correct else 0
waited = int(stat[1]) if correct else 0
pstree.insert(pid, ppid, worked, waited)
ticks[name] = pstree.total_ticks()
return ticks, []
def handle(self, *args, **options):
context = {
'username': options.get('username'),
'manage_file': os.path.join(get_site_root(), 'manage.py'),
}
exists = "crontab -u %(username)s -l | grep ' monitorlocalsystem' &> /dev/null"
exists = run(exists % context, display=False,
err_codes=[0, 1]).return_code
if exists != 0:
cmd = (
'{ crontab -l -u %(username)s;'
' echo "*/5 * * * * python %(manage_file)s monitorlocalsystem --email --quiet"; '
'} | crontab -u %(username)s -') % context
cmd = run(cmd)
print cmd.return_code
else:
self.stdout.write('Done nothing, crontab was already installed')
def get_existing_pip_installation():
""" returns current pip installation path """
if run("pip freeze|grep confine-controller", err_codes=[0,1]).return_code == 0:
for lib_path in get_python_lib(), get_python_lib(prefix="/usr/local"):
existing_path = os.path.abspath(os.path.join(lib_path, "controller"))
if os.path.exists(existing_path):
return existing_path
return None
def pre_umount(self, image, build, *args, **kwargs):
""" Creating ssh authorized keys file """
admins_keys = kwargs.get('admins_keys', '')
additional_keys = kwargs.get('additional_keys', '')
auth_keys = ("# Group and node admins' keys:\n%s\n"
"# Additional keys:\n%s\n"
"# Other keys:\n" % (admins_keys, additional_keys))
context = {
'auth_keys':
auth_keys,
'auth_keys_path':
os.path.join(image.mnt, 'etc/dropbear/authorized_keys')
}
run('echo "%(auth_keys)s" > %(auth_keys_path)s' % context)
os.chown(context['auth_keys_path'], 0, 0)
key_stat = os.stat(context['auth_keys_path'])
assert 0 == key_stat.st_uid == key_stat.st_gid, "Failing when changing ownership!"
run('chmod 0600 %(auth_keys_path)s' % context)
def execute(self):
problems = []
value = json.loads(run(self.cmd).stdout)
limit = MONITOR_DISK_FREE_LIMIT * MONITOR_DISK_WARN_RATIO
if value['free'] < limit:
msg = 'Low disk space left (%i), potential Rabbitiqm alarm that '\
'will block the producer! (more info on issue report #326)'
problems.append(msg % value['free'])
return value, problems
def execute(self):
problems = []
value = json.loads(run(self.cmd).stdout)
limit = MONITOR_DISK_FREE_LIMIT * MONITOR_DISK_WARN_RATIO
if value['free'] < limit:
msg = 'Low disk space left (%i), potential Rabbitiqm alarm that '\
'will block the producer! (more info on issue report #326)'
problems.append(msg % value['free'])
return value, problems
def run_tuple(services, action, options, optional=False):
if isinstance(services, str):
services = [services]
for service in services:
if options.get(service):
err_codes = [0, 1] if optional else [0]
e = run('service %s %s' % (service, action), err_codes=err_codes)
if e.return_code == 1:
return False
return True
def run_tuple(services, action, options, optional=False):
if isinstance(services, str):
services = [services]
for service in services:
if options.get(service):
err_codes = [0,1] if optional else [0]
e = run('service %s %s' % (service, action), err_codes=err_codes)
if e.return_code == 1:
return False
return True
def update_settings(monkey_patch=None, **options):
""" Warning this only works with a very simple setting format: NAME = value """
from django.conf import settings
for name, value in options.iteritems():
if getattr(settings, name, None) != value:
settings_file = os.path.join(get_project_root(), 'settings.py')
context = {
'name': name,
'value': value if not isinstance(value, basestring) else "'%s'" % value,
'settings': settings_file,
}
if run("grep '^%(name)s *=' %(settings)s" % context, err_codes=[0,1]):
# Update existing settings_file
run("sed -i \"s#%(name)s *=.*#%(name)s = %(value)s#\" %(settings)s" % context)
else:
run("echo \"%(name)s = %(value)s\" >> %(settings)s" % context)
# Monkeypatch controller settings to make available the change globally
if monkey_patch is not None:
module = import_module(monkey_patch)
setattr(module, name, value)
def update_settings(monkey_patch=None, **options):
""" Warning this only works with a very simple setting format: NAME = value """
from django.conf import settings
for name, value in options.iteritems():
if getattr(settings, name, None) != value:
settings_file = os.path.join(get_project_root(), 'settings.py')
context = {
'name': name,
'value': value if not isinstance(value, basestring) else "'%s'" % value,
'settings': settings_file,
}
if run("grep '^%(name)s *=' %(settings)s" % context, err_codes=[0,1]):
# Update existing settings_file
run("sed -i \"s#%(name)s *=.*#%(name)s = %(value)s#\" %(settings)s" % context)
else:
run("echo \"%(name)s = %(value)s\" >> %(settings)s" % context)
# Monkeypatch controller settings to make available the change globally
if monkey_patch is not None:
module = import_module(monkey_patch)
setattr(module, name, value)
def execute(self):
env = "export LINES=1000; export COLUMNS=1000; "
result = {}
problems = []
try:
result = json.loads(run(env + self.cmd).stdout)
except ValueError as error:
msg = "Error on monitor '%s': %s ('%s')" % (self.type, error, self.cmd)
problems.append(msg)
# TODO(santiago) configure logging and log this error
#import logging
#logger = logging.getLogger('ERROR')
#logger.error(msg)
return result, problems
def execute(self):
problems = []
value = {}
for name,regex,min_procs,max_procs in self.processes:
ps = run('ps -A -o pid,cmd | grep -E "%s" | grep -v "grep" | wc -l' % regex )
num = int(ps.stdout)
if min_procs and num < min_procs:
msg = 'Process %s has less than %i running instances (%i)'
problems.append(msg % (name, min_procs, num))
elif max_procs and num > max_procs:
msg = 'Process %s has more than %i running instances (%i)'
problems.append(msg % (name, max_procs, num))
value[name] = num
return value, problems
def execute(self):
env = "export LINES=1000; export COLUMNS=1000; "
result = {}
problems = []
try:
result = json.loads(run(env + self.cmd).stdout)
except ValueError as error:
msg = "Error on monitor '%s': %s ('%s')" % (self.type, error,
self.cmd)
problems.append(msg)
# TODO(santiago) configure logging and log this error
#import logging
#logger = logging.getLogger('ERROR')
#logger.error(msg)
return result, problems
def pem2der_string(pem_string):
"""
Converts private key or certificate from ascii-armored
PEM format to binary DER format.
Based on openssl as described in OpenWrt Wiki:
http://wiki.openwrt.org/doc/howto/certificates.overview
"""
# FIXME: temporal files can be avoid using load_cert_string?
# write PEM content to a temporal file
pem_file = tempfile.NamedTemporaryFile()
pem_file.write(pem_string)
pem_file.seek(0)
# use another temporal file to write binary DER
der_file = tempfile.NamedTemporaryFile()
if "CERTIFICATE" in pem_string:
run("openssl x509 -in %s -outform DER -out %s" % (pem_file.name, der_file.name))
elif "RSA PRIVATE KEY" in pem_string:
run("openssl rsa -in %s -outform DER -out %s" % (pem_file.name, der_file.name))
else:
raise ValueError("Invalid literal for pem2der_string provided.")
return der_file.read()
def handle(self, *args, **options):
# Configure firmware generation
username = options.get('username')
if username is None:
raise CommandError('Can not find default celeryd username')
if run('grep "^fuse:" /etc/group', err_codes=[0,1]).return_code == 1:
run('addgroup fuse')
if run('groups %s|grep fuse' % username, err_codes=[0,1]).return_code == 1:
run('adduser %s fuse' % username)
def get_vct_config(var):
""" Get options from vct config file """
vct_root = get_vct_root()
context = {
'var': var,
'source': """
if [ -f %(vct_root)s/vct.conf.overrides ]; then
. %(vct_root)s/vct.conf.default
. %(vct_root)s/vct.conf.overrides
elif [ -f %(vct_root)s/vct.conf ]; then
. %(vct_root)s/vct.conf
elif [ -f %(vct_root)s/vct.conf.default ]; then
. %(vct_root)s/vct.conf.default
fi """ % { 'vct_root': vct_root} }
out = run("bash -c '%(source)s; echo $%(var)s'" % context, display=False, silent=False)
return out.stdout
def execute(self):
problems = []
value = {}
for name, regex, min_procs, max_procs in self.processes:
ps = run(
'ps -A -o pid,cmd | grep -E "%s" | grep -v "grep" | wc -l' %
regex)
num = int(ps.stdout)
if min_procs and num < min_procs:
msg = 'Process %s has less than %i running instances (%i)'
problems.append(msg % (name, min_procs, num))
elif max_procs and num > max_procs:
msg = 'Process %s has more than %i running instances (%i)'
problems.append(msg % (name, max_procs, num))
value[name] = num
return value, problems
def handle(self, *args, **options):
# Configure firmware generation
username = options.get('username')
if username is None:
raise CommandError('Can not find default celeryd username')
if run('grep "^fuse:" /etc/group', err_codes=[0, 1]).return_code == 1:
run('addgroup fuse')
if run('groups %s|grep fuse' % username,
err_codes=[0, 1]).return_code == 1:
run('adduser %s fuse' % username)
def get_vct_config(var):
""" Get options from vct config file """
vct_root = get_vct_root()
context = {
'var': var,
'source': """
if [ -f %(vct_root)s/vct.conf.overrides ]; then
. %(vct_root)s/vct.conf.default
. %(vct_root)s/vct.conf.overrides
elif [ -f %(vct_root)s/vct.conf ]; then
. %(vct_root)s/vct.conf
elif [ -f %(vct_root)s/vct.conf.default ]; then
. %(vct_root)s/vct.conf.default
fi """ % {
'vct_root': vct_root
}
}
out = run("bash -c '%(source)s; echo $%(var)s'" % context,
display=False,
silent=False)
return out.stdout
def update_tincd():
"""
Generates all local tinc/hosts/* and reloads tincd
"""
from .models import TincHost
hosts_path = '%s/hosts/' % os.path.join(TINC_TINCD_ROOT, TINC_NET_NAME)
# Dirty mechanism to know if there has been any change while running
dirtyfile = os.path.join(hosts_path, '.dirty')
now = time.time()
touch(dirtyfile, times=(now, now))
# File-based lock mechanism to prevent concurrency problems
with LockFile(hosts_path+'.lock', expire=60):
hosts = TincHost.objects.hosts() # clients + gateways
# Batch processing of tinc clients for efficiency/max_arg_length tradeoff
scripts = []
total = len(hosts)
for start in range(0, total, 100):
end = min(start + 100, total)
script = ''
for host in hosts[start:end]:
host_file = os.path.join(hosts_path, host.name)
script += 'echo -e "%s" > %s;\n' % (host.get_host(), host_file)
scripts.append(script)
# delete all tinc hosts
run('rm -f -- %s{host_,node_}*' % hosts_path)
# create all tinc hosts
for script in scripts:
run(script)
# retry if there is any pending modification
if os.path.getmtime(dirtyfile) > now:
raise update_tincd.retry(countdown=1)
# In some systems it is mandatory to send a HUP signal in order to reload new subnets
if TINC_TINCD_SEND_HUP:
context = {
'tincd_bin': TINC_TINCD_BIN,
'net_name': TINC_NET_NAME
}
tinc_hup = "sudo %(tincd_bin)s -kHUP -n %(net_name)s" % context
run(tinc_hup)
def forwards(self, orm):
"""Create default NodeApi and ServerApi for existing objects."""
# Note: Don't use "from appname.models import ModelName".
# Use orm.ModelName to refer to models in this application,
# and orm['appname.ModelName'] for models in other applications.
# Create default NodeApi
for node in orm.Node.objects.all():
mgmt_addr = node_mgmt_address(node)
url = NODES_NODE_API_BASE_URI_DEFAULT % {'mgmt_addr': mgmt_addr}
orm.NodeApi.objects.create(node=node, base_uri=url, cert=node.cert)
# Create two ServerApi for server (one for REGISTRY and another for CONTROLLER)
if not orm.Server.objects.exists():
# Create the main server
description = run('hostname', display=False).stdout
server = orm.Server.objects.create(description=description)
for server in orm.Server.objects.all():
mgmt_addr = server_mgmt_address(server)
url = NODES_SERVER_API_BASE_URI_DEFAULT % {'mgmt_addr': mgmt_addr}
orm.ServerApi.objects.create(server=server, base_uri=url, type=ServerApi.REGISTRY)
orm.ServerApi.objects.create(server=server, base_uri=url, type=ServerApi.CONTROLLER)
def update_tincd():
"""
Generates all local tinc/hosts/* and reloads tincd
"""
from .models import TincHost
hosts_path = '%s/hosts/' % os.path.join(TINC_TINCD_ROOT, TINC_NET_NAME)
# Dirty mechanism to know if there has been any change while running
dirtyfile = os.path.join(hosts_path, '.dirty')
now = time.time()
touch(dirtyfile, times=(now, now))
# File-based lock mechanism to prevent concurrency problems
with LockFile(hosts_path + '.lock', expire=60):
hosts = TincHost.objects.hosts() # clients + gateways
# Batch processing of tinc clients for efficiency/max_arg_length tradeoff
scripts = []
total = len(hosts)
for start in range(0, total, 100):
end = min(start + 100, total)
script = ''
for host in hosts[start:end]:
host_file = os.path.join(hosts_path, host.name)
script += 'echo -e "%s" > %s;\n' % (host.get_host(), host_file)
scripts.append(script)
# delete all tinc hosts
run('rm -f -- %s{host_,node_}*' % hosts_path)
# create all tinc hosts
for script in scripts:
run(script)
# retry if there is any pending modification
if os.path.getmtime(dirtyfile) > now:
raise update_tincd.retry(countdown=1)
# In some systems it is mandatory to send a HUP signal in order to reload new subnets
if TINC_TINCD_SEND_HUP:
context = {'tincd_bin': TINC_TINCD_BIN, 'net_name': TINC_NET_NAME}
tinc_hup = "sudo %(tincd_bin)s -kHUP -n %(net_name)s" % context
run(tinc_hup)
def vct_node(action, node, silent=True):
node_id = hex(node.id).split('0x')[1]
node_id = '0' * (4 - len(node_id)) + node_id
wrapper = path.join(get_vct_root(), '%s')
cmd = 'vct_node_%s %s' % (action, node_id)
return run(wrapper % cmd, display=False, silent=silent)
def handle(self, *args, **options):
# Configure firmware generation
context = {
'db_name': options.get('db_name'),
'db_user': options.get('db_user'),
'db_password': options.get('db_password'),
'db_host': options.get('db_host'),
'db_port': options.get('db_port')
}
run('su postgres -c "psql -c \\"CREATE USER %(db_user)s PASSWORD \'%(db_password)s\';\\""'
% context,
err_codes=[0, 1])
run('su postgres -c "psql -c \\"CREATE DATABASE %(db_name)s OWNER %(db_user)s;\\""'
% context,
err_codes=[0, 1])
context.update(
{'settings': os.path.join(get_project_root(), 'settings.py')})
if run("grep 'DATABASES' %(settings)s" % context,
err_codes=[0, 1]).return_code == 0:
# Update existing settings_file
run("sed -i \"s/'ENGINE': '\w*',/'ENGINE': 'django.db.backends.postgresql_psycopg2',/\" %(settings)s"
% context)
run("sed -i \"s/'NAME': '.*',/'NAME': '%(db_name)s',/\" %(settings)s"
% context)
run("sed -i \"s/'USER': '******',/'USER': '******',/\" %(settings)s"
% context)
run("sed -i \"s/'PASSWORD': '******',/'PASSWORD': '******',/\" %(settings)s"
% context)
run("sed -i \"s/'HOST': '.*',/'HOST': '%(db_host)s',/\" %(settings)s"
% context)
run("sed -i \"s/'PORT': '.*',/'PORT': '%(db_port)s',/\" %(settings)s"
% context)
else:
db_config = (
"DATABASES = {\n"
" 'default': {\n"
" 'ENGINE': 'django.db.backends.postgresql_psycopg2',\n"
" 'NAME': '%(db_name)s',\n"
" 'USER': '******',\n"
" 'PASSWORD': '******',\n"
" 'HOST': '%(db_host)s',\n"
" 'PORT': '%(db_port)s',\n"
" 'ATOMIC_REQUESTS': True,\n"
" }\n"
"}\n" % context)
context.update({'db_config': db_config})
run('echo "%(db_config)s" >> %(settings)s' % context)
def handle(self, *args, **options):
interactive = options.get('interactive')
# Avoid import errors
from nodes.models import Server
server = Server.objects.first()
context = {
'project_name': get_project_name(),
'project_root': get_project_root(),
'site_root': get_site_root(),
'media_root': settings.MEDIA_ROOT,
'static_root': settings.STATIC_ROOT,
'cert_path': ca.cert_path,
'cert_key_path': ca.priv_key_path,
'mgmt_addr': str(server.mgmt_net.addr),
'user': options.get('user'),
'group': options.get('group') or options.get('user'),
'processes': int(options.get('processes')),
'threads': int(options.get('threads'))
}
apache_conf = (
'WSGIDaemonProcess %(project_name)s user=%(user)s group=%(group)s processes=%(processes)d \\\n'
' threads=%(threads)d python-path=%(site_root)s \\\n'
' display-name=%%{GROUP}\n'
'WSGIProcessGroup %(project_name)s\n'
'WSGIScriptAlias / %(project_root)s/wsgi.py\n'
'WSGIPassAuthorization On\n\n'
'<Directory %(project_root)s>\n'
' <Files wsgi.py>\n'
' Order deny,allow\n'
' Allow from all\n'
' </Files>\n'
'</Directory>\n\n'
'Alias /media/ %(media_root)s/\n'
'Alias /static/ %(static_root)s/\n'
'<Directory %(static_root)s>\n'
' ExpiresActive On\n'
' ExpiresByType image/gif A1209600\n'
' ExpiresByType image/jpeg A1209600\n'
' ExpiresByType image/png A1209600\n'
' ExpiresByType text/css A1209600\n'
' ExpiresByType text/javascript A1209600\n'
' ExpiresByType application/x-javascript A1209600\n'
' <FilesMatch "\.(css|js|gz|png|gif|jpe?g|flv|swf|ico|pdf|txt|html|htm)$">\n'
' ContentDigest On\n'
' FileETag MTime Size\n'
' </FilesMatch>\n'
'</Directory>\n\n'
'RedirectMatch ^/$ /admin\n\n'
'<VirtualHost [%(mgmt_addr)s]:443>\n'
' ServerName localhost\n'
' SSLEngine on\n'
' SSLCertificateFile %(cert_path)s\n'
' SSLCertificateKeyFile %(cert_key_path)s\n'
' SSLCACertificateFile %(cert_path)s\n'
' SSLVerifyClient None\n'
'</VirtualHost>' % context
)
context.update({
'apache_conf': apache_conf,
'apache_conf_file': '/etc/apache2/conf.d/%(project_name)s.conf' % context
})
# Apache 2.4 compatibility - feature #684
run('mkdir -p /etc/apache2/conf.d')
run('mkdir -p /etc/apache2/conf-available')
compat_conf = (
'# This enables configuration files in the legacy Apache directory.\n'
'IncludeOptional conf.d/*.conf\n'
'\n'
'# These correct access control to Controller files for Apache 2.4.\n'
'<Directory %(project_root)s>\n'
' <Files wsgi.py>\n'
' Require all granted\n'
' </Files>\n'
'</Directory>\n'
'<Location /media>\n'
' Require all granted\n'
'</Location>\n'
'<Location /static>\n'
' Require all granted\n'
'</Location>' % context
)
context.update({
'compat_conf': compat_conf,
'compat_conf_file': '/etc/apache2/conf-available/local-%(project_name)s-compat.conf' % context
})
run("echo '%(compat_conf)s' > %(compat_conf_file)s" % context)
# Store apache2 configuration keeping existing one (if any).
diff = run("echo '%(apache_conf)s'|diff - %(apache_conf_file)s" % context, err_codes=[0,1,2])
if diff.return_code == 2:
# File does not exist
run("echo '%(apache_conf)s' > %(apache_conf_file)s" % context)
elif diff.return_code == 1:
# File is different, save the old one
if interactive:
msg = ("\n\nFile %(apache_conf_file)s should be updated, do "
"you like to override it? (yes/no): " % context)
confirm = input(msg)
while 1:
if confirm not in ('yes', 'no'):
confirm = input('Please enter either "yes" or "no": ')
continue
if confirm == 'no':
return
break
run("cp %(apache_conf_file)s %(apache_conf_file)s.\$save" % context)
run("echo '%(apache_conf)s' > %(apache_conf_file)s" % context)
self.stdout.write("\033[1;31mA new version of %(apache_conf_file)s "
"has been installed.\n The old version has been placed at "
"%(apache_conf_file)s.$save\033[m" % context)
run('a2enmod wsgi')
run('a2enmod expires')
run('a2enmod deflate')
run('a2enmod ssl')
# catch 127 error 'command not found' for apache 2.2 installations
run('a2enconf local-%(project_name)s-compat' % context, err_codes=[0, 127])
# Give read permissions to cert key file
run('chmod g+r %(cert_key_path)s' % context)
class Command(BaseCommand):
"""
Creates the tincd config files and Server.tinc object.
This method must be called by a superuser
"""
def __init__(self, *args, **kwargs):
# Options are defined in an __init__ method to support swapping out
# custom user models in tests.
super(Command, self).__init__(*args, **kwargs)
default_username = get_default_celeryd_username()
self.option_list = BaseCommand.option_list + (
make_option('--username',
dest='username',
default=default_username,
help='Specifies the login for the superuser.'),
make_option(
'--mgmt_prefix',
dest='mgmt_prefix',
default=settings.MGMT_IPV6_PREFIX,
help='Mgmt prefix, the settings file will be updated.'),
make_option(
'--default_port',
dest='default_port',
default=TINC_PORT_DFLT,
help='Tinc port default, the settings file will be updated.'),
make_option('--address',
dest='address',
default='0.0.0.0',
help='Tinc BindToAddress'),
make_option('--net_name',
dest='net_name',
default=TINC_NET_NAME,
help='Tinc net name'),
make_option(
'--nohup',
dest='nohup',
default=TINC_TINCD_SEND_HUP,
help=
'Whether we want to send a HUP signal to tinc after an update'
' or not. It requires sudo'),
make_option(
'--noinput',
action='store_false',
dest='interactive',
help=
'Tells Django to NOT prompt the user for input of any kind. '
'You must use --username with --noinput, and must contain the '
'cleeryd process owner, which is the user how will perform '
'tincd updates',
default=True),
)
option_list = BaseCommand.option_list
help = 'Creates the tincd config files and Server.tinc object'
@transaction.atomic
@check_root
def handle(self, *args, **options):
from tinc.models import TincHost, TincAddress
interactive = options.get('interactive')
username = options.get('username')
if not interactive:
# validate username
if not username:
raise CommandError("You must use --username with --noinput.")
try:
username = pwd.getpwnam(username).pw_name
except KeyError:
raise CommandError("Username doesn't exists.")
else:
# Prompt for username
# Enclose this whole thing in a try/except to trap for a
# keyboard interrupt and exit gracefully.
prompt_username = None
while prompt_username is None:
if not prompt_username:
input_msg = "Celeryd username"
if username:
input_msg += " (leave blank to use '%s')" % username
value = input(input_msg + ': ')
if username and value == '':
value = username
# validate username
try:
prompt_username = pwd.getpwnam(value).pw_name
except KeyError, e:
self.stderr.write("Error: %s" % e)
prompt_username = None
continue
server = Server.objects.first()
server_ct = ContentType.objects.get_for_model(Server)
tinc_server, __ = TincHost.objects.get_or_create(
object_id=server.pk, content_type=server_ct)
tinc_port = options.get('default_port')
tinc_address = options.get('address')
tinc_net_name = options.get('net_name')
mgmt_prefix = options.get('mgmt_prefix')
if mgmt_prefix != settings.MGMT_IPV6_PREFIX:
update_settings(MGMT_IPV6_PREFIX=mgmt_prefix,
monkey_patch='controller.settings')
if tinc_port != TINC_PORT_DFLT:
update_settings(TINC_PORT_DFLT=tinc_port)
if tinc_net_name != TINC_NET_NAME:
update_settings(TINC_NET_NAME=tinc_net_name)
context = {
'tincd_root':
TINC_TINCD_ROOT,
'tincd_bin':
TINC_TINCD_BIN,
'net_name':
tinc_net_name,
'net_root':
os.path.join(TINC_TINCD_ROOT, tinc_net_name),
'tinc_conf': ("BindToAddress = %s\n"
"Port = %s\n"
"Name = server\n"
"StrictSubnets = yes" % (tinc_address, tinc_port)),
'tinc_up':
tinc_server.get_tinc_up(),
'tinc_down':
tinc_server.get_tinc_down(),
'mgmt_prefix':
mgmt_prefix.split('::')[0],
'user':
username
}
r = functools.partial(run, silent=False)
boots = run("grep %(net_name)s %(tincd_root)s/nets.boot" % context,
err_codes=[0, 1])
if boots.return_code == 1:
r("echo %(net_name)s >> %(tincd_root)s/nets.boot" % context)
r("mkdir -p %(net_root)s/hosts" % context)
r("echo '%(tinc_conf)s' > %(net_root)s/tinc.conf" % context)
r('echo "Subnet = %(mgmt_prefix)s:0:0:0:0:2/128" > %(net_root)s/hosts/server'
% context)
r("echo '%(tinc_up)s' > %(net_root)s/tinc-up" % context)
r("echo '%(tinc_down)s' > %(net_root)s/tinc-down" % context)
r("chown %(user)s %(net_root)s/hosts" % context)
r("chmod +x %(net_root)s/tinc-up" % context)
r("chmod +x %(net_root)s/tinc-down" % context)
TincAddress.objects.get_or_create(host=tinc_server,
addr=tinc_address,
port=tinc_port)
priv_key = os.path.join(TINC_TINCD_ROOT, tinc_net_name, 'rsa_key.priv')
try:
priv_key = RSA.load_key(priv_key)
except:
# generate a new key
r('tincd -c %(net_root)s -K' % context)
priv_key = RSA.load_key(priv_key)
bio = BIO.MemoryBuffer()
priv_key.save_pub_key_bio(bio)
pub_key = bio.getvalue()
tinc_server.pubkey = pub_key
tinc_server.save()
if options.get('nohup'):
sudoers_hup = (
"%(user)s\s\s*ALL=NOPASSWD:\s\s*%(tincd_bin)s\s\s*-kHUP\s\s*-n %(net_name)s"
) % context
sudoers_exists = run('grep "%s" /etc/sudoers' % sudoers_hup,
err_codes=[0, 1, 2])
if sudoers_exists.return_code == 1:
cmd = "%(user)s ALL=NOPASSWD: %(tincd_bin)s -kHUP -n %(net_name)s" % context
r("echo '%s' >> /etc/sudoers" % cmd)
elif sudoers_exists.return_code == 2:
raise CommandError(
'Sudo is not installed on your system. \n'
'You may want to use --nohup option, so sudo will not be a requirement'
)
else:
update_settings(TINC_TINCD_SEND_HUP=False)
self.stdout.write('Tincd server successfully created and configured.')
self.stdout.write(
'NOTE: restarting the following services is required '
'to apply updated configuration:\n'
'tincd, uwsgi, celeryd.\n'
'Please run: "sudo python manage.py restartservices" or '
'restart them manually.')
def handle(self, *args, **options):
# Warn about deprecated options
if options.get('local'):
self.stderr.write("Warning: 'local' option is deprecated and will be ignored.\n")
version = options.get('version')
upgrade_notes = []
if version:
try:
major, major2, minor = decode_version(version)
except ValueError as e:
raise CommandError(e)
# Represent version as two digits per number: 1.2.2 -> 10202
version = int(str(major) + "%02d" % int(major2) + "%02d" % int(minor))
# Pre-upgrade operations (version specific)
if version < 835:
# prevent schema migrations from failing
if is_installed('firmware'):
from firmware.models import Build
Build.objects.filter(base_image=None).update(base_image='')
if version <= 902:
if is_installed('maintenance'):
# Apply losed migrations
from south.models import MigrationHistory
migrated = MigrationHistory.objects.filter(app_name='maintenance').exists()
if not migrated:
run('python manage.py migrate maintenance 0001 --fake')
if version < 1002:
# Update monitor settings (fix typo and add DiskFreeMonitor)
context = {
'settings': run("find . -type f -name 'settings.py'|grep -v 'vct/'")
}
# Try automaticate update (making a backup)
if context['settings']:
run("cp %(settings)s %(settings)s.upgrade.bak" % context)
# fix NumProcessesMonitor typo
run("sed -i 's/NumPocessesMonitor/NumProcessesMonitor/g' "
"%(settings)s" % context)
# append disk monitor (if needed)
# this is a rude check (but runned in a conservative way)
if 'DiskFreeMonitor' not in open(context['settings']).read():
run("sed -i '/MONITOR_MONITORS = ($/ a\ "
" (\"monitor.monitors.DiskFreeMonitor\",),' "
"%(settings)s" % context)
# warn the user about settings changes
autoupdate_status = 'OK' if context['settings'] else 'FAIL'
upgrade_notes.append('The monitor application has changed and .'
'some settings updates are required:\n'
' - Fix typo on NumProcessesMonitor (missing "r")\n'
' - Enable disk monitor\n'
' Please read the monitor app doc (MONITOR_MONITORS setting)\n'
'AUTOUPDATE: %s' % autoupdate_status)
if version <= 1102:
# Handle InconsistentMigrationHistory on tinc app
# * v0.11.2 tinc includes 0022, 0028..0030
# * v0.11.3 tinc adds 0023..0027
# We can force south to merge migrations because
# are implemented to be runned without dependencies
run('python manage.py migrate tinc 0030 --merge --noinput')
if not options.get('specifics_only'):
# Common stuff
development = options.get('development')
controller_admin = os.path.join(os.path.dirname(__file__), '../../bin/')
controller_admin = os.path.join(controller_admin, 'controller-admin.sh')
run('chmod +x %s' % controller_admin)
extra = '--development' if development else ''
if options.get('proxy'):
extra += ' --proxy %s' % options.get('proxy')
run("%s install_requirements " % controller_admin + extra)
run("python manage.py collectstatic --noinput")
run("python manage.py syncdb --noinput")
run("python manage.py migrate --noinput")
if is_installed('firmware'):
run("python manage.py syncfirmwareplugins")
if is_installed('notifications'):
run("python manage.py syncnotifications")
if is_installed('resources'):
run("python manage.py syncresources")
if options.get('restart'):
run("python manage.py restartservices")
if not version:
self.stderr.write('\nNext time you migth want to provide a --from argument '
'in order to run version specific upgrade operations\n')
return
# Post-upgrade operations (version specific)
if version <= 629:
# Clean existing sessions because of change on auth backend
run('echo "delete from django_session;" | python manage.py dbshell')
if version < 801:
deprecate_periodic_tasks(('state.ping',))
if version < 809:
# Add PKI directories
from pki import ca
from controller.utils.paths import get_site_root
site_root = get_site_root()
username = run("stat -c %%U %s" % site_root)
get_dir = lambda f: os.path.dirname(getattr(ca, f+'_path'))
for d in set( get_dir(f) for f in ['priv_key', 'pub_key', 'cert'] ):
run('mkdir -p %s' % d)
run('chown %s %s' % (username, d))
upgrade_notes.append('HTTPS certificate support for the management '
'network has been introduced in version 0.8.9.\n'
'In order to use it you sould run:\n'
' > python manage.py setuppki\n'
' > sudo python manage.py setupapache\n')
if version < 838:
# Purge communitynetworks.periodic_cache_node_db
from djcelery.models import PeriodicTask
PeriodicTask.objects.filter(name='communitynetworks.periodic_cache_node_db').delete()
run('rabbitmqctl stop_app')
run('rabbitmqctl reset')
run('rabbitmqctl start_app')
run('service celeryd restart')
upgrade_notes.append('New Celeryd init.d configuration has been '
'introduced in 0.8.38.\nIt is strongly recommended to upgrade by\n'
' > sudo python manage.py setupceleryd\n')
# Deprecate x86 and amd64 architectures
from nodes.models import Node
Node.objects.filter(arch='x86').update(arch='i686')
Node.objects.filter(arch='amd64').update(arch='x86_64')
upgrade_notes.append('In order to support Base authentication while downloading '
'firmwares you should add "WSGIPassAuthorization On" on your apache config.\n'
'Alternatively you can perform this operation with the following command\n'
' > sudo python manage.py setupapache\n'
' > /etc/init.d/apache2 reload\n')
if version < 900:
upgrade_notes.append('Apache configuration is now placed under '
'/etc/apache2/conf.d/<project_name>.conf. It is convenient for you '
'to migrate your current configuration located on /etc/apache2/httpd.conf '
'to this new location.\n')
upgrade_notes.append('Celery workers configuration has been updated. '
'Please update it by running:\n'
' > sudo python manage.py setupceleryd\n')
if version < 905:
# TODO find the root cause of this
# maybe is shit imported on settings that import settings like add_app
# Prevent crazy import erros to appear :S
from django.utils import translation
translation.activate('en-us')
# Change template types for more generic ones
from slices.models import Template
from slices.settings import SLICES_TEMPLATE_TYPES
template_types = [ t[0] for t in SLICES_TEMPLATE_TYPES ]
if 'debian' in template_types:
Template.objects.filter(type='debian6').update(type='debian')
if 'openwrt' in template_types:
Template.objects.filter(type='openwrt-backfire').update(type='openwrt')
if version < 906:
deprecate_periodic_tasks(('state.nodestate', 'state.sliverstate'))
if version <= 907:
# Generate sha256
from slices.models import Template
for template in Template.objects.all():
template.save()
upgrade_notes.append("It is extremly recommended to update your database "
"settings to enable atomic request behaviour:\n"
" https://docs.djangoproject.com/en/dev/topics/db/transactions/#tying-transactions-to-http-requests\n"
"Just add:\n"
" 'ATOMIC_REQUESTS': True,\n"
"into DATABASES setting within <project_name>/<project_name>/settings.py")
if version <= 1003:
# Update firmware configuration after Island refactor (#264)
from firmware.models import ConfigFile
try:
cfg_file = ConfigFile.objects.get(path__contains="node.tinc.connect_to")
except (ConfigFile.DoesNotExist, ConfigFile.MultipleObjectsReturned):
# Warn the user that needs to perform manual update
msg = "Firmware configuration update has failed. "
else:
cfg_file.content = cfg_file.content.replace("node.tinc.island", "node.island")
cfg_file.save()
msg = "Firmware configuration updated successfully. Updated ConfigFile ID: %i." % cfg_file.pk
upgrade_notes.append("%s\nPlease check version 0.10.4 release notes:\n"
"https://wiki.confine-project.eu/soft:server-release-notes#section0104" % msg)
if version < 1103:
# Update mgmt network Server APIs certificate
# perform raw SQL querie because models hasn't been
# reloaded yet and cannot access e.g. server.api
from django.db import connection
from nodes.models import Server
from pki import ca
server_id = Server.objects.order_by('id').first().pk
try:
cert = ca.get_cert().as_pem()
except IOError:
msg = ("Failed to update Server APIs certificate. Missing "
"server certificate '%s'.\n"
"Run 'python manage.py setuppki --help'" % ca.cert_path)
upgrade_notes.append(msg)
else:
update_sql = ('UPDATE "nodes_serverapi" SET cert = %s '
'WHERE "nodes_serverapi"."server_id" = %s')
cursor = connection.cursor()
cursor.execute(update_sql, [cert, server_id])
del cursor
upgrade_notes.append("Updated Server APIs certificate.")
if upgrade_notes and options.get('print_upgrade_notes'):
self.stdout.write('\n\033[1m\n'
' ===================\n'
' ** UPGRADE NOTES **\n'
' ===================\n\n' +
'\n'.join(upgrade_notes) + '\033[m\n')
def vct_node(action, node, silent=True):
node_id = hex(node.id).split('0x')[1]
node_id = '0'*(4-len(node_id)) + node_id
wrapper = path.join(get_vct_root(), '%s')
cmd = 'vct_node_%s %s' % (action, node_id)
return run(wrapper % cmd, display=False, silent=silent)
def handle(self, *args, **options):
# Configure firmware generation
context = {
'db_name': options.get('db_name'),
'db_user': options.get('db_user'),
'db_password': options.get('db_password'),
'db_host': options.get('db_host'),
'db_port': options.get('db_port') }
#Below 2 lines will be running where postgres container is running
if os.path.exists('/home/vct/vct_true') is True:
run('su postgres -c "psql -c \\"CREATE USER %(db_user)s PASSWORD \'%(db_password)s\';\\""' % context, err_codes=[0,1])
run('su postgres -c "psql -c \\"CREATE DATABASE %(db_name)s OWNER %(db_user)s;\\""' % context, err_codes=[0,1])
context.update({'settings': os.path.join(get_project_root(), 'settings.py')})
if run("grep 'DATABASES' %(settings)s" % context, err_codes=[0,1]).return_code == 0:
# Update existing settings_file
run("sed -i \"s/'ENGINE': '\w*',/'ENGINE': 'django.db.backends.postgresql_psycopg2',/\" %(settings)s" % context)
run("sed -i \"s/'NAME': '.*',/'NAME': '%(db_name)s',/\" %(settings)s" % context)
run("sed -i \"s/'USER': '******',/'USER': '******',/\" %(settings)s" % context)
run("sed -i \"s/'PASSWORD': '******',/'PASSWORD': '******',/\" %(settings)s" % context)
run("sed -i \"s/'HOST': '.*',/'HOST': '%(db_host)s',/\" %(settings)s" % context)
run("sed -i \"s/'PORT': '.*',/'PORT': '%(db_port)s',/\" %(settings)s" % context)
else:
db_config = (
"DATABASES = {\n"
" 'default': {\n"
" 'ENGINE': 'django.db.backends.postgresql_psycopg2',\n"
" 'NAME': '%(db_name)s',\n"
" 'USER': '******',\n"
" 'PASSWORD': '******',\n"
" 'HOST': '%(db_host)s',\n"
" 'PORT': '%(db_port)s',\n"
" 'ATOMIC_REQUESTS': True,\n"
" }\n"
"}\n" % context)
context.update({'db_config': db_config})
run('echo "%(db_config)s" >> %(settings)s' % context)
def handle(self, *args, **options):
current_version = get_version()
current_path = get_existing_pip_installation()
proxy = '--proxy %s' % options.get('proxy') if options.get(
'proxy') else ''
if current_path is not None:
desired_version = options.get('version')
validate_controller_version(desired_version)
if current_version == desired_version:
msg = "Not upgrading, you already have version %s installed"
raise CommandError(msg % desired_version)
# Create a backup of current installation
base_path = os.path.abspath(os.path.join(current_path, '..'))
char_set = string.ascii_uppercase + string.digits
rand_name = ''.join(random.sample(char_set, 6))
backup = os.path.join(base_path, 'controller.' + rand_name)
run("mv %s %s" % (current_path, backup))
# collect existing eggs previous to the installation
eggs_regex = os.path.join(base_path,
'confine_controller-*.egg-info')
eggs = run('ls -d %s' % eggs_regex)
eggs = eggs.stdout.splitlines()
try:
if desired_version:
if desired_version == 'dev':
r('pip install -e git+http://git.confine-project.eu/confine/controller.git@master#egg=confine-controller'
)
elif desired_version == 'beta':
r('pip install -e git+http://git.confine-project.eu/confine/controller.git@beta#egg=confine-controller'
)
else:
r('pip %s install confine-controller==%s' %
(proxy, desired_version))
else:
# Did I mentioned how I hate PIP?
if run('pip --version|cut -d" " -f2').stdout == '1.0':
r('pip %s install confine-controller --upgrade' %
proxy)
else:
# (F*****g pip)^2, it returns exit code 0 even when fails
# because requirement already up-to-date
r('pip %s install confine-controller --upgrade --force'
% proxy)
except CommandError:
# Restore backup
run('rm -rf %s' % current_path)
run('mv %s %s' % (backup, current_path))
raise CommandError("Problem runing pip upgrade, aborting...")
else:
# Some old versions of pip do not performe this cleaning ...
# Remove all backups
run('rm -fr %s' % os.path.join(base_path, 'controller\.*'))
# Clean old egg files, yeah, cleaning PIP shit :P
c_version = 'from controller import get_version; print get_version()'
version = run('python -c "%s;"' % c_version).stdout
for egg in eggs:
# Do not remove the actual egg file when upgrading twice the same version
if egg.split('/')[
-1] != "confine_controller-%s.egg-info" % version:
run('rm -fr %s' % egg)
else:
raise CommandError(
"You don't seem to have any previous PIP installation")
# version specific upgrade operations
if not options.get('pip_only'):
call_command("postupgradecontroller",
version=current_version,
proxy=options.get('proxy'))
def handle(self, *args, **options):
if options.get('username'):
warnings.warn('"username" has been deprecated in favor of "user"',
DeprecationWarning)
if options.get('user') and options.get('username'):
raise CommandError(
"Only one of this options should be provided: --user OR --username"
)
user = options.get('user') or options.get('username') or 'confine'
context = {
'site_root': get_site_root(),
'user': user,
'group': options.get('group') or user,
'bin_path': path.join(get_controller_root(), 'bin'),
'processes': options.get('processes'),
'greenlets': options.get('greenlets')
}
celery_config = (
'# Name of nodes to start, here we have a single node\n'
'CELERYD_NODES="w1 w2"\n'
'\n'
'# Where to chdir at start.\n'
'CELERYD_CHDIR="%(site_root)s"\n'
'\n'
'# How to call "manage.py celeryd_multi"\n'
'CELERYD_MULTI="$CELERYD_CHDIR/manage.py celeryd_multi"\n'
'\n'
'# Extra arguments to celeryd\n'
'CELERYD_OPTS="-P:w1 processes -c:w1 %(processes)s -Q:w1 celery \\\n'
' -P:w2 gevent -c:w2 %(greenlets)s -Q:w2 gevent --time-limit=300"\n'
'\n'
'# Name of the celery config module.\n'
'CELERY_CONFIG_MODULE="celeryconfig"\n'
'\n'
'# %%n will be replaced with the nodename.\n'
'CELERYD_LOG_FILE="/var/log/celery/%%n.log"\n'
'CELERYD_PID_FILE="/var/run/celery/%%n.pid"\n'
'CELERY_CREATE_DIRS=1\n'
'\n'
'# Full path to the celeryd logfile.\n'
'CELERYEV_LOG_FILE="/var/log/celery/celeryev.log"\n'
'CELERYEV_PID_FILE="/var/run/celery/celeryev.pid"\n'
'\n'
'# Workers should run as an unprivileged user.\n'
'CELERYD_USER="******"\n'
'CELERYD_GROUP="%(group)s"\n'
'\n'
'# Persistent revokes\n'
'CELERYD_STATE_DB="$CELERYD_CHDIR/persistent_revokes"\n'
'\n'
'# Celeryev\n'
'CELERYEV="$CELERYD_CHDIR/manage.py"\n'
'CELERYEV_CAM="djcelery.snapshot.Camera"\n'
'CELERYEV_USER="******"\n'
'CELERYEV_GROUP="$CELERYD_GROUP"\n'
'CELERYEV_OPTS="celeryev"\n'
'\n'
'# Celerybeat\n'
'CELERYBEAT="${CELERYD_CHDIR}/manage.py celerybeat"\n'
'CELERYBEAT_USER="******"\n'
'CELERYBEAT_GROUP="$CELERYD_GROUP"\n'
'CELERYBEAT_CHDIR="$CELERYD_CHDIR"\n'
'CELERYBEAT_OPTS="--schedule=/var/run/celerybeat-schedule"\n' %
context)
run("echo '%s' > /etc/default/celeryd" % celery_config)
# https://raw.github.com/celery/celery/master/extra/generic-init.d/
for script in ['celeryevcam', 'celeryd', 'celerybeat']:
context['script'] = script
run('cp %(bin_path)s/%(script)s /etc/init.d/%(script)s' % context)
run('chmod +x /etc/init.d/%(script)s' % context)
run('update-rc.d %(script)s defaults' % context)
rotate = ('/var/log/celery/*.log {\n'
' weekly\n'
' missingok\n'
' rotate 10\n'
' compress\n'
' delaycompress\n'
' notifempty\n'
' copytruncate\n'
'}')
run("echo '%s' > /etc/logrotate.d/celeryd" % rotate)
def handle(self, *args, **options):
current_version = get_version()
current_path = get_existing_pip_installation()
proxy = '--proxy %s' % options.get('proxy') if options.get('proxy') else ''
# # Getting version that will be installed, yeah pip doesn't support it :)
# pypi_url = 'https://pypi.python.org/pypi/confine-controller'
# grep = 'href="/pypi?:action=doap&name=confine-controller&version='
# extract = '| head -n1 | cut -d"=" -f5 | cut -d\'"\' -f1'
# pypi_version = "wget -q '%s' -O - | grep '%s' %s" % (pypi_url, grep, extract)
# pypi_version = run(pypi_version).stdout
#
# if current_version == pypi_version:
# msg = "Not upgrading, you already have version %s installed"
# raise CommandError(msg % current_version)
if current_path is not None:
desired_version = options.get('version')
validate_controller_version(desired_version)
if current_version == desired_version:
msg = "Not upgrading, you already have version %s installed"
raise CommandError(msg % desired_version)
# Create a backup of current installation
base_path = os.path.abspath(os.path.join(current_path, '..'))
char_set = string.ascii_uppercase + string.digits
rand_name = ''.join(random.sample(char_set, 6))
backup = os.path.join(base_path, 'controller.' + rand_name)
run("mv %s %s" % (current_path, backup))
# collect existing eggs previous to the installation
eggs_regex = os.path.join(base_path, 'confine_controller-*.egg-info')
eggs = run('ls -d %s' % eggs_regex)
eggs = eggs.stdout.splitlines()
try:
if desired_version:
# if desired_version == 'dev':
# r('pip install -e git+http://git.confine-project.eu/confine/controller.git@master#egg=confine-controller')
# else:
r('pip %s install confine-controller==%s' % (proxy, desired_version))
else:
# Did I mentioned how I hate PIP?
if run('pip --version|cut -d" " -f2').stdout == '1.0':
r('pip %s install confine-controller --upgrade' % proxy)
else:
# (F*****g pip)^2, it returns exit code 0 even when fails
# because requirement already up-to-date
r('pip %s install confine-controller --upgrade --force' % proxy)
except CommandError:
# Restore backup
run('rm -rf %s' % current_path)
run('mv %s %s' % (backup, current_path))
raise CommandError("Problem runing pip upgrade, aborting...")
else:
# Some old versions of pip do not performe this cleaning ...
# Remove all backups
run('rm -fr %s' % os.path.join(base_path, 'controller\.*'))
# Clean old egg files, yeah, cleaning PIP shit :P
c_version = 'from controller import get_version; print get_version()'
version = run('python -c "%s;"' % c_version).stdout
for egg in eggs:
# Do not remove the actual egg file when upgrading twice the same version
if egg.split('/')[-1] != "confine_controller-%s.egg-info" % version:
run('rm -fr %s' % egg)
else:
raise CommandError("You don't seem to have any previous PIP installation")
# version specific upgrade operations
if not options.get('pip_only'):
call_command("postupgradecontroller", version=current_version,
proxy=options.get('proxy'))
def handle(self, *args, **options):
interactive = options.get('interactive')
# Avoid import errors
from nodes.models import Server
server = Server.objects.first()
context = {
'project_name': get_project_name(),
'project_root': get_project_root(),
'site_root': get_site_root(),
'media_root': settings.MEDIA_ROOT,
'static_root': settings.STATIC_ROOT,
'cert_path': ca.cert_path,
'cert_key_path': ca.priv_key_path,
'mgmt_addr': str(server.mgmt_net.addr),
'user': options.get('user'),
'group': options.get('group') or options.get('user'),
'home': expanduser("~%s" % options.get('user')),
'processes': int(options.get('processes')),}
nginx_conf = (
'server {\n'
' listen 80;\n'
' listen [::]:80 ipv6only=on;\n'
' rewrite ^/$ /admin;\n'
' client_max_body_size 500m;\n'
' location / {\n'
' uwsgi_pass unix:///var/run/uwsgi/app/%(project_name)s/socket;\n'
' include uwsgi_params;\n'
' }\n'
' location /media {\n'
' alias %(media_root)s;\n'
' expires 30d;\n'
' }\n'
' location /static {\n'
' alias %(static_root)s;\n'
' expires 30d;\n'
' }\n'
'}\n'
'\n'
'server {\n'
' listen [%(mgmt_addr)s]:443 default_server ssl;\n'
' ssl_certificate %(cert_path)s;\n'
' ssl_certificate_key %(cert_key_path)s;\n'
' rewrite ^/$ /admin;\n'
' client_max_body_size 50m;\n'
' location / {\n'
' uwsgi_pass unix:///var/run/uwsgi/app/%(project_name)s/socket;\n'
' include uwsgi_params;\n'
' }\n'
' location /media {\n'
' alias %(media_root)s;\n'
' expires 30d;\n'
' }\n'
' location /static {\n'
' alias %(static_root)s;\n'
' expires 30d;\n'
' }\n'
'}\n') % context
uwsgi_conf = (
'[uwsgi]\n'
'plugins = python\n'
'chdir = %(site_root)s\n'
'module = %(project_name)s.wsgi\n'
'master = true\n'
'processes = %(processes)d\n'
'chmod-socket = 664\n'
'stats = /run/uwsgi/%%(deb-confnamespace)/%%(deb-confname)/statsocket\n'
'vacuum = true\n'
'uid = %(user)s\n'
'gid = %(group)s\n'
'env = HOME=%(home)s\n') % context
nginx = {
'file': '/etc/nginx/conf.d/%(project_name)s.conf' % context,
'conf': nginx_conf }
uwsgi = {
'file': '/etc/uwsgi/apps-available/%(project_name)s.ini' % context,
'conf': uwsgi_conf }
for extra_context in (nginx, uwsgi):
context.update(extra_context)
diff = run("echo '%(conf)s'|diff - %(file)s" % context, err_codes=[0,1,2])
if diff.return_code == 2:
# File does not exist
run("echo '%(conf)s' > %(file)s" % context)
elif diff.return_code == 1:
# File is different, save the old one
if interactive:
msg = ("\n\nFile %(file)s be updated, do you like to override "
"it? (yes/no): " % context)
confirm = input(msg)
while 1:
if confirm not in ('yes', 'no'):
confirm = input('Please enter either "yes" or "no": ')
continue
if confirm == 'no':
return
break
run("cp %(file)s %(file)s.save" % context)
run("echo '%(conf)s' > %(file)s" % context)
self.stdout.write("\033[1;31mA new version of %(file)s has been installed.\n "
"The old version has been placed at %(file)s.save\033[m" % context)
run('ln -s /etc/uwsgi/apps-available/%(project_name)s.ini /etc/uwsgi/apps-enabled/' % context, err_codes=[0,1])
# Give read permissions to cert key file
run('chmod g+r %(cert_key_path)s' % context)
# nginx should start after tincd
current = "\$local_fs \$remote_fs \$network \$syslog"
run('sed -i "s/ %s$/ %s \$named/g" /etc/init.d/nginx' % (current, current))
rotate = (
'/var/log/nginx/*.log {\n'
' daily\n'
' missingok\n'
' rotate 30\n'
' compress\n'
' delaycompress\n'
' notifempty\n'
' create 640 root adm\n'
' sharedscripts\n'
' postrotate\n'
' [ ! -f /var/run/nginx.pid ] || kill -USR1 `cat /var/run/nginx.pid`\n'
' endscript\n'
'}\n')
run("echo '%s' > /etc/logrotate.d/nginx" % rotate)
# Allow nginx to write to uwsgi socket
run('adduser www-data %(group)s' % context)
def execute(self):
env = "export LINES=1000; export COLUMNS=1000; "
return json.loads(run(env + self.cmd).stdout), []
def handle(self, *args, **options):
interactive = options.get('interactive')
# Avoid import errors
from nodes.models import Server
server = Server.objects.first()
context = {
'project_name': get_project_name(),
'project_root': get_project_root(),
'site_root': get_site_root(),
'media_root': settings.MEDIA_ROOT,
'static_root': settings.STATIC_ROOT,
'cert_path': ca.cert_path,
'cert_key_path': ca.priv_key_path,
'mgmt_addr': str(server.mgmt_net.addr),
'user': options.get('user'),
'group': options.get('group') or options.get('user'),
'home': expanduser("~%s" % options.get('user')),
'processes': int(options.get('processes')),
}
nginx_conf = (
'server {\n'
' listen 80;\n'
' listen [::]:80 ipv6only=on;\n'
' rewrite ^/$ /admin;\n'
' client_max_body_size 500m;\n'
' location / {\n'
' uwsgi_pass unix:///var/run/uwsgi/app/%(project_name)s/socket;\n'
' include uwsgi_params;\n'
' }\n'
' location /media {\n'
' alias %(media_root)s;\n'
' expires 30d;\n'
' }\n'
' location /static {\n'
' alias %(static_root)s;\n'
' expires 30d;\n'
' }\n'
'}\n'
'\n'
'server {\n'
' listen [%(mgmt_addr)s]:443 default_server ssl;\n'
' ssl_certificate %(cert_path)s;\n'
' ssl_certificate_key %(cert_key_path)s;\n'
' rewrite ^/$ /admin;\n'
' client_max_body_size 50m;\n'
' location / {\n'
' uwsgi_pass unix:///var/run/uwsgi/app/%(project_name)s/socket;\n'
' include uwsgi_params;\n'
' }\n'
' location /media {\n'
' alias %(media_root)s;\n'
' expires 30d;\n'
' }\n'
' location /static {\n'
' alias %(static_root)s;\n'
' expires 30d;\n'
' }\n'
'}\n') % context
uwsgi_conf = (
'[uwsgi]\n'
'plugins = python\n'
'chdir = %(site_root)s\n'
'module = %(project_name)s.wsgi\n'
'master = true\n'
'processes = %(processes)d\n'
'chmod-socket = 664\n'
'stats = /run/uwsgi/%%(deb-confnamespace)/%%(deb-confname)/statsocket\n'
'vacuum = true\n'
'uid = %(user)s\n'
'gid = %(group)s\n'
'env = HOME=%(home)s\n') % context
nginx = {
'file': '/etc/nginx/conf.d/%(project_name)s.conf' % context,
'conf': nginx_conf
}
uwsgi = {
'file': '/etc/uwsgi/apps-available/%(project_name)s.ini' % context,
'conf': uwsgi_conf
}
for extra_context in (nginx, uwsgi):
context.update(extra_context)
diff = run("echo '%(conf)s'|diff - %(file)s" % context,
err_codes=[0, 1, 2])
if diff.return_code == 2:
# File does not exist
run("echo '%(conf)s' > %(file)s" % context)
elif diff.return_code == 1:
# File is different, save the old one
if interactive:
msg = (
"\n\nFile %(file)s be updated, do you like to override "
"it? (yes/no): " % context)
confirm = input(msg)
while 1:
if confirm not in ('yes', 'no'):
confirm = input(
'Please enter either "yes" or "no": ')
continue
if confirm == 'no':
return
break
run("cp %(file)s %(file)s.save" % context)
run("echo '%(conf)s' > %(file)s" % context)
self.stdout.write(
"\033[1;31mA new version of %(file)s has been installed.\n "
"The old version has been placed at %(file)s.save\033[m" %
context)
run('ln -s /etc/uwsgi/apps-available/%(project_name)s.ini /etc/uwsgi/apps-enabled/'
% context,
err_codes=[0, 1])
# Disable default site
run('rm -f /etc/nginx/sites-enabled/default')
# Give read permissions to cert key file
run('chmod g+r %(cert_key_path)s' % context)
# nginx should start after tincd
current = "\$local_fs \$remote_fs \$network \$syslog"
run('sed -i "s/ %s$/ %s \$named/g" /etc/init.d/nginx' %
(current, current))
rotate = (
'/var/log/nginx/*.log {\n'
' daily\n'
' missingok\n'
' rotate 30\n'
' compress\n'
' delaycompress\n'
' notifempty\n'
' create 640 root adm\n'
' sharedscripts\n'
' postrotate\n'
' [ ! -f /var/run/nginx.pid ] || kill -USR1 `cat /var/run/nginx.pid`\n'
' endscript\n'
'}\n')
run("echo '%s' > /etc/logrotate.d/nginx" % rotate)
# Allow nginx to write to uwsgi socket
run('adduser www-data %(group)s' % context)
# Restart nginx to apply new configuration and also uwsgi
# to force reloading permissions (user added to a group)
run('service nginx restart')
run('service uwsgi restart')
def handle(self, *args, **options):
if options.get('username'):
warnings.warn('"username" has been deprecated in favor of "user"', DeprecationWarning)
if options.get('user') and options.get('username'):
raise CommandError("Only one of this options should be provided: --user OR --username")
user = options.get('user') or options.get('username') or 'confine'
context = {'site_root': get_site_root(),
'user': user,
'group': options.get('group') or user,
'bin_path': path.join(get_controller_root(), 'bin'),
'processes': options.get('processes'),
'greenlets': options.get('greenlets') }
celery_config = (
'# Name of nodes to start, here we have a single node\n'
'CELERYD_NODES="w1 w2"\n'
'\n'
'# Where to chdir at start.\n'
'CELERYD_CHDIR="%(site_root)s"\n'
'\n'
'# How to call "manage.py celeryd_multi"\n'
'CELERYD_MULTI="$CELERYD_CHDIR/manage.py celeryd_multi"\n'
'\n'
'# Extra arguments to celeryd\n'
'CELERYD_OPTS="-P:w1 processes -c:w1 %(processes)s -Q:w1 celery \\\n'
' -P:w2 gevent -c:w2 %(greenlets)s -Q:w2 gevent --time-limit=300"\n'
'\n'
'# Name of the celery config module.\n'
'CELERY_CONFIG_MODULE="celeryconfig"\n'
'\n'
'# %%n will be replaced with the nodename.\n'
'CELERYD_LOG_FILE="/var/log/celery/%%n.log"\n'
'CELERYD_PID_FILE="/var/run/celery/%%n.pid"\n'
'CELERY_CREATE_DIRS=1\n'
'\n'
'# Full path to the celeryd logfile.\n'
'CELERYEV_LOG_FILE="/var/log/celery/celeryev.log"\n'
'CELERYEV_PID_FILE="/var/run/celery/celeryev.pid"\n'
'\n'
'# Workers should run as an unprivileged user.\n'
'CELERYD_USER="******"\n'
'CELERYD_GROUP="%(group)s"\n'
'\n'
'# Persistent revokes\n'
'CELERYD_STATE_DB="$CELERYD_CHDIR/persistent_revokes"\n'
'\n'
'# Celeryev\n'
'CELERYEV="$CELERYD_CHDIR/manage.py"\n'
'CELERYEV_CAM="djcelery.snapshot.Camera"\n'
'CELERYEV_USER="******"\n'
'CELERYEV_GROUP="$CELERYD_GROUP"\n'
'CELERYEV_OPTS="celeryev"\n'
'\n'
'# Celerybeat\n'
'CELERYBEAT="${CELERYD_CHDIR}/manage.py celerybeat"\n'
'CELERYBEAT_USER="******"\n'
'CELERYBEAT_GROUP="$CELERYD_GROUP"\n'
'CELERYBEAT_CHDIR="$CELERYD_CHDIR"\n'
'CELERYBEAT_OPTS="--schedule=/var/run/celerybeat-schedule"\n' % context)
run("echo '%s' > /etc/default/celeryd" % celery_config)
# https://raw.github.com/celery/celery/master/extra/generic-init.d/
for script in ['celeryevcam', 'celeryd', 'celerybeat']:
context['script'] = script
run('cp %(bin_path)s/%(script)s /etc/init.d/%(script)s' % context)
run('chmod +x /etc/init.d/%(script)s' % context)
run('update-rc.d %(script)s defaults' % context)
rotate = ('/var/log/celery/*.log {\n'
' weekly\n'
' missingok\n'
' rotate 10\n'
' compress\n'
' delaycompress\n'
' notifempty\n'
' copytruncate\n'
'}')
run("echo '%s' > /etc/logrotate.d/celeryd" % rotate)
def handle(self, *args, **options):
interactive = options.get('interactive')
# Avoid import errors
from nodes.models import Server
server = Server.objects.first()
context = {
'project_name': get_project_name(),
'project_root': get_project_root(),
'site_root': get_site_root(),
# TODO end with single /
'media_root': settings.MEDIA_ROOT,
'static_root': settings.STATIC_ROOT,
'cert_path': ca.cert_path,
'cert_key_path': ca.priv_key_path,
'mgmt_addr': str(server.mgmt_net.addr),
'user': options.get('user'),
'group': options.get('group') or options.get('user'),
'processes': int(options.get('processes')),
'threads': int(options.get('threads')) }
apache_conf = (
'WSGIDaemonProcess %(project_name)s user=%(user)s group=%(group)s processes=%(processes)d \\\n'
' threads=%(threads)d python-path=%(site_root)s \\\n'
' display-name=%%{GROUP}\n'
'WSGIProcessGroup %(project_name)s\n'
'WSGIScriptAlias / %(project_root)s/wsgi.py\n'
'WSGIPassAuthorization On\n\n'
'<Directory %(project_root)s>\n'
' Require all granted\n'
' <Files wsgi.py>\n'
' Order deny,allow\n'
' Allow from all\n'
' </Files>\n'
'</Directory>\n\n'
'Alias /media/ %(media_root)s/\n'
'Alias /static/ %(static_root)s/\n'
'<Directory %(static_root)s>\n'
' Require all granted\n'
' ExpiresActive On\n'
' ExpiresByType image/gif A1209600\n'
' ExpiresByType image/jpeg A1209600\n'
' ExpiresByType image/png A1209600\n'
' ExpiresByType text/css A1209600\n'
' ExpiresByType text/javascript A1209600\n'
' ExpiresByType application/x-javascript A1209600\n'
' <FilesMatch "\.(css|js|gz|png|gif|jpe?g|flv|swf|ico|pdf|txt|html|htm)$">\n'
' ContentDigest On\n'
' FileETag MTime Size\n'
' </FilesMatch>\n'
'</Directory>\n\n'
'RedirectMatch ^/$ /admin\n\n'
'<VirtualHost [%(mgmt_addr)s]:443>\n'
' ServerName localhost\n'
' SSLEngine on\n'
' SSLCertificateFile %(cert_path)s\n'
' SSLCertificateKeyFile %(cert_key_path)s\n'
' SSLCACertificateFile %(cert_path)s\n'
' SSLVerifyClient None\n'
'</VirtualHost>' % context)
context.update({
'apache_conf': apache_conf,
'apache_conf_file': '/etc/apache2/sites-enabled/%(project_name)s.conf' % context})
diff = run("echo '%(apache_conf)s'|diff - %(apache_conf_file)s" % context, err_codes=[0,1,2])
if diff.return_code == 2:
# File does not exist
run("echo '%(apache_conf)s' > %(apache_conf_file)s" % context)
elif diff.return_code == 1:
# File is different, save the old one
if interactive:
msg = ("\n\nFile %(apache_conf_file)s should be updated, do "
"you like to override it? (yes/no): " % context)
confirm = input(msg)
while 1:
if confirm not in ('yes', 'no'):
confirm = input('Please enter either "yes" or "no": ')
continue
if confirm == 'no':
return
break
run("cp %(apache_conf_file)s %(apache_conf_file)s.\$save" % context)
run("echo '%(apache_conf)s' > %(apache_conf_file)s" % context)
self.stdout.write("\033[1;31mA new version of %(apache_conf_file)s "
"has been installed.\n The old version has been placed at "
"%(apache_conf_file)s.$save\033[m" % context)
# include_httpd = run("grep '^\s*Include\s\s*httpd.conf\s*' /etc/apache2/apache2.conf", err_codes=[0,1])
# if include_httpd.return_code == 1:
# run("echo 'Include httpd.conf' >> /etc/apache2/apache2.conf")
# run('a2ensite %s' % project_name)
run('a2enmod wsgi')
run('a2enmod expires')
run('a2enmod deflate')
run('a2enmod ssl')
# Give upload file permissions to apache
# username = run("stat -c %%U %(project_root)s" % context)
# run('adduser www-data %s' % username)
# run('chmod g+w %(media_root)s' % context)
# Give read permissions to cert key file
run('chmod g+r %(cert_key_path)s' % context)
def handle(self, *args, **options):
# Warn about deprecated options
if options.get('local'):
self.stderr.write(
"Warning: 'local' option is deprecated and will be ignored.\n")
version = options.get('version')
upgrade_notes = []
if version:
try:
major, major2, minor = decode_version(version)
except ValueError as e:
raise CommandError(e)
# Represent version as two digits per number: 1.2.2 -> 10202
version = int(
str(major) + "%02d" % int(major2) + "%02d" % int(minor))
# Pre-upgrade operations (version specific)
if version < 835:
# prevent schema migrations from failing
if is_installed('firmware'):
from firmware.models import Build
Build.objects.filter(base_image=None).update(base_image='')
if version <= 902:
if is_installed('maintenance'):
# Apply losed migrations
from south.models import MigrationHistory
migrated = MigrationHistory.objects.filter(
app_name='maintenance').exists()
if not migrated:
run('python manage.py migrate maintenance 0001 --fake')
if version < 1002:
# Update monitor settings (fix typo and add DiskFreeMonitor)
context = {
'settings':
run("find . -type f -name 'settings.py'|grep -v 'vct/'")
}
# Try automaticate update (making a backup)
if context['settings']:
run("cp %(settings)s %(settings)s.upgrade.bak" % context)
# fix NumProcessesMonitor typo
run("sed -i 's/NumPocessesMonitor/NumProcessesMonitor/g' "
"%(settings)s" % context)
# append disk monitor (if needed)
# this is a rude check (but runned in a conservative way)
if 'DiskFreeMonitor' not in open(
context['settings']).read():
run("sed -i '/MONITOR_MONITORS = ($/ a\ "
" (\"monitor.monitors.DiskFreeMonitor\",),' "
"%(settings)s" % context)
# warn the user about settings changes
autoupdate_status = 'OK' if context['settings'] else 'FAIL'
upgrade_notes.append(
'The monitor application has changed and .'
'some settings updates are required:\n'
' - Fix typo on NumProcessesMonitor (missing "r")\n'
' - Enable disk monitor\n'
' Please read the monitor app doc (MONITOR_MONITORS setting)\n'
'AUTOUPDATE: %s' % autoupdate_status)
if version <= 1102:
# Handle InconsistentMigrationHistory on tinc app
# * v0.11.2 tinc includes 0022, 0028..0030
# * v0.11.3 tinc adds 0023..0027
# We can force south to merge migrations because
# are implemented to be runned without dependencies
run('python manage.py migrate tinc 0030 --merge --noinput')
if not options.get('specifics_only'):
# Common stuff
development = options.get('development')
controller_admin = os.path.join(os.path.dirname(__file__),
'../../bin/')
controller_admin = os.path.join(controller_admin,
'controller-admin.sh')
run('chmod +x %s' % controller_admin)
extra = '--development' if development else ''
if options.get('proxy'):
extra += ' --proxy %s' % options.get('proxy')
run("%s install_requirements " % controller_admin + extra)
run("python manage.py collectstatic --noinput")
run("python manage.py syncdb --noinput")
run("python manage.py migrate --noinput")
if is_installed('firmware'):
run("python manage.py syncfirmwareplugins")
if is_installed('notifications'):
run("python manage.py syncnotifications")
if is_installed('resources'):
run("python manage.py syncresources")
if options.get('restart'):
run("python manage.py restartservices")
if not version:
self.stderr.write(
'\nNext time you migth want to provide a --from argument '
'in order to run version specific upgrade operations\n')
return
# Post-upgrade operations (version specific)
if version <= 629:
# Clean existing sessions because of change on auth backend
run('echo "delete from django_session;" | python manage.py dbshell'
)
if version < 801:
deprecate_periodic_tasks(('state.ping', ))
if version < 809:
# Add PKI directories
from pki import ca
from controller.utils.paths import get_site_root
site_root = get_site_root()
username = run("stat -c %%U %s" % site_root)
get_dir = lambda f: os.path.dirname(getattr(ca, f + '_path'))
for d in set(get_dir(f) for f in ['priv_key', 'pub_key', 'cert']):
run('mkdir -p %s' % d)
run('chown %s %s' % (username, d))
upgrade_notes.append(
'HTTPS certificate support for the management '
'network has been introduced in version 0.8.9.\n'
'In order to use it you sould run:\n'
' > python manage.py setuppki\n'
' > sudo python manage.py setupapache\n')
if version < 838:
# Purge communitynetworks.periodic_cache_node_db
from djcelery.models import PeriodicTask
PeriodicTask.objects.filter(
name='communitynetworks.periodic_cache_node_db').delete()
run('rabbitmqctl stop_app')
run('rabbitmqctl reset')
run('rabbitmqctl start_app')
run('service celeryd restart')
upgrade_notes.append(
'New Celeryd init.d configuration has been '
'introduced in 0.8.38.\nIt is strongly recommended to upgrade by\n'
' > sudo python manage.py setupceleryd\n')
# Deprecate x86 and amd64 architectures
from nodes.models import Node
Node.objects.filter(arch='x86').update(arch='i686')
Node.objects.filter(arch='amd64').update(arch='x86_64')
upgrade_notes.append(
'In order to support Base authentication while downloading '
'firmwares you should add "WSGIPassAuthorization On" on your apache config.\n'
'Alternatively you can perform this operation with the following command\n'
' > sudo python manage.py setupapache\n'
' > /etc/init.d/apache2 reload\n')
if version < 900:
upgrade_notes.append(
'Apache configuration is now placed under '
'/etc/apache2/conf.d/<project_name>.conf. It is convenient for you '
'to migrate your current configuration located on /etc/apache2/httpd.conf '
'to this new location.\n')
upgrade_notes.append(
'Celery workers configuration has been updated. '
'Please update it by running:\n'
' > sudo python manage.py setupceleryd\n')
if version < 905:
# TODO find the root cause of this
# maybe is shit imported on settings that import settings like add_app
# Prevent crazy import erros to appear :S
from django.utils import translation
translation.activate('en-us')
# Change template types for more generic ones
from slices.models import Template
from slices.settings import SLICES_TEMPLATE_TYPES
template_types = [t[0] for t in SLICES_TEMPLATE_TYPES]
if 'debian' in template_types:
Template.objects.filter(type='debian6').update(type='debian')
if 'openwrt' in template_types:
Template.objects.filter(type='openwrt-backfire').update(
type='openwrt')
if version < 906:
deprecate_periodic_tasks(('state.nodestate', 'state.sliverstate'))
if version <= 907:
# Generate sha256
from slices.models import Template
for template in Template.objects.all():
template.save()
upgrade_notes.append(
"It is extremly recommended to update your database "
"settings to enable atomic request behaviour:\n"
" https://docs.djangoproject.com/en/dev/topics/db/transactions/#tying-transactions-to-http-requests\n"
"Just add:\n"
" 'ATOMIC_REQUESTS': True,\n"
"into DATABASES setting within <project_name>/<project_name>/settings.py"
)
if version <= 1003:
# Update firmware configuration after Island refactor (#264)
from firmware.models import ConfigFile
try:
cfg_file = ConfigFile.objects.get(
path__contains="node.tinc.connect_to")
except (ConfigFile.DoesNotExist,
ConfigFile.MultipleObjectsReturned):
# Warn the user that needs to perform manual update
msg = "Firmware configuration update has failed. "
else:
cfg_file.content = cfg_file.content.replace(
"node.tinc.island", "node.island")
cfg_file.save()
msg = "Firmware configuration updated successfully. Updated ConfigFile ID: %i." % cfg_file.pk
upgrade_notes.append(
"%s\nPlease check version 0.10.4 release notes:\n"
"https://wiki.confine-project.eu/soft:server-release-notes#section0104"
% msg)
if version < 1103:
# Update mgmt network Server APIs certificate
# perform raw SQL querie because models hasn't been
# reloaded yet and cannot access e.g. server.api
from django.db import connection
from nodes.models import Server
from pki import ca
server_id = Server.objects.order_by('id').first().pk
try:
cert = ca.get_cert().as_pem()
except IOError:
msg = ("Failed to update Server APIs certificate. Missing "
"server certificate '%s'.\n"
"Run 'python manage.py setuppki --help'" % ca.cert_path)
upgrade_notes.append(msg)
else:
update_sql = ('UPDATE "nodes_serverapi" SET cert = %s '
'WHERE "nodes_serverapi"."server_id" = %s')
cursor = connection.cursor()
cursor.execute(update_sql, [cert, server_id])
del cursor
upgrade_notes.append("Updated Server APIs certificate.")
if upgrade_notes and options.get('print_upgrade_notes'):
self.stdout.write('\n\033[1m\n'
' ===================\n'
' ** UPGRADE NOTES **\n'
' ===================\n\n' +
'\n'.join(upgrade_notes) + '\033[m\n')