def allocate_virtual_compute_from_hangar(model, event):
if not IVirtualizationContainer.providedBy(model.__parent__):
return
if IDeployed.providedBy(model):
return
auto_allocate = get_config().getboolean('vms', 'auto_allocate', True)
if not auto_allocate:
return
if IHangar.providedBy(model.__parent__.__parent__):
action = AllocateAction
msg = 'Allocated compute %s'
elif ICompute.providedBy(model.__parent__.__parent__):
action = DeployAction
msg = 'Deployed compute %s'
else:
return
try:
path = canonical_path(model)
owner = model.__owner__
ul = UserLogger(subject=model, owner=owner)
log.msg('Attempting %s for %s (%s, %s)' % (action.__name__, model, path, owner),
system='create-event')
d = task.deferLater(reactor, 2.0, virtual_compute_action, action, path, event)
d.addCallback(lambda r: ul.log(msg % path))
if not get_config().getboolean('stats', 'only_report_on_sync', True):
d.addCallback(lambda r: defer.maybeDeferred(getUtility(IUserStatisticsProvider).update, owner))
d.addErrback(log.err)
except Exception:
log.err(system='create-event')
def auto_tags(self):
res = [u'state:' + self.context.state] if self.context.state else []
if self.context.architecture:
for i in self.context.architecture:
res.append(u'arch:' + i)
from opennode.knot.model.virtualizationcontainer import IVirtualizationContainer
p = sudo(self.context)
if (IVirtualCompute.providedBy(p) and
IVirtualizationContainer.providedBy(p.__parent__)):
res.append(u'virt_type:' + p.__parent__.backend)
res.append(u'virt:yes')
else:
res.append(u'virt:no')
config = get_config()
if config.has_section('netenv-tags'):
for tag, nets in config.items('netenv-tags'):
try:
if (self.context.ipv4_address is not None and
len(netaddr.all_matching_cidrs(self.context.ipv4_address.split('/')[0],
nets.split(','))) > 0):
res.append(u'env:' + tag)
except ValueError:
# graceful ignoring of incorrect ips
pass
return res
def auto_tags(self):
res = [u'state:' + self.context.state] if self.context.state else []
if self.context.architecture:
for i in self.context.architecture:
res.append(u'arch:' + i)
from opennode.knot.model.virtualizationcontainer import IVirtualizationContainer
p = sudo(self.context)
if (IVirtualCompute.providedBy(p) and
IVirtualizationContainer.providedBy(p.__parent__)):
res.append(u'virt_type:' + p.__parent__.backend)
res.append(u'virt:yes')
else:
res.append(u'virt:no')
config = get_config()
if config.has_section('netenv-tags'):
for tag, nets in config.items('netenv-tags'):
try:
if (self.context.ipv4_address is not None and
len(netaddr.all_matching_cidrs(self.context.ipv4_address.split('/')[0],
nets.split(','))) > 0):
res.append(u'env:' + tag)
except ValueError:
# graceful ignoring of incorrect ips
pass
return res
def sync_vms(self):
for vms in self.context.listcontent():
try:
if not IVirtualizationContainer.providedBy(vms):
continue
yield SyncVmsAction(vms)._execute(DetachedProtocol(), object())
except Exception:
log.err(system='sync-action')
def allowed_classes_gen(item):
from opennode.knot.model.machines import Machines
from opennode.knot.model.virtualizationcontainer import IVirtualizationContainer
yield isinstance(item, Machines)
yield isinstance(item, Computes)
yield ICompute.providedBy(item)
yield IVirtualizationContainer.providedBy(item)
yield IHangar.providedBy(item)
def sync_vms(self):
for vms in self.context.listcontent():
try:
if not IVirtualizationContainer.providedBy(vms):
continue
yield SyncVmsAction(vms)._execute(DetachedProtocol(), object())
except Exception:
log.err(system='sync-action')
def allowed_classes_gen(item):
from opennode.knot.model.compute import ICompute, IVirtualCompute
from opennode.knot.model.machines import Machines
from opennode.knot.model.virtualizationcontainer import IVirtualizationContainer
yield isinstance(item, Machines)
yield isinstance(item, Templates)
yield IVirtualizationContainer.providedBy(item)
yield ICompute.providedBy(item)
yield IVirtualCompute.providedBy(item)
def iterate_recursively(container):
seen = set()
for item in container.listcontent():
if ICompute.providedBy(item):
setattr(item, status_name, bool(status))
if (IVirtualizationContainer.providedBy(item) or ICompute.providedBy(item)):
if item.__name__ not in seen:
seen.add(item.__name__)
iterate_recursively(item)
def create_default_console(self, default):
if not default or not self.context.consoles[default]:
if (IVirtualizationContainer.providedBy(self.context.__parent__)
and self.context.__parent__.backend == 'openvz'
and self.context.consoles['tty0']):
default = 'tty0'
else:
default = 'ssh'
self.context.consoles.add(Symlink('default', self.context.consoles[default]))
def create_default_console(self, default):
if not default or not self.context.consoles[default]:
if (IVirtualizationContainer.providedBy(self.context.__parent__)
and self.context.__parent__.backend == 'openvz'
and self.context.consoles['tty0']):
default = 'tty0'
else:
default = 'ssh'
self.context.consoles.add(
Symlink('default', self.context.consoles[default]))
def iterate_recursively(container):
seen = set()
for item in container.listcontent():
if ICompute.providedBy(item):
setattr(item, status_name, bool(status))
if (IVirtualizationContainer.providedBy(item)
or ICompute.providedBy(item)):
if item.__name__ not in seen:
seen.add(item.__name__)
iterate_recursively(item)
def ensure_hangar_v12ncontainers():
machines = db.get_root()['oms_root']['machines']
backends = set()
for machine in machines.listcontent():
for o in machine.listcontent():
if IVirtualizationContainer.providedBy(o):
backends.add(o.backend)
hangar = machines['hangar']
for backend in map(unicode, backends):
if backend not in map(lambda vms: vms.backend, hangar.listcontent()):
vms = VirtualizationContainer(backend)
hangar.add(vms)
def ensure_hangar_v12ncontainers():
machines = db.get_root()['oms_root']['machines']
backends = set()
for machine in machines.listcontent():
for o in machine.listcontent():
if IVirtualizationContainer.providedBy(o):
backends.add(o.backend)
hangar = machines['hangar']
for backend in map(unicode, backends):
if backend not in map(lambda vms: vms.backend,
hangar.listcontent()):
vms = VirtualizationContainer(backend)
hangar.add(vms)
def allocate_virtual_compute_from_hangar(model, event):
if not IVirtualizationContainer.providedBy(model.__parent__):
return
if IDeployed.providedBy(model):
return
auto_allocate = get_config().getboolean('vms', 'auto_allocate', True)
if not auto_allocate:
return
if IHangar.providedBy(model.__parent__.__parent__):
action = AllocateAction
msg = 'Allocated compute %s'
elif ICompute.providedBy(model.__parent__.__parent__):
action = DeployAction
msg = 'Deployed compute %s'
else:
return
try:
path = canonical_path(model)
owner = model.__owner__
ul = UserLogger(subject=model, owner=owner)
log.msg('Attempting %s for %s (%s, %s)' %
(action.__name__, model, path, owner),
system='create-event')
d = task.deferLater(reactor, 2.0, virtual_compute_action, action, path,
event)
d.addCallback(lambda r: ul.log(msg % path))
if not get_config().getboolean('stats', 'only_report_on_sync', True):
d.addCallback(lambda r: defer.maybeDeferred(
getUtility(IUserStatisticsProvider).update, owner))
d.addErrback(log.err)
except Exception:
log.err(system='create-event')
def _execute(self, cmd, args):
template = yield db.get(self.context, 'template')
if not template:
self._action_log(
cmd,
'Cannot deploy %s (%s) because no template was specified' %
(self.context.hostname, self.context),
system='deploy',
logLevel=ERROR)
return
if (yield db.ro_transact(IDeployed.providedBy)(self.context)):
log.msg('Attempt to deploy a deployed compute: %s' %
(self.context),
system='deploy')
return
@db.transact
def allocate_ip_address():
ippools = db.get_root()['oms_root']['ippools']
ip = ippools.allocate()
if ip is not None:
self._action_log(cmd,
'Allocated IP: %s for %s' %
(ip, self.context),
system='deploy')
ulog = UserLogger(principal=cmd.protocol.interaction.
participations[0].principal,
subject=self.context,
owner=self.context.__owner__)
ulog.log('Allocated IP for %s: %s' % (self.context, ip))
return ip
else:
raise Exception(
'Could not allocate IP for the new compute: pools exhausted or undefined'
)
@db.transact
def cleanup_root_password():
if getattr(self.context, 'root_password', None) is not None:
self.context.root_password = None
@db.transact
def adjust_cpulimit():
"""Set cpulimit to a configured percentage * cores"""
cores = getattr(self.context, 'num_cores', 1)
cpu_limit_factor = get_config().getfloat('vms', 'cpu_limit', 80)
cpu_limit = cores * cpu_limit_factor / 100.0
log.msg("Updating cpulimit to %s" % cpu_limit, system='deploy')
self.context.cpu_limit = cpu_limit
target = (args if IVirtualizationContainer.providedBy(args) else
(yield db.get(self.context, '__parent__')))
try:
yield db.transact(alsoProvides)(self.context, IDeploying)
vm_parameters = yield self.get_parameters()
ipaddr = netaddr.IPAddress(vm_parameters['ip_address'])
if vm_parameters['ip_address'] in (None, u'0.0.0.0/32', u'0.0.0.0',
'0.0.0.0/32', '0.0.0.0'):
ipaddr = yield allocate_ip_address()
vm_parameters.update({'ip_address': str(ipaddr)})
utils = getAllUtilitiesRegisteredFor(IPreDeployHook)
for util in utils:
yield defer.maybeDeferred(util.execute, self.context, cmd,
vm_parameters)
log.msg('Deploying %s to %s: issuing agent command' %
(self.context, target),
system='deploy')
res = yield IVirtualizationContainerSubmitter(target).submit(
IDeployVM, vm_parameters)
yield cleanup_root_password()
yield adjust_cpulimit()
name = yield db.get(self.context, '__name__')
hostname = yield db.get(self.context, 'hostname')
owner = yield db.get(self.context, '__owner__')
owner_obj = getUtility(IAuthentication).getPrincipal(owner)
log.msg('Checking post-deploy...', system='deploy')
@db.transact
def set_notify_admin():
if self.context.notify_admin:
self.context.license_activated = False
admin_logger.warning(
'%s (hostname=%s; owner=%s; targethost=%s(%s); ipaddr=%s) '
'requires activation!', self.context,
self.context.hostname, self.context.__owner__,
target.__parent__, target.__parent__.hostname,
vm_parameters['ip_address'])
yield set_notify_admin()
if not (yield self._check_vm_post(cmd, name, hostname, target)):
self._action_log(cmd,
'Deployment failed. Request result: %s' % res,
system='deploy')
return
@db.transact
def add_deployed_model_remove_from_hangar(c, target):
path = canonical_path(target)
target = traverse1(path)
cpath = canonical_path(c)
c = traverse1(cpath)
if c is None:
raise Exception('Compute not found: "%s"' % cpath)
new_compute = Compute(unicode(hostname), u'inactive')
new_compute.__name__ = name
new_compute.__owner__ = owner_obj
new_compute.template = unicode(template)
new_compute._ipv4_address = unicode(ipaddr)
new_compute.mac_address = getattr(c, 'mac_address', None)
new_compute.memory = getattr(c, 'memory', 0)
new_compute.diskspace = getattr(c, 'diskspace', {u'total': 0})
new_compute.num_cores = getattr(c, 'num_cores', 0)
new_compute.license_activated = getattr(
c, 'license_activated', True)
alsoProvides(new_compute, IVirtualCompute)
alsoProvides(new_compute, IDeployed)
noLongerProvides(new_compute, IManageable)
target.add(new_compute)
container = c.__parent__
del container[name]
timestamp = int(time.time() * 1000)
IStream(new_compute).add((timestamp, {
'event': 'change',
'name': 'features',
'value': new_compute.features,
'old_value': self.context.features
}))
IStream(new_compute).add((timestamp, {
'event':
'change',
'name':
'ipv4_address',
'value':
new_compute._ipv4_address,
'old_value':
self.context._ipv4_address
}))
yield add_deployed_model_remove_from_hangar(self.context, target)
self._action_log(
cmd,
'Deployment of "%s"(%s) is finished' %
(vm_parameters['hostname'], self.context.__name__),
system='deploy')
auto_allocate = get_config().getboolean('vms', 'auto_allocate',
True)
if not auto_allocate and not get_config().getboolean(
'stats', 'only_report_on_sync', True):
yield defer.maybeDeferred(
getUtility(IUserStatisticsProvider).update, owner)
except Exception as e:
log.err(system='deploy')
@db.transact
def cleanup_deploying():
noLongerProvides(self.context, IDeploying)
yield cleanup_deploying()
raise e
def find_compute_v12n_container(compute, backend):
for v12nc in compute:
if IVirtualizationContainer.providedBy(
v12nc) and v12nc.backend == backend:
return v12nc
def sync_vms(self):
for vms in self.context.listcontent():
if not IVirtualizationContainer.providedBy(vms):
continue
yield SyncVmsAction(vms)._execute(DetachedProtocol(), object())
def _execute(self, cmd, args):
if not any_stack_installed(self.context):
return
@db.transact
def update_templates(container, templates):
if not container['templates']:
container.add(Templates())
template_container = container['templates']
for template in templates:
name = template['template_name']
if not template_container['by-name'][name]:
template_container.add(Template(unicode(name), get_u(template, 'domain_type')))
t = template_container['by-name'][name].target
t.cores = (get_i(template, 'vcpu_min'),
get_i(template, 'vcpu'),
max(-1, get_i(template, 'vcpu_max')))
t.memory = (get_f(template, 'memory_min'),
get_f(template, 'memory'),
max(-1.0, get_f(template, 'memory_max')))
t.swap = (get_f(template, 'swap_min'),
get_f(template, 'swap'),
max(-1.0, get_f(template, 'swap_max')))
t.disk = (get_f(template, 'disk_min'),
get_f(template, 'disk'),
max(-1.0, get_f(template, 'disk_max')))
t.nameserver = get_u(template, 'nameserver')
t.username = get_u(template, 'username')
t.password = get_u(template, 'passwd')
t.cpu_limit = (get_i(template, 'vcpulimit_min'),
get_i(template, 'vcpulimit'))
t.ip = get_u(template, 'ip_address')
# delete templates no more offered upstream
template_names = template_container['by-name'].listnames()
vanished_template_names = set(template_names).difference(
template['template_name'] for template in templates)
for template in vanished_template_names:
template_container.remove(follow_symlinks(template_container['by-name'][template]))
for container in self.context.listcontent():
if not IVirtualizationContainer.providedBy(container):
continue
submitter = IVirtualizationContainerSubmitter(container)
templates = yield submitter.submit(IGetLocalTemplates)
if not templates:
log.msg('Did not find any templates on %s/%s' % (self.context, container),
system='sync-templates')
continue
log.msg('Synced templates on %s (%s). Updating %s templates' %
(self.context, container, len(templates)), system='sync-templates')
yield update_templates(container, templates)
def _execute(self, cmd, args):
template = yield db.get(self.context, 'template')
if not template:
self._action_log(cmd, 'Cannot deploy %s (%s) because no template was specified' %
(self.context.hostname, self.context), system='deploy', logLevel=ERROR)
return
if (yield db.ro_transact(IDeployed.providedBy)(self.context)):
log.msg('Attempt to deploy a deployed compute: %s' % (self.context), system='deploy')
return
@db.transact
def allocate_ip_address():
ippools = db.get_root()['oms_root']['ippools']
ip = ippools.allocate()
if ip is not None:
self._action_log(cmd, 'Allocated IP: %s for %s' % (ip, self.context), system='deploy')
ulog = UserLogger(principal=cmd.protocol.interaction.participations[0].principal,
subject=self.context, owner=self.context.__owner__)
ulog.log('Allocated IP for %s: %s' % (self.context, ip))
return ip
else:
raise Exception('Could not allocate IP for the new compute: pools exhausted or undefined')
@db.transact
def cleanup_root_password():
if getattr(self.context, 'root_password', None) is not None:
self.context.root_password = None
target = (args if IVirtualizationContainer.providedBy(args)
else (yield db.get(self.context, '__parent__')))
try:
yield db.transact(alsoProvides)(self.context, IDeploying)
vm_parameters = yield self.get_parameters()
ipaddr = netaddr.IPAddress(vm_parameters['ip_address'])
if vm_parameters['ip_address'] in (None, u'0.0.0.0/32', u'0.0.0.0', '0.0.0.0/32', '0.0.0.0'):
ipaddr = yield allocate_ip_address()
vm_parameters.update({'ip_address': str(ipaddr)})
utils = getAllUtilitiesRegisteredFor(IPreDeployHook)
for util in utils:
yield defer.maybeDeferred(util.execute, self.context, cmd, vm_parameters)
log.msg('Deploying %s to %s: issuing agent command' % (self.context, target), system='deploy')
res = yield IVirtualizationContainerSubmitter(target).submit(IDeployVM, vm_parameters)
yield cleanup_root_password()
name = yield db.get(self.context, '__name__')
hostname = yield db.get(self.context, 'hostname')
owner = yield db.get(self.context, '__owner__')
owner_obj = getUtility(IAuthentication).getPrincipal(owner)
log.msg('Checking post-deploy...', system='deploy')
if not (yield self._check_vm_post(cmd, name, hostname, target)):
self._action_log(cmd, 'Deployment failed. Deployment request result: %s' % res,
system='deploy')
return
@db.transact
def add_deployed_model_remove_from_hangar(c, target):
path = canonical_path(target)
target = traverse1(path)
cpath = canonical_path(c)
c = traverse1(cpath)
if c is None:
raise Exception('Compute not found: "%s"' % cpath)
new_compute = Compute(unicode(hostname), u'inactive')
new_compute.__name__ = name
new_compute.__owner__ = owner_obj
new_compute.template = unicode(template)
new_compute._ipv4_address = unicode(ipaddr)
new_compute.mac_address = getattr(c, 'mac_address', None)
new_compute.memory = getattr(c, 'memory', 0)
new_compute.diskspace = getattr(c, 'diskspace', {u'total': 0})
new_compute.num_cores = getattr(c, 'num_cores', 0)
alsoProvides(new_compute, IVirtualCompute)
alsoProvides(new_compute, IDeployed)
noLongerProvides(new_compute, IManageable)
target.add(new_compute)
container = c.__parent__
del container[name]
timestamp = int(time.time() * 1000)
IStream(new_compute).add((timestamp, {'event': 'change',
'name': 'features',
'value': new_compute.features,
'old_value': self.context.features}))
IStream(new_compute).add((timestamp, {'event': 'change',
'name': 'ipv4_address',
'value': new_compute._ipv4_address,
'old_value': self.context._ipv4_address}))
yield add_deployed_model_remove_from_hangar(self.context, target)
self._action_log(cmd, 'Deployment of "%s"(%s) is finished'
% (vm_parameters['hostname'], self.context.__name__), system='deploy')
auto_allocate = get_config().getboolean('vms', 'auto_allocate', True)
if not auto_allocate and not get_config().getboolean('stats', 'only_report_on_sync', True):
yield defer.maybeDeferred(getUtility(IUserStatisticsProvider).update, owner)
except Exception as e:
log.err(system='deploy')
@db.transact
def cleanup_deploying():
noLongerProvides(self.context, IDeploying)
yield cleanup_deploying()
raise e
def get_destination():
return (args.__parent__ if IVirtualizationContainer.providedBy(args)
else cmd.traverse(args.dest_path))
def _execute(self, cmd, args):
if not any_stack_installed(self.context):
return
@db.transact
def update_templates(container, templates):
if not container['templates']:
container.add(Templates())
template_container = container['templates']
for template in templates:
name = template['template_name']
if not template_container['by-name'][name]:
template_container.add(
Template(unicode(name), get_u(template,
'domain_type')))
t = template_container['by-name'][name].target
t.cores = (get_i(template,
'vcpu_min'), get_i(template, 'vcpu'),
max(-1, get_i(template, 'vcpu_max')))
t.memory = (get_f(template,
'memory_min'), get_f(template, 'memory'),
max(-1.0, get_f(template, 'memory_max')))
t.swap = (get_f(template, 'swap_min'), get_f(template, 'swap'),
max(-1.0, get_f(template, 'swap_max')))
t.disk = (get_f(template, 'disk_min'), get_f(template, 'disk'),
max(-1.0, get_f(template, 'disk_max')))
t.nameserver = get_u(template, 'nameserver')
t.username = get_u(template, 'username')
t.password = get_u(template, 'passwd')
t.cpu_limit = (get_i(template, 'vcpulimit_min'),
get_i(template, 'vcpulimit'))
t.ip = get_u(template, 'ip_address')
# delete templates no more offered upstream
template_names = template_container['by-name'].listnames()
vanished_template_names = set(template_names).difference(
template['template_name'] for template in templates)
for template in vanished_template_names:
template_container.remove(
follow_symlinks(template_container['by-name'][template]))
for container in self.context.listcontent():
if not IVirtualizationContainer.providedBy(container):
continue
submitter = IVirtualizationContainerSubmitter(container)
templates = yield submitter.submit(IGetLocalTemplates)
if not templates:
log.msg('Did not find any templates on %s/%s' %
(self.context, container),
system='sync-templates')
continue
log.msg('Synced templates on %s (%s). Updating %s templates' %
(self.context, container, len(templates)),
system='sync-templates')
yield update_templates(container, templates)
def get_destination():
return (args.__parent__
if IVirtualizationContainer.providedBy(args) else
cmd.traverse(args.dest_path))
def find_compute_v12n_container(compute, backend):
for v12nc in compute:
if IVirtualizationContainer.providedBy(v12nc) and v12nc.backend == backend:
return v12nc
