def test_admin_backend_create_edit_delete(self):
self.login_as_user()
url = reverse('admin_backend_create')
resp = self.client.post(url, {'is_enabled': False, 'name': 'backend1',
'ip': '8.8.8.8', 'cpu_cores': '2',
'memory_mb': '512'})
self.assertEqual(resp.status_code, 302)
backend = BackendServer.objects(name='backend1').first()
self.assertNotEqual(backend, None)
self.assertEqual(backend.name, 'backend1')
self.assertEqual(backend.is_enabled, False)
self.assertEqual(backend.ip.strNormal(), '8.8.8.8')
self.assertEqual(backend.cpu_cores, 2)
self.assertEqual(backend.memory_mb, 512)
url = reverse('admin_backend_edit', args=[backend.name])
resp = self.client.post(url, {'is_enabled': True, 'name': 'backend2',
'ip': '7.7.7.7', 'cpu_cores': '8',
'memory_mb': '128'})
self.assertEqual(resp.status_code, 302)
backend.reload()
self.assertEqual(backend.name, 'backend2')
self.assertEqual(backend.is_enabled, True)
self.assertEqual(backend.ip.strNormal(), '7.7.7.7')
self.assertEqual(backend.cpu_cores, 8)
self.assertEqual(backend.memory_mb, 128)
backend.delete()
self.assertEqual(BackendServer.objects().first(), None)
def register_backend(self):
name = gethostname()
local_ip = None
backend = BackendServer.objects(name=name).first()
if not backend:
for local_ip in local_ipv4_addresses():
backend = BackendServer.objects(ip=local_ip).first()
if backend:
break
if not backend and not local_ip:
log.error(_("No IP address found for local backend!"))
return
if backend:
if backend.autodetect:
self.update_backend(backend)
else:
log.info(_("Local backend not found, registering as '{name}' "
"with IP {ip}").format(name=name, ip=local_ip))
backend = BackendServer(name=name, ip=local_ip, is_enabled=False,
cpu_cores=cpu_count(),
memory_mb=backend_total_memory())
backend.save()
return backend
def clean_failed_remote_tasks(self, local_backend):
name = 'remote_tasks'
if not self.can_clean(name):
return
# look for tasks locked at backends that did not ack itself to the
# database for at least 600 seconds
timestamp = datetime.now() - timedelta(seconds=600)
backends = BackendServer.objects(**{
'id__ne': local_backend.id,
'is_enabled__ne': False,
'worker_ping__%s__lte' % self.name: timestamp
})
if backends:
log.debug(_("{len} non responsive backends: {names}").format(
len=len(backends), names=[b.name for b in backends]))
for task in Task.objects(backend__in=backends,
status=TaskStatus.running,
date_created__lte=timestamp):
log.warning(_("Task '{name}' with id {tid} is locked on "
"backend {backend}, but it didn't send any "
"pings for 10 minutes, marking as "
"failed").format(
name=task.title, tid=task.safe_id,
backend=task.backend.name))
task.update(set__status=TaskStatus.failed,
set__date_finished=datetime.now())
self.last_clean[name] = datetime.now()
def create_backend(request):
name = gethostname()
backend = BackendServer.objects(name=name).first()
if backend:
backend.delete()
backend = BackendServer(name=name, ip='127.0.0.1', memory_mb=512,
cpu_cores=1)
backend.save()
def cleanup():
backend.delete()
request.addfinalizer(cleanup)
request.instance.backend = backend
request.instance.backend_name = name
def create_backend_list(request):
backends = []
for i in range(0, 10):
name = 'backend%d' % i
backend = BackendServer.objects(name=name).first()
if backend:
backend.delete()
backend = BackendServer(name=name, ip='127.0.0.%d' % i, memory_mb=512,
cpu_cores=1)
backend.save()
backends.append(backend)
def cleanup():
for backend in backends:
backend.delete()
request.addfinalizer(cleanup)
request.instance.backend_list = backends
def __init__(self):
self.backends = BackendServer.objects(is_enabled=True)
self.backend_by_id = dict((b.safe_id, b) for b in self.backends)
self.backends_count = len(self.backends)
self.default_worker_memory = \
settings.UPAAS_CONFIG.defaults.limits.memory_per_worker
self.allocated_mem = {}
self.allocated_cpu = {}
self.mem_load = {}
self.cpu_load = {}
self.scores = {}
def clean_failed_remote_locks(self, local_backend):
name = 'remote_tasks'
if not self.can_clean(name):
return
# look for tasks locked at backends that did not ack itself to the
# database for at least 600 seconds
timestamp = datetime.now() - timedelta(seconds=600)
backends = BackendServer.objects(**{
'id__ne': local_backend.id,
'is_enabled__ne': False,
'worker_ping__%s__lte' % self.name: timestamp
})
if backends:
log.debug(_("{len} non responsive backends: {names}").format(
len=len(backends), names=[b.name for b in backends]))
for lock in FlagLock.objects(backend__in=backends,
date_created__lte=timestamp):
log.warning(_("Found old lock on backend {backend}, but it "
"didn't send any pings for 10 minutes, "
"removing").format(backend=lock.backend))
self.reset_pending_state(lock)
lock.delete()
self.last_clean[name] = datetime.now()
def cleanup():
BackendServer.objects().delete()
def migrate_backends(self):
for backend in BackendServer.objects():
backend.update(unset__worker_ping=True)