class Vmmaster(Flask):
def __init__(self, *args, **kwargs):
from core.db import Database
from core.sessions import Sessions
from vmpool.virtual_machines_pool import VirtualMachinesPool
super(Vmmaster, self).__init__(*args, **kwargs)
self.running = True
self.uuid = str(uuid1())
self.database = Database()
self.pool = VirtualMachinesPool(self)
self.sessions = Sessions(self)
self.json_encoder = JSONEncoder
self.register()
def register(self):
self.database.register_platforms(self.uuid, self.pool.platforms.info())
def unregister(self):
self.database.unregister_platforms(self.uuid)
def cleanup(self):
log.info("Shutting down...")
self.pool.stop_workers()
self.sessions.worker.stop()
self.pool.free()
self.unregister()
self.pool.platforms.cleanup()
log.info("Server gracefully shut down.")
def setUp(self):
setup_config('data/config.py')
self.platform = "test_origin_1"
with patch('core.connection.Virsh',
Mock()), patch('core.network.Network', Mock()):
from vmpool.virtual_machines_pool import VirtualMachinesPool
self.pool = VirtualMachinesPool()
def __init__(self, *args, **kwargs):
from core.db import Database
from core.sessions import Sessions
from vmpool.virtual_machines_pool import VirtualMachinesPool
super(Provider, self).__init__(*args, **kwargs)
self.running = True
self.json_encoder = JSONEncoder
self.database = Database()
self.sessions = Sessions(self.database, self.app_context)
self.pool = VirtualMachinesPool(app=self, name=config.PROVIDER_NAME)
self.pool.start_workers()
def setUp(self):
setup_config('data/config.py')
self.platform = "test_origin_1"
self.app = Flask(__name__)
self.ctx = self.app.app_context()
self.ctx.push()
with patch('core.connection.Virsh',
Mock()), patch('core.network.Network', Mock()):
from vmpool.virtual_machines_pool import VirtualMachinesPool
self.pool = VirtualMachinesPool(self.app)
def __init__(self, *args, **kwargs):
from core.db import Database
from core.sessions import Sessions
from vmpool.virtual_machines_pool import VirtualMachinesPool
super(Vmmaster, self).__init__(*args, **kwargs)
self.running = True
self.uuid = str(uuid1())
self.database = Database()
self.pool = VirtualMachinesPool(self)
self.sessions = Sessions(self)
self.json_encoder = JSONEncoder
self.register()
def setUp(self):
from core.config import setup_config
setup_config('data/config_openstack.py')
with patch('core.utils.init.home_dir',
Mock(return_value=fake_home_dir())), patch(
'core.db.Database', DatabaseMock()), patch(
'vmpool.virtual_machines_pool.VirtualMachinesPool',
Mock()):
from vmpool.server import create_app
from vmpool.virtual_machines_pool import VirtualMachinesPool
self.app = create_app()
self.app.pool = VirtualMachinesPool(
self.app,
'unnamed',
platforms_class=Mock(),
preloader_class=Mock(),
artifact_collector_class=Mock(),
endpoint_preparer_class=Mock(),
endpoint_remover_class=Mock())
self.vmmaster_client = self.app.test_client()
self.platform = 'origin_1'
self.ctx = self.app.app_context()
self.ctx.push()
def setUp(self):
setup_config('data/config_openstack.py')
self.platform = "origin_1"
mocked_image = Mock(id=1,
status='active',
get=Mock(return_value='snapshot'),
min_disk=20,
min_ram=2,
instance_type_flavorid=1)
type(mocked_image).name = PropertyMock(return_value='test_origin_1')
with patch('core.network.Network', Mock()), patch(
'core.connection.Virsh', Mock()), patch.multiple(
'core.utils.openstack_utils',
nova_client=Mock(),
neutron_client=Mock(return_value=Mock()),
glance_client=Mock(return_value=Mock())), patch(
'vmpool.platforms.OpenstackPlatforms.images',
Mock(return_value=[mocked_image])):
from flask import Flask
self.app = Flask(__name__)
from vmpool.virtual_machines_pool import VirtualMachinesPool
self.app.pool = VirtualMachinesPool(self.app)
self.ctx = self.app.test_request_context()
self.ctx.push()
def setUp(self):
from core.db import Database
setup_config('data/config.py')
self.platform_name = "origin_1"
self.app = Flask(__name__)
self.ctx = self.app.app_context()
self.ctx.push()
self.app.database = Database()
self.app.sessions = Mock()
self.mocked_image = Mock(
id=1,
status='active',
get=Mock(return_value='snapshot'),
min_disk=20,
min_ram=2,
instance_type_flavorid=1,
)
type(self.mocked_image).name = PropertyMock(
return_value='test_origin_1')
with patch.multiple(
'vmpool.platforms.OpenstackPlatforms',
images=Mock(return_value=[self.mocked_image]),
flavor_params=Mock(return_value={
'vcpus': 1,
'ram': 2
}),
limits=Mock(
return_value={
'maxTotalCores': 10,
'maxTotalInstances': 10,
'maxTotalRAMSize': 100,
'totalCoresUsed': 0,
'totalInstancesUsed': 0,
'totalRAMUsed': 0
}),
):
from vmpool.virtual_machines_pool import VirtualMachinesPool
self.pool = VirtualMachinesPool(self.app,
preloader_class=Mock(),
artifact_collector_class=Mock(),
endpoint_preparer_class=Mock())
self.ctx = self.app.app_context()
self.ctx.push()
self.pool.register()
def setUp(self):
setup_config('data/config_openstack.py')
self.platform = "origin_1"
self.address = ("localhost", 9001)
self.mocked_image = Mock(id=1,
status='active',
get=Mock(return_value='snapshot'),
min_disk=20,
min_ram=2,
instance_type_flavorid=1)
type(self.mocked_image).name = PropertyMock(
return_value='test_origin_1')
with patch(
'core.connection.Virsh',
Mock(),
), patch('core.network.Network', Mock()), patch.multiple(
'core.utils.openstack_utils',
neutron_client=Mock(return_value=Mock()),
nova_client=Mock(return_value=Mock()),
glance_client=Mock(return_value=Mock())), patch.multiple(
'vmpool.platforms.OpenstackPlatforms',
images=Mock(return_value=[self.mocked_image]),
flavor_params=Mock(return_value={
'vcpus': 1,
'ram': 2
}),
limits=Mock(
return_value={
'maxTotalCores': 10,
'maxTotalInstances': 10,
'maxTotalRAMSize': 100,
'totalCoresUsed': 0,
'totalInstancesUsed': 0,
'totalRAMUsed': 0
})):
from flask import Flask
self.app = Flask(__name__)
from vmpool.virtual_machines_pool import VirtualMachinesPool
self.app.pool = VirtualMachinesPool(self.app)
from vmpool.platforms import Platforms
self.app.platforms = Platforms()
self.ctx = self.app.test_request_context()
self.ctx.push()
class Provider(Flask):
def __init__(self, *args, **kwargs):
from core.db import Database
from core.sessions import Sessions
from vmpool.virtual_machines_pool import VirtualMachinesPool
super(Provider, self).__init__(*args, **kwargs)
self.running = True
self.json_encoder = JSONEncoder
self.database = Database()
self.sessions = Sessions(self.database, self.app_context)
self.pool = VirtualMachinesPool(app=self, name=config.PROVIDER_NAME)
self.pool.start_workers()
def cleanup(self):
try:
log.info("Cleanup...")
self.pool.stop_workers()
log.info("Cleanup was done")
except:
log.exception("Cleanup was finished with errors")
def stop(self):
self.running = False
class TestVirtualMachinePool(BaseTestCase):
def setUp(self):
setup_config('data/config.py')
self.platform = "test_origin_1"
with patch('core.connection.Virsh',
Mock()), patch('core.network.Network', Mock()):
from vmpool.virtual_machines_pool import VirtualMachinesPool
self.pool = VirtualMachinesPool()
def tearDown(self):
with patch('core.utils.delete_file', Mock()):
self.pool.free()
def test_pool_count(self):
self.assertEqual(0, self.pool.count())
self.pool.add(self.platform)
self.assertEqual(1, self.pool.count())
def test_get_parallel_two_vm(self):
from multiprocessing.pool import ThreadPool
threads = ThreadPool(processes=1)
self.pool.preload(self.platform)
self.pool.preload(self.platform)
self.assertEqual(2, len(self.pool.pool))
deffered1 = threads.apply_async(self.pool.get_by_platform,
args=(self.platform, ))
deffered2 = threads.apply_async(self.pool.get_by_platform,
args=(self.platform, ))
deffered1.wait()
deffered2.wait()
self.assertEqual(2, len(self.pool.using))
def test_vm_preloading(self):
self.assertEqual(0, len(self.pool.pool))
self.pool.preload(self.platform)
from vmpool import VirtualMachine
self.assertIsInstance(self.pool.pool[0], VirtualMachine)
self.assertEqual(1, len(self.pool.pool))
def test_vm_adding(self):
self.assertEqual(0, len(self.pool.pool))
self.pool.add(self.platform)
from vmpool import VirtualMachine
self.assertIsInstance(self.pool.using[0], VirtualMachine)
self.assertEqual(1, len(self.pool.using))
def test_vm_deletion(self):
self.assertEqual(0, len(self.pool.pool))
self.pool.preload(self.platform)
self.assertEqual(1, len(self.pool.pool))
vm = self.pool.get_by_platform(self.platform)
with patch('core.utils.delete_file', Mock()):
vm.delete()
self.assertEqual(0, self.pool.count())
def test_max_vm_count(self):
config.KVM_MAX_VM_COUNT = 2
self.pool.add(self.platform)
self.pool.add(self.platform)
self.assertIsNone(self.pool.add(self.platform))
def test_platform_from_config(self):
desired_caps = {'desiredCapabilities': {'platform': "test_origin_1"}}
config.PLATFORM = "test_origin_2"
self.app = Flask(__name__)
self.app.pool = self.pool
self.ctx = self.app.app_context()
self.ctx.push()
from vmpool.endpoint import get_vm
for vm in get_vm(desired_caps):
self.assertEqual(vm.platform, config.PLATFORM)
break
self.ctx.pop()
class TestVirtualMachinePool(BaseTestCase):
def setUp(self):
from core.db import Database
setup_config('data/config.py')
self.platform_name = "origin_1"
self.app = Flask(__name__)
self.ctx = self.app.app_context()
self.ctx.push()
self.app.database = Database()
self.app.sessions = Mock()
self.mocked_image = Mock(
id=1,
status='active',
get=Mock(return_value='snapshot'),
min_disk=20,
min_ram=2,
instance_type_flavorid=1,
)
type(self.mocked_image).name = PropertyMock(
return_value='test_origin_1')
with patch.multiple(
'vmpool.platforms.OpenstackPlatforms',
images=Mock(return_value=[self.mocked_image]),
flavor_params=Mock(return_value={
'vcpus': 1,
'ram': 2
}),
limits=Mock(
return_value={
'maxTotalCores': 10,
'maxTotalInstances': 10,
'maxTotalRAMSize': 100,
'totalCoresUsed': 0,
'totalInstancesUsed': 0,
'totalRAMUsed': 0
}),
):
from vmpool.virtual_machines_pool import VirtualMachinesPool
self.pool = VirtualMachinesPool(self.app,
preloader_class=Mock(),
artifact_collector_class=Mock(),
endpoint_preparer_class=Mock())
self.ctx = self.app.app_context()
self.ctx.push()
self.pool.register()
def tearDown(self):
self.pool.endpoint_remover.remove_all()
self.pool.unregister()
self.pool.platforms.cleanup()
self.ctx.pop()
def test_run_workers(self):
self.pool.endpoint_remover.start = Mock()
self.pool.start_workers()
self.assertTrue(self.pool.endpoint_preparer.start.called)
self.assertTrue(self.pool.endpoint_remover.start.called)
self.assertTrue(self.pool.preloader.start.called)
def test_stop_workers(self):
self.pool.endpoint_remover.stop = Mock()
self.pool.stop_workers()
self.assertTrue(self.pool.endpoint_preparer.stop.called)
self.assertTrue(self.pool.endpoint_remover.stop.called)
self.assertTrue(self.pool.artifact_collector.stop.called)
self.assertTrue(self.pool.preloader.stop.called)
def test_pool_count(self):
self.assertEqual(0, len(self.pool.active_endpoints))
self.pool.add(self.platform_name)
self.assertEqual(1, len(self.pool.active_endpoints))
def test_get_parallel_two_vm(self):
from multiprocessing.pool import ThreadPool
threads = ThreadPool(processes=1)
self.pool.preload(self.platform_name)
self.pool.preload(self.platform_name)
self.assertEqual(2, len(self.pool.active_endpoints))
deffered1 = threads.apply_async(self.pool.get_by_platform,
args=(self.platform_name, ))
deffered2 = threads.apply_async(self.pool.get_by_platform,
args=(self.platform_name, ))
deffered1.wait()
deffered2.wait()
self.assertEqual(2, len(self.pool.active_endpoints))
def test_vm_preloading(self):
self.assertEqual(0, len(self.pool.active_endpoints))
self.pool.preload(self.platform_name)
from core.db.models import OpenstackClone
self.assertIsInstance(self.pool.active_endpoints[0], OpenstackClone)
self.assertEqual(1, len(self.pool.active_endpoints))
def test_vm_adding(self):
self.assertEqual(0, len(self.pool.active_endpoints))
self.pool.add(self.platform_name)
from core.db.models import OpenstackClone
self.assertIsInstance(self.pool.active_endpoints[0], OpenstackClone)
self.assertEqual(1, len(self.pool.active_endpoints))
def test_vm_deletion(self):
self.assertEqual(0, len(self.pool.active_endpoints))
clone = self.pool.preload(self.platform_name)
self.assertEqual(1, len(self.pool.active_endpoints))
clone.delete()
self.app.database.delete(clone)
self.assertEqual(0, len(self.pool.active_endpoints))
def test_max_vm_count(self):
config.OPENSTACK_MAX_VM_COUNT = 2
self.pool.add(self.platform_name)
self.pool.add(self.platform_name)
self.assertIsNone(self.pool.add(self.platform_name))
def test_got_ondemand_vm_if_got_env_vars(self):
dc = {"environmentVars": {"TEST_ENV2": 0}}
self.assertEqual(0, len(self.pool.active_endpoints))
self.pool.preload(self.platform_name)
self.assertEqual(1, len(self.pool.active_endpoints))
vm = self.pool.get_vm(self.platform_name, dc=json.dumps(dc))
self.assertTrue(vm.is_ondemand())
self.assertDictEqual(vm.environment_variables, {u"TEST_ENV2": 0})
self.assertEqual(2, len(self.pool.active_endpoints))
def test_got_preloaded_vm_if_env_vars_is_None(self):
dc = {"desiredCapabilities": {}}
self.assertEqual(0, len(self.pool.active_endpoints))
self.pool.preload(self.platform_name)
self.assertEqual(1, len(self.pool.active_endpoints))
vm = self.pool.get_vm(self.platform_name, dc=json.dumps(dc))
self.assertTrue(vm.is_preloaded())
self.assertEqual(1, len(self.pool.active_endpoints))