def get_memory_limit(self, app_repr):
"""
Validation for memory limits.
"""
min_memory_limit = app_repr.get('min_memory_limit')
max_memory_limit = app_repr.get('max_memory_limit')
if max_memory_limit:
max_memory_limit = MemoryInt(max_memory_limit)
if max_memory_limit < Plugin.defaults['memory_limit']:
max_memory_limit = MemoryInt(Plugin.defaults['memory_limit'])
if min_memory_limit:
min_memory_limit = MemoryInt(min_memory_limit)
if max_memory_limit and min_memory_limit and max_memory_limit < min_memory_limit:
raise ValueError(
"min memory Limit should be less than max memory limit.")
return max_memory_limit, min_memory_limit
def save_plugin_inst(self, plg_inst_dict):
"""
Custom method to save a plugin instance and its parameters to the DB.
"""
plg_inst = plg_inst_dict['plugin_inst']
plg = plg_inst.plugin
pipeline_inst = self.pipeline_inst
plg_inst.pipeline_inst = pipeline_inst
plg_inst.title = pipeline_inst.title
# set cpu limit
if pipeline_inst.cpu_limit:
plg_inst.cpu_limit = pipeline_inst.cpu_limit
if plg_inst.cpu_limit < CPUInt(plg.min_cpu_limit):
plg_inst.cpu_limit = CPUInt(plg.min_cpu_limit)
if plg_inst.cpu_limit > CPUInt(plg.max_cpu_limit):
plg_inst.cpu_limit = CPUInt(plg.max_cpu_limit)
# set memory limit
if pipeline_inst.memory_limit:
plg_inst.memory_limit = pipeline_inst.memory_limit
if plg_inst.memory_limit < MemoryInt(plg.min_memory_limit):
plg_inst.memory_limit = MemoryInt(plg.min_memory_limit)
if plg_inst.memory_limit > MemoryInt(plg.max_memory_limit):
plg_inst.memory_limit = MemoryInt(plg.max_memory_limit)
# set number of workers
if pipeline_inst.number_of_workers:
plg_inst.number_of_workers = pipeline_inst.number_of_workers
if plg_inst.number_of_workers < plg.number_of_workers:
plg_inst.number_of_workers = plg.min_number_of_workers
if plg_inst.number_of_workers > plg.max_number_of_workers:
plg_inst.number_of_workers = plg.max_number_of_workers
# set gpu limit
if pipeline_inst.gpu_limit:
plg_inst.gpu_limit = pipeline_inst.gpu_limit
if plg_inst.gpu_limit < plg.min_gpu_limit:
plg_inst.gpu_limit = plg.min_gpu_limit
if plg_inst.gpu_limit > plg.max_gpu_limit:
plg_inst.gpu_limit = plg.max_gpu_limit
# save plugin instance
previous = plg_inst.previous
plg_inst.save()
plg_inst.previous = previous
plg_inst.save()
# save plugin instance's parameters
for param, param_serializer in plg_inst_dict['parameter_serializers']:
param_serializer.save(plugin_inst=plg_inst, plugin_param=param)
def add_plugin(self, dock_image_name, compute_resource_identifier):
"""
Register/add a new plugin to the system.
"""
# get representation from the corresponding app
app_repr = self.get_plugin_app_representation(dock_image_name)
name = self.get_plugin_name(app_repr)
max_cpu_limit, min_cpu_limit = self.get_cpu_limit(app_repr)
max_memory_limit, min_memory_limit = self.get_memory_limit(app_repr)
max_number_of_workers, min_number_of_workers = self.get_number_of_workers(
app_repr)
max_gpu_limit, min_gpu_limit = self.get_gpu_limit(app_repr)
# check wether the plugin already exist
existing_plugin_names = [
plugin.name for plugin in Plugin.objects.all()
]
if name in existing_plugin_names:
raise ValueError("Plugin '%s' already exists in the system" % name)
# add plugin to the db
plugin = Plugin()
plugin.name = name
plugin.dock_image = dock_image_name
plugin.type = app_repr['type']
plugin.authors = app_repr['authors']
plugin.title = app_repr['title']
plugin.category = app_repr['category']
plugin.description = app_repr['description']
plugin.documentation = app_repr['documentation']
plugin.license = app_repr['license']
plugin.version = app_repr['version']
(plugin.compute_resource, tf) = ComputeResource.objects.get_or_create(
compute_resource_identifier=compute_resource_identifier)
plugin.max_cpu_limit = self.insert_default(
max_cpu_limit, CPUInt(Plugin.defaults['max_limit']))
plugin.min_cpu_limit = self.insert_default(
min_cpu_limit, Plugin.defaults['min_cpu_limit'])
plugin.max_memory_limit = self.insert_default(
max_memory_limit, MemoryInt(Plugin.defaults['max_limit']))
plugin.min_memory_limit = self.insert_default(
min_memory_limit, Plugin.defaults['min_memory_limit'])
plugin.max_number_of_workers = self.insert_default(
max_number_of_workers, Plugin.defaults['max_limit'])
plugin.min_number_of_workers = self.insert_default(
min_number_of_workers, 1)
plugin.max_gpu_limit = self.insert_default(
max_gpu_limit, Plugin.defaults['max_limit'])
plugin.min_gpu_limit = self.insert_default(min_gpu_limit, 0)
plugin.save()
# add plugin's parameters to the db
params = app_repr['parameters']
for param in params:
self._save_plugin_param(plugin, param)
def _set_compute_defaults(self):
"""
Custom internal method to set compute-related defaults.
"""
if not self.cpu_limit:
self.cpu_limit = CPUInt(self.plugin.min_cpu_limit)
if not self.memory_limit:
self.memory_limit = MemoryInt(self.plugin.min_memory_limit)
if not self.number_of_workers:
self.number_of_workers = self.plugin.min_number_of_workers
if not self.gpu_limit:
self.gpu_limit = self.plugin.min_gpu_limit
def create(self, validated_data):
"""
Overriden to provide compute-related defaults before creating a new plugin
instance.
"""
plugin = self.context['view'].get_object()
if 'gpu_limit' not in validated_data:
validated_data['gpu_limit'] = plugin.min_gpu_limit
if 'number_of_workers' not in validated_data:
validated_data['number_of_workers'] = plugin.min_number_of_workers
if 'cpu_limit' not in validated_data:
validated_data['cpu_limit'] = CPUInt(plugin.min_cpu_limit)
if 'memory_limit' not in validated_data:
validated_data['memory_limit'] = MemoryInt(plugin.min_memory_limit)
return super(PluginInstanceSerializer, self).create(validated_data)
def test_create(self):
"""
Test whether overriden 'create' method adds default values for gpu_limit,
number_of_workers, cpu_limit and memory_limit.
"""
data = self.data
plugin = self.plugin
plg_inst_serializer = PluginInstanceSerializer(data=data)
plg_inst_serializer.is_valid(raise_exception=True)
plg_inst_serializer.context = {'view': mock.Mock()}
plg_inst_serializer.context['view'].get_object = mock.Mock(
return_value=plugin)
plg_inst_serializer.create(data)
self.assertEqual(data['gpu_limit'], plugin.min_gpu_limit)
self.assertEqual(data['number_of_workers'],
plugin.min_number_of_workers)
self.assertEqual(data['cpu_limit'], CPUInt(plugin.min_cpu_limit))
self.assertEqual(data['memory_limit'],
MemoryInt(plugin.min_memory_limit))