class PluginInstance(models.Model):
title = models.CharField(max_length=100, blank=True)
start_date = models.DateTimeField(auto_now_add=True)
end_date = models.DateTimeField(auto_now_add=True)
status = models.CharField(max_length=30,
choices=STATUS_CHOICES,
default='created')
summary = models.CharField(max_length=4000, blank=True, default='')
raw = models.TextField(blank=True, default='')
previous = models.ForeignKey("self",
on_delete=models.CASCADE,
null=True,
related_name='next')
plugin = models.ForeignKey(Plugin,
on_delete=models.CASCADE,
related_name='instances')
feed = models.ForeignKey(Feed,
on_delete=models.CASCADE,
related_name='plugin_instances')
owner = models.ForeignKey('auth.User', on_delete=models.CASCADE)
compute_resource = models.ForeignKey(ComputeResource,
null=True,
on_delete=models.SET_NULL,
related_name='plugin_instances')
pipeline_inst = models.ForeignKey(PipelineInstance,
null=True,
on_delete=models.SET_NULL,
related_name='plugin_instances')
cpu_limit = CPUField(null=True)
memory_limit = MemoryField(null=True)
number_of_workers = models.IntegerField(null=True)
gpu_limit = models.IntegerField(null=True)
class Meta:
ordering = ('-start_date', )
def __str__(self):
return self.title
def save(self, *args, **kwargs):
"""
Overriden to save a new feed to the DB the first time 'fs' instances are saved.
For 'ds' instances the feed of the previous instance is assigned.
"""
if not hasattr(self, 'feed'):
if self.plugin.meta.type == 'fs':
self.feed = self._save_feed()
if self.plugin.meta.type == 'ds':
self.feed = self.previous.feed
self._set_compute_defaults()
super(PluginInstance, self).save(*args, **kwargs)
def _save_feed(self):
"""
Custom internal method to create and save a new feed to the DB.
"""
feed = Feed()
feed.name = self.plugin.meta.name
feed.save()
feed.owner.set([self.owner])
feed.save()
return feed
def _set_compute_defaults(self):
"""
Custom internal method to set compute-related defaults.
"""
if not self.cpu_limit:
self.cpu_limit = CPUInt(self.plugin.min_cpu_limit)
if not self.memory_limit:
self.memory_limit = MemoryInt(self.plugin.min_memory_limit)
if not self.number_of_workers:
self.number_of_workers = self.plugin.min_number_of_workers
if not self.gpu_limit:
self.gpu_limit = self.plugin.min_gpu_limit
def get_root_instance(self):
"""
Custom method to return the root plugin instance for this plugin instance.
"""
current = self
while not current.plugin.meta.type == 'fs':
current = current.previous
return current
def get_descendant_instances(self):
"""
Custom method to return all the plugin instances that are a descendant of this
plugin instance.
"""
descendant_instances = []
queue = [self]
while len(queue) > 0:
visited = queue.pop()
queue.extend(list(visited.next.all()))
descendant_instances.append(visited)
return descendant_instances
def get_output_path(self):
"""
Custom method to get the output directory for files generated by
the plugin instance object.
"""
# 'fs' plugins will output files to:
# SWIFT_CONTAINER_NAME/<username>/feed_<id>/plugin_name_plugin_inst_<id>/data
# 'ds' plugins will output files to:
# SWIFT_CONTAINER_NAME/<username>/feed_<id>/...
#/previous_plugin_name_plugin_inst_<id>/plugin_name_plugin_inst_<id>/data
current = self
path = '/{0}_{1}/data'.format(current.plugin.meta.name, current.id)
while not current.plugin.meta.type == 'fs':
current = current.previous
path = '/{0}_{1}'.format(current.plugin.meta.name,
current.id) + path
username = self.owner.username
output_path = '{0}/feed_{1}'.format(username, current.feed.id) + path
return output_path
def get_parameter_instances(self):
"""
Custom method to get all the parameter instances associated with this plugin
instance regardless of their type.
"""
parameter_instances = []
parameter_instances.extend(list(self.unextpath_param.all()))
parameter_instances.extend(list(self.path_param.all()))
parameter_instances.extend(list(self.string_param.all()))
parameter_instances.extend(list(self.integer_param.all()))
parameter_instances.extend(list(self.float_param.all()))
parameter_instances.extend(list(self.boolean_param.all()))
return parameter_instances
def register_output_files(self, **kwargs):
"""
Custom method to register files generated by the plugin instance object
with the REST API.
"""
# If using `rpudb` need to have the creation point in an
# exception condition to prevent multiple attempts to create
# the same telnet server...
# try:
# rpudb.set_trace(addr='0.0.0.0', port=7901)
# except:
# pass
# pudb.set_trace()
d_swiftstate = kwargs['swiftState'] if 'swiftState' in kwargs else {}
swift_manager = SwiftManager(settings.SWIFT_CONTAINER_NAME,
settings.SWIFT_CONNECTION_PARAMS)
output_path = self.get_output_path()
# the following gets the full list of objects in the swift storage
# with prefix of <output_path>. Since there is a lag in consistency
# of swift state from different clients, we poll here using the
# information returned from pfcon that indicates how many files
l_objCurrentlyReportedBySwift = []
try:
l_objCurrentlyReportedBySwift = swift_manager.ls(output_path)
except ClientException as e:
logger.error('Swift storage error, detail: %s' % str(e))
if 'd_swift_ls' in d_swiftstate.keys():
# This conditional processes the case where a remote process has
# returned data that has been pushed into swift by ancillary services
l_objPutIntoSwift = d_swiftstate['d_swift_ls']['lsList']
else:
# This conditional addressed the case where an internal CUBE
# process and *not* a remote execution pushed data into swift
l_objPutIntoSwift = l_objCurrentlyReportedBySwift
maxWaitPoll = 20
waitPoll = 0
while (waitPoll < maxWaitPoll) and not all(
obj in l_objCurrentlyReportedBySwift
for obj in l_objPutIntoSwift):
time.sleep(0.2)
waitPoll += 1
try:
l_objCurrentlyReportedBySwift = swift_manager.ls(output_path)
except ClientException as e:
logger.error('Swift storage error, detail: %s' % str(e))
b_status = False
if all(obj in l_objCurrentlyReportedBySwift
for obj in l_objPutIntoSwift):
b_status = True
file_count = 0
for obj_name in l_objCurrentlyReportedBySwift:
logger.info('Registering -->%s<--', obj_name)
plg_inst_file = PluginInstanceFile(plugin_inst=self)
plg_inst_file.fname.name = obj_name
try:
plg_inst_file.save()
except IntegrityError: # avoid re-register a file already registered
logger.info('-->%s<-- already registered', obj_name)
file_count += 1
return {
'status': b_status,
'l_object': l_objCurrentlyReportedBySwift,
'total': file_count,
'outputPath': output_path,
'pollLoop': waitPoll
}
class PluginInstance(models.Model):
title = models.CharField(max_length=100, blank=True)
start_date = models.DateTimeField(auto_now_add=True)
end_date = models.DateTimeField(auto_now_add=True)
status = models.CharField(max_length=30,
choices=STATUS_CHOICES,
default='created')
summary = models.CharField(max_length=4000, blank=True, default='')
raw = models.TextField(blank=True, default='')
previous = models.ForeignKey("self",
on_delete=models.CASCADE,
null=True,
related_name='next')
plugin = models.ForeignKey(Plugin,
on_delete=models.CASCADE,
related_name='instances')
feed = models.ForeignKey(Feed,
on_delete=models.CASCADE,
related_name='plugin_instances')
owner = models.ForeignKey('auth.User', on_delete=models.CASCADE)
compute_resource = models.ForeignKey(ComputeResource,
null=True,
on_delete=models.SET_NULL,
related_name='plugin_instances')
pipeline_inst = models.ForeignKey(PipelineInstance,
null=True,
on_delete=models.SET_NULL,
related_name='plugin_instances')
cpu_limit = CPUField(null=True)
memory_limit = MemoryField(null=True)
number_of_workers = models.IntegerField(null=True)
gpu_limit = models.IntegerField(null=True)
class Meta:
ordering = ('-start_date', )
def __str__(self):
return self.title
def save(self, *args, **kwargs):
"""
Overriden to save a new feed to the DB the first time 'fs' instances are saved.
For 'ds' instances the feed of the previous instance is assigned.
"""
if not hasattr(self, 'feed'):
if self.plugin.meta.type == 'fs':
self.feed = self._save_feed()
if self.plugin.meta.type == 'ds':
self.feed = self.previous.feed
self._set_compute_defaults()
super(PluginInstance, self).save(*args, **kwargs)
def _save_feed(self):
"""
Custom internal method to create and save a new feed to the DB.
"""
feed = Feed()
feed.name = self.plugin.meta.name
feed.save()
feed.owner.set([self.owner])
feed.save()
return feed
def _set_compute_defaults(self):
"""
Custom internal method to set compute-related defaults.
"""
if not self.cpu_limit:
self.cpu_limit = CPUInt(self.plugin.min_cpu_limit)
if not self.memory_limit:
self.memory_limit = MemoryInt(self.plugin.min_memory_limit)
if not self.number_of_workers:
self.number_of_workers = self.plugin.min_number_of_workers
if not self.gpu_limit:
self.gpu_limit = self.plugin.min_gpu_limit
def get_root_instance(self):
"""
Custom method to return the root plugin instance for this plugin instance.
"""
current = self
while not current.plugin.meta.type == 'fs':
current = current.previous
return current
def get_descendant_instances(self):
"""
Custom method to return all the plugin instances that are a descendant of this
plugin instance.
"""
descendant_instances = []
queue = [self]
while len(queue) > 0:
visited = queue.pop()
queue.extend(list(visited.next.all()))
descendant_instances.append(visited)
return descendant_instances
def get_output_path(self):
"""
Custom method to get the output directory for files generated by
the plugin instance object.
"""
# 'fs' plugins will output files to:
# SWIFT_CONTAINER_NAME/<username>/feed_<id>/plugin_name_plugin_inst_<id>/data
# 'ds' plugins will output files to:
# SWIFT_CONTAINER_NAME/<username>/feed_<id>/...
#/previous_plugin_name_plugin_inst_<id>/plugin_name_plugin_inst_<id>/data
current = self
path = '/{0}_{1}/data'.format(current.plugin.meta.name, current.id)
while not current.plugin.meta.type == 'fs':
current = current.previous
path = '/{0}_{1}'.format(current.plugin.meta.name,
current.id) + path
username = self.owner.username
output_path = '{0}/feed_{1}'.format(username, current.feed.id) + path
return output_path
def get_parameter_instances(self):
"""
Custom method to get all the parameter instances associated with this plugin
instance regardless of their type.
"""
parameter_instances = []
parameter_instances.extend(list(self.unextpath_param.all()))
parameter_instances.extend(list(self.path_param.all()))
parameter_instances.extend(list(self.string_param.all()))
parameter_instances.extend(list(self.integer_param.all()))
parameter_instances.extend(list(self.float_param.all()))
parameter_instances.extend(list(self.boolean_param.all()))
return parameter_instances
def register_output_files(self, **kwargs):
"""
Custom method to register files generated by the plugin instance object
with the REST API.
"""
d_swiftstate = kwargs['swiftState'] if 'swiftState' in kwargs else {}
swift_manager = SwiftManager(settings.SWIFT_CONTAINER_NAME,
settings.SWIFT_CONNECTION_PARAMS)
output_path = self.get_output_path()
# the following gets the full list of objects in the swift storage
# with prefix of <output_path>. Since there is a lag in consistency
# of swift state from different clients, we poll here using the
# information returned from pfcon that indicates how many files
object_name_list = []
objects_reported_in_swift = 0
if 'd_swiftstore' in d_swiftstate:
objects_reported_in_swift = d_swiftstate['d_swiftstore'][
'filesPushed']
for poll_loop in range(20):
if len(object_name_list) == objects_reported_in_swift: break
time.sleep(0.3)
try:
object_name_list = swift_manager.ls(output_path)
except Exception as e:
logger.error('Swift storage error, detail: %s' % str(e))
file_count = 0
for obj_name in object_name_list:
# avoid re-register a file already registered which makes this idempotent
try:
self.files.get(fname=obj_name)
except ObjectDoesNotExist:
plg_inst_file = PluginInstanceFile(plugin_inst=self)
plg_inst_file.fname.name = obj_name
plg_inst_file.save()
file_count += 1
return {
'status': True,
'l_object': object_name_list,
'total': file_count,
'outputPath': output_path,
'pollLoop': poll_loop
}
class PluginInstance(models.Model):
title = models.CharField(max_length=100, blank=True)
start_date = models.DateTimeField(auto_now_add=True)
end_date = models.DateTimeField(auto_now_add=True)
status = models.CharField(max_length=30,
choices=STATUS_CHOICES,
default='created')
summary = models.CharField(max_length=4000, blank=True)
raw = models.TextField(blank=True)
size = models.IntegerField(default=0)
error_code = models.CharField(max_length=7, blank=True)
previous = models.ForeignKey("self",
on_delete=models.CASCADE,
null=True,
related_name='next')
plugin = models.ForeignKey(Plugin,
on_delete=models.CASCADE,
related_name='instances')
feed = models.ForeignKey(Feed,
on_delete=models.CASCADE,
related_name='plugin_instances')
owner = models.ForeignKey('auth.User', on_delete=models.CASCADE)
compute_resource = models.ForeignKey(ComputeResource,
null=True,
on_delete=models.SET_NULL,
related_name='plugin_instances')
pipeline_inst = models.ForeignKey(PipelineInstance,
null=True,
on_delete=models.SET_NULL,
related_name='plugin_instances')
cpu_limit = CPUField(null=True)
memory_limit = MemoryField(null=True)
number_of_workers = models.IntegerField(null=True)
gpu_limit = models.IntegerField(null=True)
class Meta:
ordering = ('-start_date', )
def __str__(self):
return self.title
def save(self, *args, **kwargs):
"""
Overriden to save a new feed to the DB the first time 'fs' instances are saved.
For 'ds' and 'ts' instances the feed of the previous instance is assigned.
"""
if not hasattr(self, 'feed'):
plugin_type = self.plugin.meta.type
if plugin_type == 'fs':
self.feed = self._save_feed()
elif plugin_type in ('ds', 'ts'):
self.feed = self.previous.feed
self._set_compute_defaults()
super(PluginInstance, self).save(*args, **kwargs)
def _save_feed(self):
"""
Custom internal method to create and save a new feed to the DB.
"""
feed = Feed()
feed.name = self.title or self.plugin.meta.name
feed.save()
feed.owner.set([self.owner])
feed.save()
return feed
def _set_compute_defaults(self):
"""
Custom internal method to set compute-related defaults.
"""
if not self.cpu_limit:
self.cpu_limit = CPUInt(self.plugin.min_cpu_limit)
if not self.memory_limit:
self.memory_limit = MemoryInt(self.plugin.min_memory_limit)
if not self.number_of_workers:
self.number_of_workers = self.plugin.min_number_of_workers
if not self.gpu_limit:
self.gpu_limit = self.plugin.min_gpu_limit
def get_root_instance(self):
"""
Custom method to return the root plugin instance for this plugin instance.
"""
current = self
while not current.plugin.meta.type == 'fs':
current = current.previous
return current
def get_descendant_instances(self):
"""
Custom method to return all the plugin instances that are a descendant of this
plugin instance.
"""
descendant_instances = []
queue = [self]
while len(queue) > 0:
visited = queue.pop()
queue.extend(list(visited.next.all()))
descendant_instances.append(visited)
return descendant_instances
def get_output_path(self):
"""
Custom method to get the output directory for files generated by
the plugin instance object.
"""
# 'fs' plugins will output files to:
# SWIFT_CONTAINER_NAME/<username>/feed_<id>/plugin_name_plugin_inst_<id>/data
# 'ds' and 'ts' plugins will output files to:
# SWIFT_CONTAINER_NAME/<username>/feed_<id>/...
#/previous_plugin_name_plugin_inst_<id>/plugin_name_plugin_inst_<id>/data
current = self
path = '/{0}_{1}/data'.format(current.plugin.meta.name, current.id)
while not current.plugin.meta.type == 'fs':
current = current.previous
path = '/{0}_{1}'.format(current.plugin.meta.name,
current.id) + path
username = self.owner.username
output_path = '{0}/feed_{1}'.format(username, current.feed.id) + path
return output_path
def get_parameter_instances(self):
"""
Custom method to get all the parameter instances associated with this plugin
instance regardless of their type.
"""
parameter_instances = []
parameter_instances.extend(list(self.unextpath_param.all()))
parameter_instances.extend(list(self.path_param.all()))
parameter_instances.extend(list(self.string_param.all()))
parameter_instances.extend(list(self.integer_param.all()))
parameter_instances.extend(list(self.float_param.all()))
parameter_instances.extend(list(self.boolean_param.all()))
return parameter_instances
class PluginInstance(models.Model):
title = models.CharField(max_length=100, blank=True)
start_date = models.DateTimeField(auto_now_add=True)
end_date = models.DateTimeField(auto_now_add=True)
status = models.CharField(max_length=30, default=STATUS_TYPES[0])
previous = models.ForeignKey("self",
on_delete=models.CASCADE,
null=True,
related_name='next')
plugin = models.ForeignKey(Plugin,
on_delete=models.CASCADE,
related_name='instances')
feed = models.ForeignKey(Feed,
on_delete=models.CASCADE,
related_name='plugin_instances')
owner = models.ForeignKey('auth.User', on_delete=models.CASCADE)
compute_resource = models.ForeignKey(ComputeResource,
on_delete=models.CASCADE,
related_name='plugin_instances')
pipeline_inst = models.ForeignKey(PipelineInstance,
null=True,
on_delete=models.SET_NULL,
related_name='plugin_instances')
cpu_limit = CPUField(null=True)
memory_limit = MemoryField(null=True)
number_of_workers = models.IntegerField(null=True)
gpu_limit = models.IntegerField(null=True)
class Meta:
ordering = ('-start_date', )
def __str__(self):
return self.title
def save(self, *args, **kwargs):
"""
Overriden to save a new feed to the DB the first time 'fs' instances are saved.
For 'ds' instances the feed of the previous instance is assigned.
"""
if not hasattr(self, 'feed'):
if self.plugin.type == 'fs':
self.feed = self._save_feed()
if self.plugin.type == 'ds':
self.feed = self.previous.feed
super(PluginInstance, self).save(*args, **kwargs)
def _save_feed(self):
"""
Custom method to create and save a new feed to the DB.
"""
feed = Feed()
feed.name = self.plugin.name
feed.save()
feed.owner.set([self.owner])
feed.save()
return feed
def get_root_instance(self):
"""
Custom method to return the root plugin instance for this plugin instance.
"""
current = self
while not current.plugin.type == 'fs':
current = current.previous
return current
def get_descendant_instances(self):
"""
Custom method to return all the plugin instances that are a descendant of this
plugin instance.
"""
descendant_instances = []
queue = [self]
while len(queue) > 0:
visited = queue.pop()
queue.extend(list(visited.next.all()))
descendant_instances.append(visited)
return descendant_instances
def get_output_path(self):
"""
Custom method to get the output directory for files generated by
the plugin instance object.
"""
# 'fs' plugins will output files to:
# SWIFT_CONTAINER_NAME/<username>/feed_<id>/plugin_name_plugin_inst_<id>/data
# 'ds' plugins will output files to:
# SWIFT_CONTAINER_NAME/<username>/feed_<id>/...
#/previous_plugin_name_plugin_inst_<id>/plugin_name_plugin_inst_<id>/data
current = self
path = '/{0}_{1}/data'.format(current.plugin.name, current.id)
while not current.plugin.type == 'fs':
current = current.previous
path = '/{0}_{1}'.format(current.plugin.name, current.id) + path
username = self.owner.username
output_path = '{0}/feed_{1}'.format(username, current.feed.id) + path
return output_path
def register_output_files(self, *args, **kwargs):
"""
Custom method to register files generated by the plugin instance object
with the REST API.
"""
d_swiftState = {}
for k, v in kwargs.items():
if k == 'swiftState': d_swiftState = v
# initiate a Swift service connection
conn = swiftclient.Connection(
user=settings.SWIFT_USERNAME,
key=settings.SWIFT_KEY,
authurl=settings.SWIFT_AUTH_URL,
)
output_path = self.get_output_path()
# the following gets the full list of objects in the swift storage
# with prefix of <output_path>. Since there is a lag in consistency
# of swift state from different clients, we poll here using the
# information returned from pfcon that indicates how many files
object_list = []
pollLoop = 0
maxPolls = 20
if 'd_swiftstore' in d_swiftState.keys():
objectsReportedInSwift = d_swiftState['d_swiftstore'][
'filesPushed']
else:
objectsReportedInSwift = 0
while len(
object_list) <= objectsReportedInSwift and pollLoop < maxPolls:
object_list = conn.get_container(settings.SWIFT_CONTAINER_NAME,
prefix=output_path,
full_listing=True)[1]
time.sleep(0.2)
pollLoop += 1
fileCount = 0
for object in object_list:
plg_inst_file = PluginInstanceFile(plugin_inst=self)
plg_inst_file.fname.name = object['name']
plg_inst_file.save()
fileCount += 1
return {
'status': True,
'l_object': object_list,
'total': fileCount,
'outputPath': output_path,
'pollLoop': pollLoop
}