class SimpleMultiprocessTestCase(unittest.TestCase):
"""
Test the correct loading of a multiprocessed plugin as well as basic
communication.
"""
def setUp(self):
"""
init
"""
# create the plugin manager
self.mpPluginManager = MultiprocessPluginManager(directories_list=[
os.path.join(
os.path.dirname(os.path.abspath(__file__)),"plugins")],
plugin_info_ext="multiprocess-plugin")
# load the plugins that may be found
self.mpPluginManager.collectPlugins()
# Will be used later
self.plugin_info = None
def testUpAndRunning(self):
"""
Test if the plugin is loaded and if the communication pipe is properly setuped.
"""
numTestedPlugins = 0
for plugin in self.mpPluginManager.getAllPlugins():
content_from_parent = "hello-from-parent"
content_from_child = False
plugin.plugin_object.child_pipe.send(content_from_parent)
if plugin.plugin_object.child_pipe.poll(5):
content_from_child = plugin.plugin_object.child_pipe.recv()
self.assertEqual(content_from_child, "{0}|echo_from_child".format(content_from_parent))
numTestedPlugins += 1
self.assertTrue(numTestedPlugins >= 1)
def _create_plugin_manager(self):
self.manager = MultiprocessPluginManager(
directories_list=self._plugin_directories,
plugin_info_ext="multiprocess-plugin")
self.manager.setCategoriesFilter({
"Input": NuocaMPInputPlugin,
"Output": NuocaMPOutputPlugin,
"Transform": NuocaMPTransformPlugin
})
def setUp(self):
"""
init
"""
# create the plugin manager
self.mpPluginManager = MultiprocessPluginManager(
directories_list=[
os.path.join(os.path.dirname(os.path.abspath(__file__)),
"plugins")
],
plugin_info_ext="multiprocess-plugin")
# load the plugins that may be found
self.mpPluginManager.collectPlugins()
# Will be used later
self.plugin_info = None
def runTest(self):
topdir = nuoca_util.get_nuoca_topdir()
output_plugin_dir = os.path.join(topdir, "plugins/output")
dir_list = [output_plugin_dir]
self._PrinterPluginTest()
self.manager = MultiprocessPluginManager(
directories_list=dir_list, plugin_info_ext="multiprocess-plugin")
self._MultiprocessPluginManagerTest()
def runTest(self):
topdir = nuoca_util.get_nuoca_topdir()
input_plugin_dir = os.path.join(topdir, "plugins/input")
dir_list = [input_plugin_dir]
self._LogstashPluginTest("06a32504-c2c9-41bc-9b48-030982c5ea43.r0db0")
self._LogstashPluginTest("fa2461c7-bca2-4df5-91e3-251084e1b8d1.r0db2")
self.manager = MultiprocessPluginManager(
directories_list=dir_list, plugin_info_ext="multiprocess-plugin")
self._MultiprocessPluginManagerCompareTest()
def initYapsy(self):
"""
inicialisation of yapsi subsystems...
:return:
"""
self.multiprocessPluginManager = MultiprocessPluginManager()
self.multiprocessPluginManager.setPluginPlaces(["plugins"])
self.multiprocessPluginManager.collectPlugins()
self.logm("MWorkerLib:initYapsy:", "following pluggins ready")
for pluginInfo in self.multiprocessPluginManager.getAllPlugins():
self.logm("MWorkerLib:initYapsy:", ">" + pluginInfo.name)
def setUp(self): """ init """ # create the plugin manager self.mpPluginManager = MultiprocessPluginManager(directories_list=[ os.path.join( os.path.dirname(os.path.abspath(__file__)),"plugins")], plugin_info_ext="multiprocess-plugin") # load the plugins that may be found self.mpPluginManager.collectPlugins() # Will be used later self.plugin_info = None
class SimpleMultiprocessTestCase(unittest.TestCase):
"""
Test the correct loading of a multiprocessed plugin as well as basic
communication.
"""
def setUp(self):
"""
init
"""
# create the plugin manager
self.mpPluginManager = MultiprocessPluginManager(
directories_list=[
os.path.join(os.path.dirname(os.path.abspath(__file__)),
"plugins")
],
plugin_info_ext="multiprocess-plugin")
# load the plugins that may be found
self.mpPluginManager.collectPlugins()
# Will be used later
self.plugin_info = None
def testUpAndRunning(self):
"""
Test if the plugin is loaded and if the communication pipe is properly setuped.
"""
for plugin_index, plugin in enumerate(
self.mpPluginManager.getAllPlugins()):
child_pipe = plugin.plugin_object.child_pipe
content_from_parent = "hello-{0}-from-parent".format(plugin_index)
child_pipe.send(content_from_parent)
content_from_child = False
if child_pipe.poll(5):
content_from_child = child_pipe.recv()
self.assertEqual("{0}|echo_from_child".format(content_from_parent),
content_from_child)
num_tested_plugin = plugin_index + 1
self.assertEqual(2, num_tested_plugin)
class MWorker:
multiprocessPluginManager = "" # yapsy manager
meteorClient = "" # python-meteor client
# session=requests.Session() #http client
meteorUrl = "ws://127.0.0.1:3000/websocket"
CYCLE_DEFAULT_DELAY = 20 # 10 seconds
SYNCHRO_TIMEOUT = 10 # meteor functions are asynchron. this is delay for how long i should wait till
# it is considered as fail... 2 sec are default
subscription_status = {} # this will assure, subscription is ready... "subscription_name":true/false
workername = "localhostworker" # this is worker name - this must be initialised at moment of start, and
# must by uniqued...
data = ""
error = ""
syncwaitdone = False
def __init__(self, meteorUrl):
"""
constructor
:return:
"""
self.meteorUrl = meteorUrl
self.logm("MWorker:constructor", "python meteor client initialisation:")
self.initMeteorConnect()
self.logm("MWorker:constructor", "getting unassigned")
# self.infiniteCycle()
self.getUnassigned()
def infiniteCycle(self):
"""
this is to keep thread running - it can be interupted by ctrl+c
:return:
"""
while True:
try:
time.sleep(1)
except KeyboardInterrupt:
break
def initMeteorConnect(self):
"""
this will use library python-meteor in order to escablish session to selected meteor
:return:
"""
self.logm("MWorkerLib:initMeteorConnect:", "meteor python lib client iniciaisation...")
self.meteorClient = MeteorClient('ws://127.0.0.1:3000/websocket', auto_reconnect=True)
self.meteorClient.connect()
self.subscribeCollectionSynchron(['unassigned', 'active', 'trash'], self.SYNCHRO_TIMEOUT)
self.logm("MWorkerLib:initMeteorConnect:", "meteor python lib client iniciaisation done...")
def meteorCollectionSubscribed(self, subscription):
self.subscription_status[subscription] = True;
self.logm("MWorkerLib:subscribed:", 'SUBSCRIBED {}'.format(subscription))
def meteorConnected(self):
self.logm("MWorkerLib:connected:", ' CONNECTED')
def initYapsy(self):
"""
inicialisation of yapsi subsystems...
:return:
"""
self.multiprocessPluginManager = MultiprocessPluginManager()
self.multiprocessPluginManager.setPluginPlaces(["plugins"])
self.multiprocessPluginManager.collectPlugins()
self.logm("MWorkerLib:initYapsy:", "following pluggins ready")
for pluginInfo in self.multiprocessPluginManager.getAllPlugins():
self.logm("MWorkerLib:initYapsy:", ">" + pluginInfo.name)
def isSubscriptionProcessDone(self):
"""
this will check, if in self.subscription_status all collections are set to true
:return: true if all requested collections are subscribed, orthervice false..
"""
kk = self.subscription_status.keys()
for col in kk:
if self.subscription_status[col] == False:
self.logm("MWorkerLib:isSubscriptionProcessDone:", self.subscription_status)
return False
return True
def subscribeCollectionSynchron(self, collections, timeout):
"""
this is synchron method. this means, there is loop for "timeout" seconds,
so subscription can be estalished...
:param: timeout - number of seconds for cycle this function will have
:param: collections - array of collections to be subscribed with self.meteorClient...
['unassigned', 'active', 'trash']
:return: will return true, or collection if some is false
"""
self.logm("MWorkerLib:subscribeCollectionsynchron:", "begining with subscription")
self.meteorClient.on('subscribed', self.meteorCollectionSubscribed)
self.meteorClient.on('connected', self.meteorConnected)
for col in collections:
self.subscription_status[col] = False
self.meteorClient.subscribe(col)
self.logm("MWorkerLib:subscribeCollectionSynchron",
"subscription init done, now waiting...:" + str(self.meteorClient.subscriptions.keys()))
epoch = int(time.time())
while (((int(time.time()) - epoch) < timeout) and (
(self.isSubscriptionProcessDone() == False))):
try:
time.sleep(1)
self.logm("MWorkerLib:subscribeCollectionSynchron",
"inside waiting cycle :" + str(self.meteorClient.subscriptions.keys()))
except KeyboardInterrupt:
break
if (self.isSubscriptionProcessDone() == False):
self.logm("MWorkerLib:subscribeCollectionSynchron",
"some requested subscription failed...:" + str(self.subscription_status))
return False
else:
self.logm("MWorkerLib:subscribeCollectionSynchron", "requested subscription successfuly subscribed...")
return True
def getUnassigned(self):
"""
get list of unassigned tasks
:return: list of unassigned tasks
"""
self.logm("MWorkerLib:getUnassigned", "geting all unassigned")
all_posts = self.meteorClient.find('unassigned')
self.logm("MWorkerLib:getUnassigned", "found following unassigned:" + str(all_posts))
def subscription_callback(self, error):
if error:
self.logm("MWorkerLib:subscription_callback", "subsribing to unassigned failed")
return
self.logm("MWorkerLib:subscription_callback", "subsribing to unassigned succeed")
def logm(self, tag="undefined tag", text="undefined text"):
"""
logger wrapper ....
:param tag: tag
:param level: level ( 1..10 )
:param text: dscription of logging
:return:
"""
timestamp = self.getTimestamp()
print("{d} t:{a} t:{c} ".format(a=tag, c=text, d=timestamp))
####################################### support methods
def getTimestamp(self):
ts = time.time()
return datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
def callViewToolkit(self, method, parameters, callback):
"""
this will execute call...
please note, for simple geting collection data, no need to create method...
:param method: name of function
:param parameters: what to give to viewtoolkit
:callback - as all meteor calls are async, callback is required
def callback_function(error, result):
if error:
print(error)
return
print(result)
:return:
"""
self.logm("MWorkerLib:callViewToolkit", "method:" + method + " with parameters:" + parameters + " called")
self.meteorClient.call(method, parameters, callback)
def isPluginActiveInPluginManager(self, pluginname):
"""
this is to check if plugin which is parameter exist in worker, and if yes, if it is activated...
:param pluginname:
:return: true in case plugin is activated
"""
for pluginInfo in self.multiprocessPluginManager.getAllPlugins():
self.logm("MWorkerLib:isPlugginActiveInPluginManager:",
"checking plugin " + pluginInfo.name + "against " + pluginname)
if (pluginname == pluginInfo.name):
self.logm("MWorkerLib:isPlugginActiveInPluginManager:",
"plugin was found, let see if it is activated " + str(pluginInfo.is_activated))
if pluginInfo.is_activated:
self.logm("MWorkerLib:isPlugginActiveInPluginManager:",
"yes, plugin exist and was activated. returning true")
return True
self.logm("MWorkerLib:isPlugginActiveInPluginManager:",
"could not found plugin you uare asking for, or it is not activated, returning False")
return False
def activatePlugin(self, pluginname):
"""
plugin activation itself
:param pluginname:
:return:
"""
self.logm("MWorkerLib:isPlugginActiveInPluginManager:activatePlugin",
"plugin {a} will be activated now...".format(a=pluginname))
self.multiprocessPluginManager.activatePluginByName(pluginname)
self.logm("MWorkerLib:isPlugginActiveInPluginManager:activatePlugin",
"plugin {a} was activated...".format(a=pluginname))
def moveJobFromActiveToTrash(self,jobname):
"""
will move job by its name from active to trash
:param jobname:
:return:
"""
#kontrola, zda trash ma v sobe vytvoreneho workera:
if(self.isWorkerInTrash()==False):
self.createWorkerRecordInTrash()
job=self.getJobInActiveByName(jobname)
if (job==None):
return False
if self.createJobInTrashSynchronised(job):
self.deleteJobFromActiveSynchronised(jobname)
def deleteJobFromActiveSynchronised(self,jobname):
"""
delete job from active
:param jobname:
:return:
"""
result = False
self.syncwaitdone = False # zacatek locku
self.logm("MWorkerLib:deleteJobFromActiveSynchronised",
"worker:" + self.workername + " is going to remove job from active " + str(jobname))
self.meteorClient.remove( 'active',{'_id':self.workername,'jobs._id':jobname}, callback=self.callback_synchro)
res = self.syncWait() # konec locku
self.logm("MWorkerLib:deleteJobFromActiveSynchronised",
"worker:" + self.workername + " removed " + str(jobname)+ "result is "+str(res))
return res
def createJobInTrashSynchronised(self,job):
"""
this will create job in trash collection under worker name...
:param job:
:return:
"""
#inseritng job
self.logm("MWorkerLib:createJobInTrash",
"worker:" + self.workername + " is going to insert to trash following: " + str(job))
self.syncwaitdone = False # zacatek locku
self.meteorClient.update('trash', {'_id': self.workername}, {'$push': {"jobs":job}},
callback=self.callback_synchro)
res = self.syncWait() # konec locku
return res
def getJobInActiveByName(self,jobname):
"""
this will get job back
:param jobname: name of job in active ...
:return: return job, or None
"""
res = self.meteorClient.find_one('active',{'_id':self.workername,'jobs.jobname':jobname})
if res==[]:
return None
return res
def moveJobFromUnassignedToActiveSynchronised(self, jobname):
"""
this will shift job from unassigned to active... based on job name, which must be unique in hwhile system
:param jobname:
:return:
"""
#kontrola, zda existuje worker v active, a kdyz ne, tak se vlozi podle template
if(self.isWorkerInActive()==False):
self.logm("MWorkerLib:moveJobFromUnassignedToActiveSynchronised",
" dont see record for worker {w} in active !! creating it ".format(w=self.workername))
self.createWorkerRecordInActive()
resulttmp = self.findJobinUnassigned(jobname)
if(len(resulttmp)!=1):
raise NoJobFound(" given job name "+jobname+" doesnt exist")
job=resulttmp[0]
self.logm("MWorkerLib:moveJobFromUnassignedToActiveSynchronised",
" job {a} will be activated now...".format(a=str(job)))
if (self.createMyJobInActiveSynchronised(job)):
if(self.deleteJobFromUnassignedSynchronised(job)):
self.logm("MWorkerLib:moveJobFromUnassignedToActiveSynchronised",
" job inserting to db is ok, job deleted from unassigned ")
return True
self.logm("MWorkerLib:moveJobFromUnassignedToActiveSynchronised",
"job inserting to db failed")
return False
def isWorkerInTrash(self):
"""
will check is self.workername can be found in trash
:return: true if it can be found, false if not
"""
result = self.meteorClient.find('trash',{"_id":self.workername})
self.logm("MWorkerLib:isWorkerInTrash",
"checking, if i see my worker name in trash "+str(result))
if len(result)==1:
self.logm("MWorkerLib:isWorkerInTrash",
" worker with my name found in trash, erturning true")
return True
else:
self.logm("MWorkerLib:isWorkerInTrash",
"found nothing, returning False")
return False
def isWorkerInActive(self):
"""
will check is self.workername can be found in active
:return: true if it can be found, false if not
"""
result = self.meteorClient.find('active',{"_id":self.workername})
self.logm("MWorkerLib:isWorkerInActive",
"checking, if i see my worker name in active "+str(result))
if len(result)==1:
self.logm("MWorkerLib:isWorkerInActive",
" worker with my name found in active, erturning true")
return True
else:
self.logm("MWorkerLib:isWorkerInActive",
"found nothing, returning False")
return False
def createWorkerRecordInActive(self):
"""
will create new worker - it will not be checking if there is already worker created in active...
:return:
"""
template={"_id":self.workername,"worker":self.workername,"workerusername":"******","jobs":[]}
self.logm("MWorkerLib:createWorkerRecordInActive",
"worker:" + self.workername + " is going to by added to collection active ")
self.syncwaitdone = False # zacatek locku
self.meteorClient.insert('active', template,
callback=self.callback_synchro)
res = self.syncWait() # konec locku
return res
def createWorkerRecordInTrash(self):
"""
will create new worker - it will not be checking if there is already worker created in active...
:return:
"""
template={"_id":self.workername,"worker":self.workername,"workerusername":"******","jobs":[]}
self.logm("MWorkerLib:createWorkerRecordInTrash",
"worker:" + self.workername + " is going to by added to collection trash ")
self.syncwaitdone = False # zacatek locku
self.meteorClient.insert('trash', template,
callback=self.callback_synchro)
res = self.syncWait() # konec locku
return res
def findJobinUnassigned(self, jobname):
"""
will return job from unassigned by jobname
this is synchron method... it will wait for ( synchron_timeout ) seconds, and return false
:param job:
:return: false on timeout ( synchron_timeout ) or if return is different from 1.
"""
return self.meteorClient.find('unassigned', {"_id": jobname})
def createMyJobInActiveSynchronised(self, job):
"""
this will take as input dict of job... and add it to jobs[] in proper worker...
:param job: dict of job...
:return: false on timeout ( synchron_timeout ) or if return is different from 1.
"""
template = {
'pluginname': job['pluginname'],
'jobname': job['jobname'],
'starttime': int(time.time()), # epocha
'ttl': job['expectedttl'],
'inputparams': job['inputparams'],
'progress': 0,
'result': "undefined"
}
self.logm("MWorkerLib:createMyJobInActive",
"worker:" + self.workername + " is going to insert to active following: " + str(job))
self.syncwaitdone = False # zacatek locku
self.meteorClient.update('active', {'_id': self.workername}, {'$push': {'jobs': template}},
callback=self.callback_synchro)
res = self.syncWait() # konec locku
return res
def callback_synchro(self, error, data):
self.data = data
self.error = error
if error:
self.logm("MWorkerLib:update_callback_synchro",
"worker:" + self.workername + " failed error is " + str(error))
self.syncwaitdone = True
return
self.logm("MWorkerLib:update_callback_synchro", "is ok. number of updates is " + str(data))
self.syncwaitdone = True
def syncWait(self):
"""
this will wait till data of error are not empty.
it will do for self.SYNCHRO_TIMEOUT seconds. then it will return False
also it will check what is return value from request. if update is 0,
false, othervice true
"""
epoch = int(time.time())
while (((int(time.time()) - epoch) < self.SYNCHRO_TIMEOUT) and (self.syncwaitdone == False)):
try:
time.sleep(1)
self.logm("MWorkerLib:syncWait",
"inside waiting cycle :")
except KeyboardInterrupt:
break
if (self.syncwaitdone == False):
# cycle was broken before timeou
return False
try:
# zkouska, zda se updatoval aspon jeden radek
if (int(self.data) == 0):
# mongo changed nothing, but there was no error
return False
else:
return True
except:
pass
# nothing found, cykluus end on timeout
return False
def deleteJobFromUnassignedSynchronised(self, job):
"""
this will delete selected job from unassigned
this is synchron method... it will wait for ( synchron_timeout ) seconds, and return false
:param job:
:return: false on timeout ( synchron_timeout ) or if return is different from 1.
"""
self.syncwaitdone = False # zacatek locku
self.logm("MWorkerLib:createMyJobInActive",
"worker:" + self.workername + " is going to remove job from unassigned " + str(job))
self.meteorClient.remove('unassigned', {'_id': job['_id']}, callback=self.callback_synchro)
res = self.syncWait() # konec locku
self.logm("MWorkerLib:createMyJobInActive",
"worker:" + self.workername + " removed " + str(job)+ "result is "+str(res))
return res
#########################testing
def testScenario1(self):
"""
this is scenario 1...
this will move job from unassigned to active, then from active to trash...
also it will list unassigned tasks
:return:
self.logm("MWorkerLib:testScenario1", "start test scenario 1")
self.logm("MWorkerLib:testScenario1", "moving from unassigned to active")
all_posts = self.meteorClient.find('unassigned')
#job = (all_posts).keys()
#get first unassigned:
firstUnassigned=self.meteorClient.find('unassigned')[0]
ttltmp=100
#pro testovaci ucely jsem vytvoril v active workera:
# db.active.insert({"_id":"localhostworker","worker":"localhostworker","workerusername":"******","jobs":[]})
template={
'pluginname':firstUnassigned['pluginname'],
'starttime':int(time.time()), #epocha
'ttl':ttltmp,
'inputparams':firstUnassigned['inputparams'],
'progress':0,
'result':"undefined"
}
#insert to active:
#musim pouzit id_ protoze klient rve ze kod je insecure a musi se pouzit id
#s _id je dalsi problem. protoze interne mongodb pouziva objekt, ne stringu
#tak bude vyhodnejsi nahradit pri vytvoreni workeru v active _id s nejakym retezcem _
worker=self.meteorClient.find_one('active',{"worker" : "localhostworker"})
#worker["workerusername"]="******"
self.logm("MWorkerLib:testScenario1", "worker:"+str(worker)+" "+worker['_id'])
worker1=self.meteorClient.find_one('active',{"_id":worker["_id"]})
self.logm("MWorkerLib:testScenario1", "worker:"+str(worker1)+" "+worker1['_id'])
self.meteorClient.update('active',{'_id':worker['_id']},{'$push':{'jobs':template}},callback=update_callback)
#self.infiniteCycle()
self.logm("MWorkerLib:testScenario1", "deleting job from unassigned")
self.meteorClient.remove('unassigned',{'_id':firstUnassigned['_id']},callback=remove_callback)
time.sleep(5)
"""
job = {'_id': "asdasdsad", 'jobname': "asdasd", 'pluginname': "asdasda", 'inputparams': "assad",
'expectedttl': "100"}
#otestovane a funkcni 11.2.2016:
#tohle presune job s danym jmenem do active pod jmeno mistniho workera...
#self.moveJobFromUnassignedToActiveSynchronised("blabla")
self.moveJobFromActiveToTrash("blabla")
class NuoCA(object):
"""
NuoDB Collection Agent
"""
def __init__(self, config_file=None, collection_interval=30,
plugin_dir=None, starttime=None, verbose=False,
self_test=False, log_level=logging.INFO,
output_values=None):
"""
:param config_file: Path to NuoCA configuration file.
:type config_file: ``str``
:param collection_interval: Collection Interval in seconds
:type collection_interval: ``int``
:param plugin_dir: Path to NuoCA Plugin Directory
:type plugin_dir: ``str``
:param starttime: Epoch timestamp of start time of the first collection.
:type starttime: ``int``, ``None``
:param verbose: Flag to indicate printing of verbose messages to stdout.
:type verbose: ``bool``
:param self_test: Flag to indicate a 5 loop self test.
:type self_test: ``bool``
:param log_level: Python logging level
:type log_level: ``logging.level``
:param output_values: list of strings parsable by utils.parse_keyval_list()
:type output_values: `list` of `str`
"""
self._starttime = starttime
if self._starttime:
# Make sure that starttime is now or in the future.
current_timestamp = nuoca_gettimestamp()
if current_timestamp >= self._starttime:
msg = "starttime must be now or in the future."
nuoca_log(logging.ERROR, msg)
raise AttributeError(msg)
self._config = NuocaConfig(config_file)
initialize_logger(self._config.NUOCA_LOGFILE)
nuoca_set_log_level(log_level)
nuoca_log(logging.INFO, "nuoca server init.")
self._collection_interval = collection_interval
if not starttime:
self._starttime = None
else:
self._starttime = int(starttime)
self._plugin_topdir = plugin_dir
self._enabled = True
self._verbose = verbose # Used to make stdout verbose.
self._self_test = self_test
self._output_values = parse_keyval_list(output_values)
if self._output_values:
for output_val in self._output_values:
nuoca_log(logging.INFO, "Output Key: '%s' set to Value: '%s'" %
(output_val, self._output_values[output_val]) )
# The following self._*_plugins are dictionaries of two element
# tuples in the form: (plugin object, plugin configuration) keyed
# by the plugin name.
self._input_plugins = {}
self._output_plugins = {}
self._transform_plugins = {}
if not self._plugin_topdir:
self._plugin_topdir = os.path.join(get_nuoca_topdir(),
"plugins")
nuoca_log(logging.INFO, "plugin dir: %s" % self._plugin_topdir)
input_plugin_dir = os.path.join(self._plugin_topdir, "input")
output_plugin_dir = os.path.join(self._plugin_topdir, "output")
transform_plugin_dir = os.path.join(self._plugin_topdir, "transform")
self._plugin_directories = [input_plugin_dir,
output_plugin_dir,
transform_plugin_dir]
@property
def config(self):
return self._config
def _collection_cycle(self, collection_time):
"""
_collection_cycle is called at the end of each Collection
Interval.
"""
nuoca_log(logging.INFO, "Starting collection interval: %s" %
collection_time)
collected_inputs = self._collect_inputs()
for list_item in collected_inputs:
list_item['collection_interval'] = self._collection_interval
if 'timestamp' not in list_item:
list_item['timestamp'] = collection_time
# TODO Transformations
self._store_outputs(list_item)
def _get_activated_input_plugins(self):
"""
Get a list of "activated" input plugins
"""
input_list = self.manager.getPluginsOfCategory('Input')
activated_list = [x for x in input_list if x.is_activated]
return activated_list
def _get_activated_output_plugins(self):
"""
Get a list of "activated" output plugins
"""
output_list = self.manager.getPluginsOfCategory('Output')
activated_list = [x for x in output_list if x.is_activated]
return activated_list
def _get_plugin_respose(self, a_plugin):
"""
Get the response message from the plugin
:return: Response dictionary if successful, otherwise None.
"""
plugin_obj = a_plugin.plugin_object
# noinspection PyBroadException
try:
if plugin_obj.child_pipe.poll(self.config.PLUGIN_PIPE_TIMEOUT):
response = plugin_obj.child_pipe.recv()
if self._verbose:
print("%s:%s" % (a_plugin.name, response))
else:
nuoca_log(logging.ERROR,
"NuoCA._get_plugin_respose: "
"Timeout collecting response values from plugin: %s"
% a_plugin.name)
return None
except Exception as e:
nuoca_log(logging.ERROR,
"NuoCA._get_plugin_respose: "
"Unable to collect response from plugin: %s\n%s"
% (a_plugin.name, str(e)))
return None
# noinspection PyBroadException
try:
if not response:
nuoca_log(logging.ERROR,
"NuoCA._get_plugin_respose: "
"Missing response from plugin: %s"
% a_plugin.name)
return None
if 'status_code' not in response:
nuoca_log(logging.ERROR,
"NuoCA._get_plugin_respose: "
"status_code missing from plugin response: %s"
% a_plugin.name)
return None
except Exception as e:
nuoca_log(logging.ERROR,
"NuoCA._get_plugin_respose: Error attempting to collect"
" response from plugin: %s\n%s"
% (a_plugin.name, str(e)))
return None
return response
def _startup_plugin(self, a_plugin, config=None):
"""
Send start message to plugin.
:param a_plugin: The plugin
:param config: NuoCA Configuration
:type config: ``dict``
"""
response = None
nuoca_log(logging.INFO, "Called to start plugin: %s" % a_plugin.name)
plugin_msg = {'action': 'startup', 'config': config}
try:
a_plugin.plugin_object.child_pipe.send(plugin_msg)
except Exception as e:
nuoca_log(logging.ERROR,
"Unable to send %s message to plugin: %s\n%s"
% (plugin_msg, a_plugin.name, str(e)))
try:
response = self._get_plugin_respose(a_plugin)
except Exception as e:
nuoca_log(logging.ERROR,
"Problem with response on %s message to plugin: %s\n%s"
% (plugin_msg, a_plugin.name, str(e)))
if not response or response['status_code'] != 0:
nuoca_log(logging.ERROR,
"Disabling plugin that failed to startup: %s"
% a_plugin.name)
self.manager.deactivatePluginByName(a_plugin.name, a_plugin.category)
self._shutdown_plugin(a_plugin)
@staticmethod
def _exit_plugin(a_plugin):
"""
Send Exit message to plugin.
:param a_plugin: The plugin
"""
nuoca_log(logging.INFO, "Called to exit plugin: %s" % a_plugin.name)
plugin_msg = {'action': 'exit'}
try:
a_plugin.plugin_object.child_pipe.send(plugin_msg)
except Exception as e:
nuoca_log(logging.ERROR,
"Unable to send %s message to plugin: %s\n%s"
% (plugin_msg, a_plugin.name, str(e)))
@staticmethod
def _shutdown_plugin(a_plugin):
"""
Send stop message to plugin.
:param a_plugin: The plugin
:type a_plugin: NuocaMPPlugin
"""
nuoca_log(logging.INFO, "Called to shutdown plugin: %s" % a_plugin.name)
plugin_msg = {'action': 'shutdown'}
try:
a_plugin.plugin_object.child_pipe.send(plugin_msg)
except Exception as e:
nuoca_log(logging.ERROR,
"Unable to send %s message to plugin: %s\n%s"
% (plugin_msg, a_plugin.name, str(e)))
def _collect_inputs(self):
"""
Collect time-series data from each activated plugin.
:return: ``dict`` of time-series data
"""
# TODO - Use Threads so that we can do concurrent collection.
plugin_msg = {'action': 'collect',
'collection_interval': self._collection_interval}
rval = []
activated_plugins = self._get_activated_input_plugins()
for a_plugin in activated_plugins:
# noinspection PyBroadException
try:
a_plugin.plugin_object.child_pipe.send(plugin_msg)
except Exception as e:
nuoca_log(logging.ERROR,
"NuoCA._collect_inputs: "
"Unable to send %s message to plugin: %s\n%s"
% (plugin_msg, a_plugin.name, str(e)))
for a_plugin in activated_plugins:
response = self._get_plugin_respose(a_plugin)
if not response:
continue
resp_values = response['resp_values']
# noinspection PyBroadException
try:
if 'collected_values' not in resp_values:
nuoca_log(logging.ERROR,
"NuoCA._collect_inputs: "
"'Collected_Values' missing in response from plugin: %s"
% a_plugin.name)
continue
if not resp_values['collected_values']:
nuoca_log(logging.DEBUG,
"No time-series values were collected from plugin: %s"
% a_plugin.name)
continue
if type(resp_values['collected_values']) is not list:
nuoca_log(logging.ERROR,
"NuoCA._collect_inputs: "
"'Collected_Values' is not a list in "
"response from plugin: %s"
% a_plugin.name)
continue
list_count = len(resp_values['collected_values'])
for list_index in range(list_count):
new_values = {}
key_prefix = a_plugin.name
collected_dict = resp_values['collected_values'][list_index]
if 'nuocaCollectionName' in collected_dict:
key_prefix = collected_dict['nuocaCollectionName']
del collected_dict['nuocaCollectionName']
for collected_item in collected_dict:
key_name = key_prefix + '.' + collected_item
new_values[key_name] = collected_dict[collected_item]
if collected_item == 'TimeStamp':
new_values['timestamp'] = int(collected_dict[collected_item])
if self._output_values:
new_values.update(self._output_values)
rval.append(new_values)
except Exception as e:
nuoca_log(logging.ERROR,
"NuoCA._collect_inputs: "
"Error attempting to collect"
" response from plugin: %s\n%s"
% (a_plugin.name, str(e)))
return rval
def _store_outputs(self, collected_inputs):
if not collected_inputs:
return
rval = {}
plugin_msg = {'action': 'store', 'ts_values': collected_inputs}
activated_plugins = self._get_activated_output_plugins()
for a_plugin in activated_plugins:
# noinspection PyBroadException
try:
a_plugin.plugin_object.child_pipe.send(plugin_msg)
except Exception as e:
nuoca_log(logging.ERROR,
"Unable to send 'Store' message to plugin: %s\n%s"
% (a_plugin.name, str(e)))
for a_plugin in activated_plugins:
resp_values = self._get_plugin_respose(a_plugin)
if not resp_values:
continue
return rval
def _create_plugin_manager(self):
self.manager = MultiprocessPluginManager(
directories_list=self._plugin_directories,
plugin_info_ext="multiprocess-plugin")
self.manager.setCategoriesFilter({
"Input": NuocaMPInputPlugin,
"Output": NuocaMPOutputPlugin,
"Transform": NuocaMPTransformPlugin
})
# Activate plugins and call the plugin's startup() method.
def _activate_and_startup_plugins(self):
for input_plugin in self.config.INPUT_PLUGINS:
input_plugin_name = input_plugin.keys()[0]
if not self.manager.activatePluginByName(input_plugin_name, 'Input'):
err_msg = "Cannot activate input plugin: '%s', Skipping." % \
input_plugin_name
nuoca_log(logging.WARNING, err_msg)
else:
a_plugin = self.manager.getPluginByName(input_plugin_name, 'Input')
if a_plugin:
input_plugin_config = input_plugin.values()[0]
if not input_plugin_config:
input_plugin_config = {}
input_plugin_config['nuoca_start_ts'] = self._starttime
input_plugin_config['nuoca_collection_interval'] = \
self._collection_interval
self._startup_plugin(a_plugin, input_plugin_config)
self._input_plugins[input_plugin_name] = (a_plugin,
input_plugin_config)
for output_plugin in self.config.OUTPUT_PLUGINS:
output_plugin_name = output_plugin.keys()[0]
if not self.manager.activatePluginByName(output_plugin_name, 'Output'):
err_msg = "Cannot activate output plugin: '%s', Skipping." % \
output_plugin_name
nuoca_log(logging.WARNING, err_msg)
else:
a_plugin = self.manager.getPluginByName(output_plugin_name, 'Output')
if a_plugin:
output_plugin_config = output_plugin.values()[0]
if not output_plugin_config:
output_plugin_config = {}
output_plugin_config['nuoca_start_ts'] = self._starttime
output_plugin_config['nuoca_collection_interval'] = \
self._collection_interval
self._startup_plugin(a_plugin, output_plugin_config)
self._output_plugins[output_plugin_name] = (a_plugin,
output_plugin_config)
# TODO Transform Plugins
# test if the plugin name is configured in NuoCA.
def _is_plugin_name_configured(self, name):
for configured_input in self.config.INPUT_PLUGINS:
if name in configured_input:
return True
for configured_input in self.config.OUTPUT_PLUGINS:
if name in configured_input:
return True
for configured_input in self.config.TRANSFORM_PLUGINS:
if name in configured_input:
return True
return False
# activate only the NuoCA configured plugins.
def _activate_configured_plugins(self):
self.manager.locatePlugins()
# get a list of ALL plugin candidates
plugin_candidates = self.manager.getPluginCandidates()
for candidate in plugin_candidates:
plugin_configured = self._is_plugin_name_configured(candidate[2].name)
if not plugin_configured:
# Remove this plugin candidate because it is no configued by NuoCA
self.manager.removePluginCandidate(candidate)
self.manager.loadPlugins()
self._activate_and_startup_plugins()
def _shutdown_all_plugins(self):
for input_plugin in self._input_plugins:
self.manager.deactivatePluginByName(input_plugin, 'Input')
a_plugin = self.manager.getPluginByName(input_plugin, 'Input')
self._shutdown_plugin(a_plugin)
for output_plugin in self._output_plugins:
self.manager.deactivatePluginByName(output_plugin, 'Output')
a_plugin = self.manager.getPluginByName(output_plugin, 'Output')
self._shutdown_plugin(a_plugin)
# TODO Transform Plugins
@staticmethod
def kill_all_plugin_processes(manager, timeout=5):
"""
Kill any plugin processes that were left running after waiting up to
the timeout value..
:param manager: MultiprocessPluginManager
:type manager: MultiprocessPluginManager
:param timeout: Maximum time to wait (in seconds) for the process to
self exit before killing.
:type timeout: ``int``
"""
if not manager:
return
all_plugins = manager.getAllPlugins()
wait_count = timeout
for a_plugin in all_plugins:
while a_plugin.plugin_object.proc.is_alive() and wait_count > 0:
time.sleep(1)
wait_count -= 1
if a_plugin.plugin_object.proc.is_alive():
nuoca_log(logging.INFO, "Killing plugin subprocess: %s" % a_plugin)
a_plugin.plugin_object.proc.terminate()
def _remove_all_plugins(self, timeout=5):
"""
Remove all plugins
:param timeout: Maximum seconds to wait for subprocess to exit.
:type timeout: ``int``
"""
for input_plugin in self._input_plugins:
a_plugin = self.manager.getPluginByName(input_plugin, 'Input')
self._exit_plugin(a_plugin)
for output_plugin in self._output_plugins:
a_plugin = self.manager.getPluginByName(output_plugin, 'Output')
self._exit_plugin(a_plugin)
# TODO Transform Plugins
# At this point all configured plugin subprocesses should be exiting
# on their own. However, if there is any plugin subprocess that didn't
# exit for any reason, we must terminate them so we don't hang the
# NuoCA process at exit.
NuoCA.kill_all_plugin_processes(self.manager, timeout)
def start(self):
"""
Startup NuoCA
"""
self._create_plugin_manager()
self._activate_configured_plugins()
interval_sync = IntervalSync(interval=self._collection_interval,
seed_ts=self._starttime)
# Collection Interval Loop
loop_count = 0
while self._enabled:
loop_count += 1
collection_timestamp = interval_sync.wait_for_next_interval()
self._collection_cycle(collection_timestamp * 1000)
if self._self_test:
if loop_count >= self._config.SELFTEST_LOOP_COUNT:
self._enabled = False
def shutdown(self, timeout=5):
"""
Shutdown NuoCA
:param timeout: Maximum seconds to wait for subprocess to exit.
:type timeout: ``int``
"""
nuoca_log(logging.INFO, "nuoca server shutdown")
self._shutdown_all_plugins()
self._remove_all_plugins(timeout)
nuoca_logging_shutdown()