class SimpleMultiprocessTestCase(unittest.TestCase): """ Test the correct loading of a multiprocessed plugin as well as basic communication. """ def setUp(self): """ init """ # create the plugin manager self.mpPluginManager = MultiprocessPluginManager(directories_list=[ os.path.join( os.path.dirname(os.path.abspath(__file__)),"plugins")], plugin_info_ext="multiprocess-plugin") # load the plugins that may be found self.mpPluginManager.collectPlugins() # Will be used later self.plugin_info = None def testUpAndRunning(self): """ Test if the plugin is loaded and if the communication pipe is properly setuped. """ numTestedPlugins = 0 for plugin in self.mpPluginManager.getAllPlugins(): content_from_parent = "hello-from-parent" content_from_child = False plugin.plugin_object.child_pipe.send(content_from_parent) if plugin.plugin_object.child_pipe.poll(5): content_from_child = plugin.plugin_object.child_pipe.recv() self.assertEqual(content_from_child, "{0}|echo_from_child".format(content_from_parent)) numTestedPlugins += 1 self.assertTrue(numTestedPlugins >= 1)
def _create_plugin_manager(self): self.manager = MultiprocessPluginManager( directories_list=self._plugin_directories, plugin_info_ext="multiprocess-plugin") self.manager.setCategoriesFilter({ "Input": NuocaMPInputPlugin, "Output": NuocaMPOutputPlugin, "Transform": NuocaMPTransformPlugin })
def setUp(self): """ init """ # create the plugin manager self.mpPluginManager = MultiprocessPluginManager( directories_list=[ os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugins") ], plugin_info_ext="multiprocess-plugin") # load the plugins that may be found self.mpPluginManager.collectPlugins() # Will be used later self.plugin_info = None
def runTest(self): topdir = nuoca_util.get_nuoca_topdir() output_plugin_dir = os.path.join(topdir, "plugins/output") dir_list = [output_plugin_dir] self._PrinterPluginTest() self.manager = MultiprocessPluginManager( directories_list=dir_list, plugin_info_ext="multiprocess-plugin") self._MultiprocessPluginManagerTest()
def runTest(self): topdir = nuoca_util.get_nuoca_topdir() input_plugin_dir = os.path.join(topdir, "plugins/input") dir_list = [input_plugin_dir] self._LogstashPluginTest("06a32504-c2c9-41bc-9b48-030982c5ea43.r0db0") self._LogstashPluginTest("fa2461c7-bca2-4df5-91e3-251084e1b8d1.r0db2") self.manager = MultiprocessPluginManager( directories_list=dir_list, plugin_info_ext="multiprocess-plugin") self._MultiprocessPluginManagerCompareTest()
def initYapsy(self): """ inicialisation of yapsi subsystems... :return: """ self.multiprocessPluginManager = MultiprocessPluginManager() self.multiprocessPluginManager.setPluginPlaces(["plugins"]) self.multiprocessPluginManager.collectPlugins() self.logm("MWorkerLib:initYapsy:", "following pluggins ready") for pluginInfo in self.multiprocessPluginManager.getAllPlugins(): self.logm("MWorkerLib:initYapsy:", ">" + pluginInfo.name)
def setUp(self): """ init """ # create the plugin manager self.mpPluginManager = MultiprocessPluginManager(directories_list=[ os.path.join( os.path.dirname(os.path.abspath(__file__)),"plugins")], plugin_info_ext="multiprocess-plugin") # load the plugins that may be found self.mpPluginManager.collectPlugins() # Will be used later self.plugin_info = None
class SimpleMultiprocessTestCase(unittest.TestCase): """ Test the correct loading of a multiprocessed plugin as well as basic communication. """ def setUp(self): """ init """ # create the plugin manager self.mpPluginManager = MultiprocessPluginManager( directories_list=[ os.path.join(os.path.dirname(os.path.abspath(__file__)), "plugins") ], plugin_info_ext="multiprocess-plugin") # load the plugins that may be found self.mpPluginManager.collectPlugins() # Will be used later self.plugin_info = None def testUpAndRunning(self): """ Test if the plugin is loaded and if the communication pipe is properly setuped. """ for plugin_index, plugin in enumerate( self.mpPluginManager.getAllPlugins()): child_pipe = plugin.plugin_object.child_pipe content_from_parent = "hello-{0}-from-parent".format(plugin_index) child_pipe.send(content_from_parent) content_from_child = False if child_pipe.poll(5): content_from_child = child_pipe.recv() self.assertEqual("{0}|echo_from_child".format(content_from_parent), content_from_child) num_tested_plugin = plugin_index + 1 self.assertEqual(2, num_tested_plugin)
class MWorker: multiprocessPluginManager = "" # yapsy manager meteorClient = "" # python-meteor client # session=requests.Session() #http client meteorUrl = "ws://127.0.0.1:3000/websocket" CYCLE_DEFAULT_DELAY = 20 # 10 seconds SYNCHRO_TIMEOUT = 10 # meteor functions are asynchron. this is delay for how long i should wait till # it is considered as fail... 2 sec are default subscription_status = {} # this will assure, subscription is ready... "subscription_name":true/false workername = "localhostworker" # this is worker name - this must be initialised at moment of start, and # must by uniqued... data = "" error = "" syncwaitdone = False def __init__(self, meteorUrl): """ constructor :return: """ self.meteorUrl = meteorUrl self.logm("MWorker:constructor", "python meteor client initialisation:") self.initMeteorConnect() self.logm("MWorker:constructor", "getting unassigned") # self.infiniteCycle() self.getUnassigned() def infiniteCycle(self): """ this is to keep thread running - it can be interupted by ctrl+c :return: """ while True: try: time.sleep(1) except KeyboardInterrupt: break def initMeteorConnect(self): """ this will use library python-meteor in order to escablish session to selected meteor :return: """ self.logm("MWorkerLib:initMeteorConnect:", "meteor python lib client iniciaisation...") self.meteorClient = MeteorClient('ws://127.0.0.1:3000/websocket', auto_reconnect=True) self.meteorClient.connect() self.subscribeCollectionSynchron(['unassigned', 'active', 'trash'], self.SYNCHRO_TIMEOUT) self.logm("MWorkerLib:initMeteorConnect:", "meteor python lib client iniciaisation done...") def meteorCollectionSubscribed(self, subscription): self.subscription_status[subscription] = True; self.logm("MWorkerLib:subscribed:", 'SUBSCRIBED {}'.format(subscription)) def meteorConnected(self): self.logm("MWorkerLib:connected:", ' CONNECTED') def initYapsy(self): """ inicialisation of yapsi subsystems... :return: """ self.multiprocessPluginManager = MultiprocessPluginManager() self.multiprocessPluginManager.setPluginPlaces(["plugins"]) self.multiprocessPluginManager.collectPlugins() self.logm("MWorkerLib:initYapsy:", "following pluggins ready") for pluginInfo in self.multiprocessPluginManager.getAllPlugins(): self.logm("MWorkerLib:initYapsy:", ">" + pluginInfo.name) def isSubscriptionProcessDone(self): """ this will check, if in self.subscription_status all collections are set to true :return: true if all requested collections are subscribed, orthervice false.. """ kk = self.subscription_status.keys() for col in kk: if self.subscription_status[col] == False: self.logm("MWorkerLib:isSubscriptionProcessDone:", self.subscription_status) return False return True def subscribeCollectionSynchron(self, collections, timeout): """ this is synchron method. this means, there is loop for "timeout" seconds, so subscription can be estalished... :param: timeout - number of seconds for cycle this function will have :param: collections - array of collections to be subscribed with self.meteorClient... ['unassigned', 'active', 'trash'] :return: will return true, or collection if some is false """ self.logm("MWorkerLib:subscribeCollectionsynchron:", "begining with subscription") self.meteorClient.on('subscribed', self.meteorCollectionSubscribed) self.meteorClient.on('connected', self.meteorConnected) for col in collections: self.subscription_status[col] = False self.meteorClient.subscribe(col) self.logm("MWorkerLib:subscribeCollectionSynchron", "subscription init done, now waiting...:" + str(self.meteorClient.subscriptions.keys())) epoch = int(time.time()) while (((int(time.time()) - epoch) < timeout) and ( (self.isSubscriptionProcessDone() == False))): try: time.sleep(1) self.logm("MWorkerLib:subscribeCollectionSynchron", "inside waiting cycle :" + str(self.meteorClient.subscriptions.keys())) except KeyboardInterrupt: break if (self.isSubscriptionProcessDone() == False): self.logm("MWorkerLib:subscribeCollectionSynchron", "some requested subscription failed...:" + str(self.subscription_status)) return False else: self.logm("MWorkerLib:subscribeCollectionSynchron", "requested subscription successfuly subscribed...") return True def getUnassigned(self): """ get list of unassigned tasks :return: list of unassigned tasks """ self.logm("MWorkerLib:getUnassigned", "geting all unassigned") all_posts = self.meteorClient.find('unassigned') self.logm("MWorkerLib:getUnassigned", "found following unassigned:" + str(all_posts)) def subscription_callback(self, error): if error: self.logm("MWorkerLib:subscription_callback", "subsribing to unassigned failed") return self.logm("MWorkerLib:subscription_callback", "subsribing to unassigned succeed") def logm(self, tag="undefined tag", text="undefined text"): """ logger wrapper .... :param tag: tag :param level: level ( 1..10 ) :param text: dscription of logging :return: """ timestamp = self.getTimestamp() print("{d} t:{a} t:{c} ".format(a=tag, c=text, d=timestamp)) ####################################### support methods def getTimestamp(self): ts = time.time() return datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S') def callViewToolkit(self, method, parameters, callback): """ this will execute call... please note, for simple geting collection data, no need to create method... :param method: name of function :param parameters: what to give to viewtoolkit :callback - as all meteor calls are async, callback is required def callback_function(error, result): if error: print(error) return print(result) :return: """ self.logm("MWorkerLib:callViewToolkit", "method:" + method + " with parameters:" + parameters + " called") self.meteorClient.call(method, parameters, callback) def isPluginActiveInPluginManager(self, pluginname): """ this is to check if plugin which is parameter exist in worker, and if yes, if it is activated... :param pluginname: :return: true in case plugin is activated """ for pluginInfo in self.multiprocessPluginManager.getAllPlugins(): self.logm("MWorkerLib:isPlugginActiveInPluginManager:", "checking plugin " + pluginInfo.name + "against " + pluginname) if (pluginname == pluginInfo.name): self.logm("MWorkerLib:isPlugginActiveInPluginManager:", "plugin was found, let see if it is activated " + str(pluginInfo.is_activated)) if pluginInfo.is_activated: self.logm("MWorkerLib:isPlugginActiveInPluginManager:", "yes, plugin exist and was activated. returning true") return True self.logm("MWorkerLib:isPlugginActiveInPluginManager:", "could not found plugin you uare asking for, or it is not activated, returning False") return False def activatePlugin(self, pluginname): """ plugin activation itself :param pluginname: :return: """ self.logm("MWorkerLib:isPlugginActiveInPluginManager:activatePlugin", "plugin {a} will be activated now...".format(a=pluginname)) self.multiprocessPluginManager.activatePluginByName(pluginname) self.logm("MWorkerLib:isPlugginActiveInPluginManager:activatePlugin", "plugin {a} was activated...".format(a=pluginname)) def moveJobFromActiveToTrash(self,jobname): """ will move job by its name from active to trash :param jobname: :return: """ #kontrola, zda trash ma v sobe vytvoreneho workera: if(self.isWorkerInTrash()==False): self.createWorkerRecordInTrash() job=self.getJobInActiveByName(jobname) if (job==None): return False if self.createJobInTrashSynchronised(job): self.deleteJobFromActiveSynchronised(jobname) def deleteJobFromActiveSynchronised(self,jobname): """ delete job from active :param jobname: :return: """ result = False self.syncwaitdone = False # zacatek locku self.logm("MWorkerLib:deleteJobFromActiveSynchronised", "worker:" + self.workername + " is going to remove job from active " + str(jobname)) self.meteorClient.remove( 'active',{'_id':self.workername,'jobs._id':jobname}, callback=self.callback_synchro) res = self.syncWait() # konec locku self.logm("MWorkerLib:deleteJobFromActiveSynchronised", "worker:" + self.workername + " removed " + str(jobname)+ "result is "+str(res)) return res def createJobInTrashSynchronised(self,job): """ this will create job in trash collection under worker name... :param job: :return: """ #inseritng job self.logm("MWorkerLib:createJobInTrash", "worker:" + self.workername + " is going to insert to trash following: " + str(job)) self.syncwaitdone = False # zacatek locku self.meteorClient.update('trash', {'_id': self.workername}, {'$push': {"jobs":job}}, callback=self.callback_synchro) res = self.syncWait() # konec locku return res def getJobInActiveByName(self,jobname): """ this will get job back :param jobname: name of job in active ... :return: return job, or None """ res = self.meteorClient.find_one('active',{'_id':self.workername,'jobs.jobname':jobname}) if res==[]: return None return res def moveJobFromUnassignedToActiveSynchronised(self, jobname): """ this will shift job from unassigned to active... based on job name, which must be unique in hwhile system :param jobname: :return: """ #kontrola, zda existuje worker v active, a kdyz ne, tak se vlozi podle template if(self.isWorkerInActive()==False): self.logm("MWorkerLib:moveJobFromUnassignedToActiveSynchronised", " dont see record for worker {w} in active !! creating it ".format(w=self.workername)) self.createWorkerRecordInActive() resulttmp = self.findJobinUnassigned(jobname) if(len(resulttmp)!=1): raise NoJobFound(" given job name "+jobname+" doesnt exist") job=resulttmp[0] self.logm("MWorkerLib:moveJobFromUnassignedToActiveSynchronised", " job {a} will be activated now...".format(a=str(job))) if (self.createMyJobInActiveSynchronised(job)): if(self.deleteJobFromUnassignedSynchronised(job)): self.logm("MWorkerLib:moveJobFromUnassignedToActiveSynchronised", " job inserting to db is ok, job deleted from unassigned ") return True self.logm("MWorkerLib:moveJobFromUnassignedToActiveSynchronised", "job inserting to db failed") return False def isWorkerInTrash(self): """ will check is self.workername can be found in trash :return: true if it can be found, false if not """ result = self.meteorClient.find('trash',{"_id":self.workername}) self.logm("MWorkerLib:isWorkerInTrash", "checking, if i see my worker name in trash "+str(result)) if len(result)==1: self.logm("MWorkerLib:isWorkerInTrash", " worker with my name found in trash, erturning true") return True else: self.logm("MWorkerLib:isWorkerInTrash", "found nothing, returning False") return False def isWorkerInActive(self): """ will check is self.workername can be found in active :return: true if it can be found, false if not """ result = self.meteorClient.find('active',{"_id":self.workername}) self.logm("MWorkerLib:isWorkerInActive", "checking, if i see my worker name in active "+str(result)) if len(result)==1: self.logm("MWorkerLib:isWorkerInActive", " worker with my name found in active, erturning true") return True else: self.logm("MWorkerLib:isWorkerInActive", "found nothing, returning False") return False def createWorkerRecordInActive(self): """ will create new worker - it will not be checking if there is already worker created in active... :return: """ template={"_id":self.workername,"worker":self.workername,"workerusername":"******","jobs":[]} self.logm("MWorkerLib:createWorkerRecordInActive", "worker:" + self.workername + " is going to by added to collection active ") self.syncwaitdone = False # zacatek locku self.meteorClient.insert('active', template, callback=self.callback_synchro) res = self.syncWait() # konec locku return res def createWorkerRecordInTrash(self): """ will create new worker - it will not be checking if there is already worker created in active... :return: """ template={"_id":self.workername,"worker":self.workername,"workerusername":"******","jobs":[]} self.logm("MWorkerLib:createWorkerRecordInTrash", "worker:" + self.workername + " is going to by added to collection trash ") self.syncwaitdone = False # zacatek locku self.meteorClient.insert('trash', template, callback=self.callback_synchro) res = self.syncWait() # konec locku return res def findJobinUnassigned(self, jobname): """ will return job from unassigned by jobname this is synchron method... it will wait for ( synchron_timeout ) seconds, and return false :param job: :return: false on timeout ( synchron_timeout ) or if return is different from 1. """ return self.meteorClient.find('unassigned', {"_id": jobname}) def createMyJobInActiveSynchronised(self, job): """ this will take as input dict of job... and add it to jobs[] in proper worker... :param job: dict of job... :return: false on timeout ( synchron_timeout ) or if return is different from 1. """ template = { 'pluginname': job['pluginname'], 'jobname': job['jobname'], 'starttime': int(time.time()), # epocha 'ttl': job['expectedttl'], 'inputparams': job['inputparams'], 'progress': 0, 'result': "undefined" } self.logm("MWorkerLib:createMyJobInActive", "worker:" + self.workername + " is going to insert to active following: " + str(job)) self.syncwaitdone = False # zacatek locku self.meteorClient.update('active', {'_id': self.workername}, {'$push': {'jobs': template}}, callback=self.callback_synchro) res = self.syncWait() # konec locku return res def callback_synchro(self, error, data): self.data = data self.error = error if error: self.logm("MWorkerLib:update_callback_synchro", "worker:" + self.workername + " failed error is " + str(error)) self.syncwaitdone = True return self.logm("MWorkerLib:update_callback_synchro", "is ok. number of updates is " + str(data)) self.syncwaitdone = True def syncWait(self): """ this will wait till data of error are not empty. it will do for self.SYNCHRO_TIMEOUT seconds. then it will return False also it will check what is return value from request. if update is 0, false, othervice true """ epoch = int(time.time()) while (((int(time.time()) - epoch) < self.SYNCHRO_TIMEOUT) and (self.syncwaitdone == False)): try: time.sleep(1) self.logm("MWorkerLib:syncWait", "inside waiting cycle :") except KeyboardInterrupt: break if (self.syncwaitdone == False): # cycle was broken before timeou return False try: # zkouska, zda se updatoval aspon jeden radek if (int(self.data) == 0): # mongo changed nothing, but there was no error return False else: return True except: pass # nothing found, cykluus end on timeout return False def deleteJobFromUnassignedSynchronised(self, job): """ this will delete selected job from unassigned this is synchron method... it will wait for ( synchron_timeout ) seconds, and return false :param job: :return: false on timeout ( synchron_timeout ) or if return is different from 1. """ self.syncwaitdone = False # zacatek locku self.logm("MWorkerLib:createMyJobInActive", "worker:" + self.workername + " is going to remove job from unassigned " + str(job)) self.meteorClient.remove('unassigned', {'_id': job['_id']}, callback=self.callback_synchro) res = self.syncWait() # konec locku self.logm("MWorkerLib:createMyJobInActive", "worker:" + self.workername + " removed " + str(job)+ "result is "+str(res)) return res #########################testing def testScenario1(self): """ this is scenario 1... this will move job from unassigned to active, then from active to trash... also it will list unassigned tasks :return: self.logm("MWorkerLib:testScenario1", "start test scenario 1") self.logm("MWorkerLib:testScenario1", "moving from unassigned to active") all_posts = self.meteorClient.find('unassigned') #job = (all_posts).keys() #get first unassigned: firstUnassigned=self.meteorClient.find('unassigned')[0] ttltmp=100 #pro testovaci ucely jsem vytvoril v active workera: # db.active.insert({"_id":"localhostworker","worker":"localhostworker","workerusername":"******","jobs":[]}) template={ 'pluginname':firstUnassigned['pluginname'], 'starttime':int(time.time()), #epocha 'ttl':ttltmp, 'inputparams':firstUnassigned['inputparams'], 'progress':0, 'result':"undefined" } #insert to active: #musim pouzit id_ protoze klient rve ze kod je insecure a musi se pouzit id #s _id je dalsi problem. protoze interne mongodb pouziva objekt, ne stringu #tak bude vyhodnejsi nahradit pri vytvoreni workeru v active _id s nejakym retezcem _ worker=self.meteorClient.find_one('active',{"worker" : "localhostworker"}) #worker["workerusername"]="******" self.logm("MWorkerLib:testScenario1", "worker:"+str(worker)+" "+worker['_id']) worker1=self.meteorClient.find_one('active',{"_id":worker["_id"]}) self.logm("MWorkerLib:testScenario1", "worker:"+str(worker1)+" "+worker1['_id']) self.meteorClient.update('active',{'_id':worker['_id']},{'$push':{'jobs':template}},callback=update_callback) #self.infiniteCycle() self.logm("MWorkerLib:testScenario1", "deleting job from unassigned") self.meteorClient.remove('unassigned',{'_id':firstUnassigned['_id']},callback=remove_callback) time.sleep(5) """ job = {'_id': "asdasdsad", 'jobname': "asdasd", 'pluginname': "asdasda", 'inputparams': "assad", 'expectedttl': "100"} #otestovane a funkcni 11.2.2016: #tohle presune job s danym jmenem do active pod jmeno mistniho workera... #self.moveJobFromUnassignedToActiveSynchronised("blabla") self.moveJobFromActiveToTrash("blabla")
class NuoCA(object): """ NuoDB Collection Agent """ def __init__(self, config_file=None, collection_interval=30, plugin_dir=None, starttime=None, verbose=False, self_test=False, log_level=logging.INFO, output_values=None): """ :param config_file: Path to NuoCA configuration file. :type config_file: ``str`` :param collection_interval: Collection Interval in seconds :type collection_interval: ``int`` :param plugin_dir: Path to NuoCA Plugin Directory :type plugin_dir: ``str`` :param starttime: Epoch timestamp of start time of the first collection. :type starttime: ``int``, ``None`` :param verbose: Flag to indicate printing of verbose messages to stdout. :type verbose: ``bool`` :param self_test: Flag to indicate a 5 loop self test. :type self_test: ``bool`` :param log_level: Python logging level :type log_level: ``logging.level`` :param output_values: list of strings parsable by utils.parse_keyval_list() :type output_values: `list` of `str` """ self._starttime = starttime if self._starttime: # Make sure that starttime is now or in the future. current_timestamp = nuoca_gettimestamp() if current_timestamp >= self._starttime: msg = "starttime must be now or in the future." nuoca_log(logging.ERROR, msg) raise AttributeError(msg) self._config = NuocaConfig(config_file) initialize_logger(self._config.NUOCA_LOGFILE) nuoca_set_log_level(log_level) nuoca_log(logging.INFO, "nuoca server init.") self._collection_interval = collection_interval if not starttime: self._starttime = None else: self._starttime = int(starttime) self._plugin_topdir = plugin_dir self._enabled = True self._verbose = verbose # Used to make stdout verbose. self._self_test = self_test self._output_values = parse_keyval_list(output_values) if self._output_values: for output_val in self._output_values: nuoca_log(logging.INFO, "Output Key: '%s' set to Value: '%s'" % (output_val, self._output_values[output_val]) ) # The following self._*_plugins are dictionaries of two element # tuples in the form: (plugin object, plugin configuration) keyed # by the plugin name. self._input_plugins = {} self._output_plugins = {} self._transform_plugins = {} if not self._plugin_topdir: self._plugin_topdir = os.path.join(get_nuoca_topdir(), "plugins") nuoca_log(logging.INFO, "plugin dir: %s" % self._plugin_topdir) input_plugin_dir = os.path.join(self._plugin_topdir, "input") output_plugin_dir = os.path.join(self._plugin_topdir, "output") transform_plugin_dir = os.path.join(self._plugin_topdir, "transform") self._plugin_directories = [input_plugin_dir, output_plugin_dir, transform_plugin_dir] @property def config(self): return self._config def _collection_cycle(self, collection_time): """ _collection_cycle is called at the end of each Collection Interval. """ nuoca_log(logging.INFO, "Starting collection interval: %s" % collection_time) collected_inputs = self._collect_inputs() for list_item in collected_inputs: list_item['collection_interval'] = self._collection_interval if 'timestamp' not in list_item: list_item['timestamp'] = collection_time # TODO Transformations self._store_outputs(list_item) def _get_activated_input_plugins(self): """ Get a list of "activated" input plugins """ input_list = self.manager.getPluginsOfCategory('Input') activated_list = [x for x in input_list if x.is_activated] return activated_list def _get_activated_output_plugins(self): """ Get a list of "activated" output plugins """ output_list = self.manager.getPluginsOfCategory('Output') activated_list = [x for x in output_list if x.is_activated] return activated_list def _get_plugin_respose(self, a_plugin): """ Get the response message from the plugin :return: Response dictionary if successful, otherwise None. """ plugin_obj = a_plugin.plugin_object # noinspection PyBroadException try: if plugin_obj.child_pipe.poll(self.config.PLUGIN_PIPE_TIMEOUT): response = plugin_obj.child_pipe.recv() if self._verbose: print("%s:%s" % (a_plugin.name, response)) else: nuoca_log(logging.ERROR, "NuoCA._get_plugin_respose: " "Timeout collecting response values from plugin: %s" % a_plugin.name) return None except Exception as e: nuoca_log(logging.ERROR, "NuoCA._get_plugin_respose: " "Unable to collect response from plugin: %s\n%s" % (a_plugin.name, str(e))) return None # noinspection PyBroadException try: if not response: nuoca_log(logging.ERROR, "NuoCA._get_plugin_respose: " "Missing response from plugin: %s" % a_plugin.name) return None if 'status_code' not in response: nuoca_log(logging.ERROR, "NuoCA._get_plugin_respose: " "status_code missing from plugin response: %s" % a_plugin.name) return None except Exception as e: nuoca_log(logging.ERROR, "NuoCA._get_plugin_respose: Error attempting to collect" " response from plugin: %s\n%s" % (a_plugin.name, str(e))) return None return response def _startup_plugin(self, a_plugin, config=None): """ Send start message to plugin. :param a_plugin: The plugin :param config: NuoCA Configuration :type config: ``dict`` """ response = None nuoca_log(logging.INFO, "Called to start plugin: %s" % a_plugin.name) plugin_msg = {'action': 'startup', 'config': config} try: a_plugin.plugin_object.child_pipe.send(plugin_msg) except Exception as e: nuoca_log(logging.ERROR, "Unable to send %s message to plugin: %s\n%s" % (plugin_msg, a_plugin.name, str(e))) try: response = self._get_plugin_respose(a_plugin) except Exception as e: nuoca_log(logging.ERROR, "Problem with response on %s message to plugin: %s\n%s" % (plugin_msg, a_plugin.name, str(e))) if not response or response['status_code'] != 0: nuoca_log(logging.ERROR, "Disabling plugin that failed to startup: %s" % a_plugin.name) self.manager.deactivatePluginByName(a_plugin.name, a_plugin.category) self._shutdown_plugin(a_plugin) @staticmethod def _exit_plugin(a_plugin): """ Send Exit message to plugin. :param a_plugin: The plugin """ nuoca_log(logging.INFO, "Called to exit plugin: %s" % a_plugin.name) plugin_msg = {'action': 'exit'} try: a_plugin.plugin_object.child_pipe.send(plugin_msg) except Exception as e: nuoca_log(logging.ERROR, "Unable to send %s message to plugin: %s\n%s" % (plugin_msg, a_plugin.name, str(e))) @staticmethod def _shutdown_plugin(a_plugin): """ Send stop message to plugin. :param a_plugin: The plugin :type a_plugin: NuocaMPPlugin """ nuoca_log(logging.INFO, "Called to shutdown plugin: %s" % a_plugin.name) plugin_msg = {'action': 'shutdown'} try: a_plugin.plugin_object.child_pipe.send(plugin_msg) except Exception as e: nuoca_log(logging.ERROR, "Unable to send %s message to plugin: %s\n%s" % (plugin_msg, a_plugin.name, str(e))) def _collect_inputs(self): """ Collect time-series data from each activated plugin. :return: ``dict`` of time-series data """ # TODO - Use Threads so that we can do concurrent collection. plugin_msg = {'action': 'collect', 'collection_interval': self._collection_interval} rval = [] activated_plugins = self._get_activated_input_plugins() for a_plugin in activated_plugins: # noinspection PyBroadException try: a_plugin.plugin_object.child_pipe.send(plugin_msg) except Exception as e: nuoca_log(logging.ERROR, "NuoCA._collect_inputs: " "Unable to send %s message to plugin: %s\n%s" % (plugin_msg, a_plugin.name, str(e))) for a_plugin in activated_plugins: response = self._get_plugin_respose(a_plugin) if not response: continue resp_values = response['resp_values'] # noinspection PyBroadException try: if 'collected_values' not in resp_values: nuoca_log(logging.ERROR, "NuoCA._collect_inputs: " "'Collected_Values' missing in response from plugin: %s" % a_plugin.name) continue if not resp_values['collected_values']: nuoca_log(logging.DEBUG, "No time-series values were collected from plugin: %s" % a_plugin.name) continue if type(resp_values['collected_values']) is not list: nuoca_log(logging.ERROR, "NuoCA._collect_inputs: " "'Collected_Values' is not a list in " "response from plugin: %s" % a_plugin.name) continue list_count = len(resp_values['collected_values']) for list_index in range(list_count): new_values = {} key_prefix = a_plugin.name collected_dict = resp_values['collected_values'][list_index] if 'nuocaCollectionName' in collected_dict: key_prefix = collected_dict['nuocaCollectionName'] del collected_dict['nuocaCollectionName'] for collected_item in collected_dict: key_name = key_prefix + '.' + collected_item new_values[key_name] = collected_dict[collected_item] if collected_item == 'TimeStamp': new_values['timestamp'] = int(collected_dict[collected_item]) if self._output_values: new_values.update(self._output_values) rval.append(new_values) except Exception as e: nuoca_log(logging.ERROR, "NuoCA._collect_inputs: " "Error attempting to collect" " response from plugin: %s\n%s" % (a_plugin.name, str(e))) return rval def _store_outputs(self, collected_inputs): if not collected_inputs: return rval = {} plugin_msg = {'action': 'store', 'ts_values': collected_inputs} activated_plugins = self._get_activated_output_plugins() for a_plugin in activated_plugins: # noinspection PyBroadException try: a_plugin.plugin_object.child_pipe.send(plugin_msg) except Exception as e: nuoca_log(logging.ERROR, "Unable to send 'Store' message to plugin: %s\n%s" % (a_plugin.name, str(e))) for a_plugin in activated_plugins: resp_values = self._get_plugin_respose(a_plugin) if not resp_values: continue return rval def _create_plugin_manager(self): self.manager = MultiprocessPluginManager( directories_list=self._plugin_directories, plugin_info_ext="multiprocess-plugin") self.manager.setCategoriesFilter({ "Input": NuocaMPInputPlugin, "Output": NuocaMPOutputPlugin, "Transform": NuocaMPTransformPlugin }) # Activate plugins and call the plugin's startup() method. def _activate_and_startup_plugins(self): for input_plugin in self.config.INPUT_PLUGINS: input_plugin_name = input_plugin.keys()[0] if not self.manager.activatePluginByName(input_plugin_name, 'Input'): err_msg = "Cannot activate input plugin: '%s', Skipping." % \ input_plugin_name nuoca_log(logging.WARNING, err_msg) else: a_plugin = self.manager.getPluginByName(input_plugin_name, 'Input') if a_plugin: input_plugin_config = input_plugin.values()[0] if not input_plugin_config: input_plugin_config = {} input_plugin_config['nuoca_start_ts'] = self._starttime input_plugin_config['nuoca_collection_interval'] = \ self._collection_interval self._startup_plugin(a_plugin, input_plugin_config) self._input_plugins[input_plugin_name] = (a_plugin, input_plugin_config) for output_plugin in self.config.OUTPUT_PLUGINS: output_plugin_name = output_plugin.keys()[0] if not self.manager.activatePluginByName(output_plugin_name, 'Output'): err_msg = "Cannot activate output plugin: '%s', Skipping." % \ output_plugin_name nuoca_log(logging.WARNING, err_msg) else: a_plugin = self.manager.getPluginByName(output_plugin_name, 'Output') if a_plugin: output_plugin_config = output_plugin.values()[0] if not output_plugin_config: output_plugin_config = {} output_plugin_config['nuoca_start_ts'] = self._starttime output_plugin_config['nuoca_collection_interval'] = \ self._collection_interval self._startup_plugin(a_plugin, output_plugin_config) self._output_plugins[output_plugin_name] = (a_plugin, output_plugin_config) # TODO Transform Plugins # test if the plugin name is configured in NuoCA. def _is_plugin_name_configured(self, name): for configured_input in self.config.INPUT_PLUGINS: if name in configured_input: return True for configured_input in self.config.OUTPUT_PLUGINS: if name in configured_input: return True for configured_input in self.config.TRANSFORM_PLUGINS: if name in configured_input: return True return False # activate only the NuoCA configured plugins. def _activate_configured_plugins(self): self.manager.locatePlugins() # get a list of ALL plugin candidates plugin_candidates = self.manager.getPluginCandidates() for candidate in plugin_candidates: plugin_configured = self._is_plugin_name_configured(candidate[2].name) if not plugin_configured: # Remove this plugin candidate because it is no configued by NuoCA self.manager.removePluginCandidate(candidate) self.manager.loadPlugins() self._activate_and_startup_plugins() def _shutdown_all_plugins(self): for input_plugin in self._input_plugins: self.manager.deactivatePluginByName(input_plugin, 'Input') a_plugin = self.manager.getPluginByName(input_plugin, 'Input') self._shutdown_plugin(a_plugin) for output_plugin in self._output_plugins: self.manager.deactivatePluginByName(output_plugin, 'Output') a_plugin = self.manager.getPluginByName(output_plugin, 'Output') self._shutdown_plugin(a_plugin) # TODO Transform Plugins @staticmethod def kill_all_plugin_processes(manager, timeout=5): """ Kill any plugin processes that were left running after waiting up to the timeout value.. :param manager: MultiprocessPluginManager :type manager: MultiprocessPluginManager :param timeout: Maximum time to wait (in seconds) for the process to self exit before killing. :type timeout: ``int`` """ if not manager: return all_plugins = manager.getAllPlugins() wait_count = timeout for a_plugin in all_plugins: while a_plugin.plugin_object.proc.is_alive() and wait_count > 0: time.sleep(1) wait_count -= 1 if a_plugin.plugin_object.proc.is_alive(): nuoca_log(logging.INFO, "Killing plugin subprocess: %s" % a_plugin) a_plugin.plugin_object.proc.terminate() def _remove_all_plugins(self, timeout=5): """ Remove all plugins :param timeout: Maximum seconds to wait for subprocess to exit. :type timeout: ``int`` """ for input_plugin in self._input_plugins: a_plugin = self.manager.getPluginByName(input_plugin, 'Input') self._exit_plugin(a_plugin) for output_plugin in self._output_plugins: a_plugin = self.manager.getPluginByName(output_plugin, 'Output') self._exit_plugin(a_plugin) # TODO Transform Plugins # At this point all configured plugin subprocesses should be exiting # on their own. However, if there is any plugin subprocess that didn't # exit for any reason, we must terminate them so we don't hang the # NuoCA process at exit. NuoCA.kill_all_plugin_processes(self.manager, timeout) def start(self): """ Startup NuoCA """ self._create_plugin_manager() self._activate_configured_plugins() interval_sync = IntervalSync(interval=self._collection_interval, seed_ts=self._starttime) # Collection Interval Loop loop_count = 0 while self._enabled: loop_count += 1 collection_timestamp = interval_sync.wait_for_next_interval() self._collection_cycle(collection_timestamp * 1000) if self._self_test: if loop_count >= self._config.SELFTEST_LOOP_COUNT: self._enabled = False def shutdown(self, timeout=5): """ Shutdown NuoCA :param timeout: Maximum seconds to wait for subprocess to exit. :type timeout: ``int`` """ nuoca_log(logging.INFO, "nuoca server shutdown") self._shutdown_all_plugins() self._remove_all_plugins(timeout) nuoca_logging_shutdown()