class MonitorCB(ACS__POA.CBdouble): """ class that defines the callback for the acs components """ def __init__(self, key): self.log = HMILog(title=__name__) self.key = key self.r = redis.StrictRedis(host='localhost', port=6379, db=0) def done(self, completion, desc, a): """ this method is called when monitoring ends and it writes to redis """ # sets the values into redis; title 10 self.r.setex(self.key, 10, json.dumps({ 'status': 'done', 'value': completion })) self.log.info( [['g', (" - PropertyMonitorLocal.done.%s - ") % (self.key)], ['p', completion]]) def working(self, completion, desc, a): """ this method is called when monitoring and it writes to redis """ self.r.setex(self.key, 10, json.dumps({ 'status': 'working', 'value': completion })) self.log.info( [['g', (" - PropertyMonitorLocal.working.%s - ") % (self.key)], ['p', completion]])
class PropertyMonitorGlobal: def __init__(self, site_type): self.log = HMILog(title=__name__) self.log.info([['y', " - PropertyMonitor Global - "], ['g', site_type]]) self.site_type = site_type self.tel_ids = tel_ids[site_type] self.r = redis.StrictRedis(host='localhost', port=redis_port[site_type], db=0) t = threading.Thread(target=self.tel_global_loop) # set the thread to daemon for quicker termination t.daemon = True t.start() return def tel_global_loop(self): """ this method calls monitor_component_status every second """ while True: self.monitor_component_status() time.sleep(GLOBAL_FREQ / 10000000) def monitor_component_status(self): """ This method monitors the global properties of the components and # writes the values into redis. :return: """ # for each component in the dict for x in dict_of_components.keys(): # Build the component property dict comp_prop_dict = dict() # get the config for the global component glob = props.get(x) # for each property in the global component for xy in glob["props"]: # eval the pollin command and save to dict comp_prop_dict[xy[0]] = eval(glob["component_name"] + [xy[1]][0]) self.log.info([[ 'g', (" - PropertyMonitorGlobal.monitor_component_status.%s.%s - " ) % (glob["component_name"], xy[0]) ], ['p', eval(glob["component_name"] + [xy[1]][0])]]) # Create key for the component rkey = COMPONENT_PREFIX_GLOBAL + ':%s' % x # Save the dict into redis self.r.set(rkey, json.dumps(comp_prop_dict)) self.r.set(COMPONENT_PREFIX_GLOBAL, dict_of_components)
def __init__(self, site_type): self.log = HMILog(title=__name__) self.log.info([['y', " - PubsubTest - "], ['g', site_type]]) self.class_name = self.__class__.__name__ self.redis = RedisManager(name=self.class_name, port=utils.redis_port, log=self.log) gevent.spawn(self.loop) gevent.spawn(self.exe_func_loop) return
def __init__(self, site_type): self.log = HMILog(title=__name__) self.log.info([['y', " - PropertyMonitor Local - "], ['g', site_type]]) self.site_type = site_type self.tel_ids = tel_ids[site_type] self.r = redis.StrictRedis(host='localhost', port=redis_port[site_type], db=0) t = threading.Thread(target=self.tel_local_loop) # set the thread to daemon for quicker termination t.daemon = True t.start() return
def __init__(self, site_type): self.log = HMILog(title=__name__) self.log.info([['y', " - PropertyMonitor Queue - "], ['g', site_type]]) self.site_type = site_type self.tel_ids = tel_ids[site_type] self.r = redis.StrictRedis(host='localhost', port=redis_port[site_type], db=0) # to be on a safe side, clean the counter self.r.delete(DICT_SET_NAME) t = threading.Thread(target=self.queue_parser_loop) # set the thread to daemon for quicker termination t.daemon = True t.start() return
class PubsubTest(): def __init__(self, site_type): self.log = HMILog(title=__name__) self.log.info([['y', " - PubsubTest - "], ['g', site_type]]) self.class_name = self.__class__.__name__ self.redis = RedisManager(name=self.class_name, port=utils.redis_port, log=self.log) gevent.spawn(self.loop) gevent.spawn(self.exe_func_loop) return # ------------------------------------------------------------------ # # ------------------------------------------------------------------ def ask_data(self, rnd_seed=-1): self.log.info([['y', " --------------------------------------------"]]) self.log.info([['g', " - PubsubTest.ask_data - "], ['p', rnd_seed]]) args = {"func_name": "test_func", "arg0": rnd_seed} self.redis.publish(channel="exe_func_loop", message=packb(args)) return # ------------------------------------------------------------------ # # ------------------------------------------------------------------ def exe_func_loop(self): self.log.info([['g', " - starting PubsubTest.exe_func_loop ..."]]) redis_pubsub = None while True: while redis_pubsub is None: redis_pubsub = self.redis.set_pubsub("exe_func_loop") sleep(0.5) msg = self.redis.get_pubsub("exe_func_loop") if msg: args = unpackb(msg["data"]) # call as blocking getattr(self, args["func_name"])(args) # call as non-blocking def exe_async_func(args): getattr(self, args["func_name"])(args) gevent.spawn(exe_async_func, args) sleep(0.01) return # ------------------------------------------------------------------ # # ------------------------------------------------------------------ def test_func(self, args=None): self.log.info([['g', " - PubsubTest.test_func - "], ['p', args["arg0"]]]) return # ------------------------------------------------------------------ # # ------------------------------------------------------------------ def loop(self): self.log.info([['g', " - starting PubsubTest.loop ..."]]) sleep(2) rnd_seed = 9823987423 while True: self.ask_data(rnd_seed=rnd_seed) rnd_seed += 1 sleep(2) return
def __init__(self, key): self.log = HMILog(title=__name__) self.key = key self.r = redis.StrictRedis(host='localhost', port=6379, db=0)
class PropertyMonitorLocal: def __init__(self, site_type): self.log = HMILog(title=__name__) self.log.info([['y', " - PropertyMonitor Local - "], ['g', site_type]]) self.site_type = site_type self.tel_ids = tel_ids[site_type] self.r = redis.StrictRedis(host='localhost', port=redis_port[site_type], db=0) t = threading.Thread(target=self.tel_local_loop) # set the thread to daemon for quicker termination t.daemon = True t.start() return def get_redis(self): """ :return: the instance of the redis client """ return self.r def tel_local_loop(self): """ this method calls monitor_component_properties every second """ while True: self.monitor_component_properties() time.sleep(1) # Polling generator def sub_polling(self, component, params, key, command): """ this method returns the code for getting the value of a specific prop in polling mode :param component: the component that has the prop :param params: dict of additional parameters :param key: the key used to build the redis key :param command: command used for polling :return: code for getting a prop value in polling mode """ # create the string containing the code command_str = "%s%s" % (component, command) print "started polling " + key + " with frequency:" + \ str(params["polling_interval"]) # save the return value while local_queue.get("Polling:" + key) > 0: # eval the string and save the value value = eval(command_str) print key + ": " + str(value) # save the value into redis # Build local component key rkey_local = COMPONENT_PREFIX_LOCAL + ':Polling:%s' % key set_name = DICT_SET_NAME + ':%s' % key # check if the value in redis different if self.r.sadd(set_name, value): # recreate the set self.r.delete(set_name) self.r.sadd(set_name, value) # Push to local component key; TTL 10sec self.r.setex(rkey_local, 10, value) else: continue # sleep for x seconds where x is specified in the config time.sleep(int(params["polling_interval"] / 10000000)) # Monitor generator def sub_monitoring(self, component, params, command): """ this method creates a string that contains monitor creation code :param component: the name of the component we are monitoring on :param params: dict of params (default monitoring rate etc. :param command: the command to create the monitor :return: monitor creation string """ # creates monitor for the specified component and prop mon_create = "mon=%s%s.create_monitor(client.activateOffShoot(cb), desc)" % ( component, command) # set the monitoring interval mon_timer_trigger = "mon.set_timer_trigger(%d)" % int( params["timer_trigger_interval"]) # create the final string that will be exec'ed mon_setup = mon_create + "\n" + mon_timer_trigger + "\n" + \ "mon.set_value_trigger(%i, %s)" % ( params["value_trigger_delta"], params["is_monitor_value"]) return mon_setup def create_monitor(self, rkey_local, key): """ spawn a new monitor in a greenlet :param rkey_local: the key used for redis :param key: component name to get the properties from the config """ cb = MonitorCB(rkey_local) # creates the monitor from the generated string exec( self.sub_monitoring(props[key]['component_name'], props[key]["Monitor"]["additional_parameters"], props[key]["Monitor"]["monitoring_command"])) # adds the reference to the newly created monitor to monitors dict encoded_mon = jsonAcs.encode(mon) # add the newly created monitor reference to the hset in redis self.r.hset(MONITOR_DICT, key, encoded_mon) def monitor_component_properties(self): """ This method monitors the local properties of a component. # Monitoring occurs only for the components that has subs listening. Monitoring can be done on three different ways # (BACI, Polling or get history from MongoDB) """ for key in local_queue.keys(): # parse to get the property name without the prefix monitor_key = key.split(':', 1)[1] if local_queue[key] == 0 and monitor_key in polling_threads.keys(): # get the thread of the property t = polling_threads.pop(monitor_key, None) # stop the thread t.stop() print key + " thread removed." # check if the property has a monitor when the counter reaches 0 if local_queue[key] == 0 and self.r.hexists( MONITOR_DICT, monitor_key): # get the monitor from redis hset redis_monitor = self.r.hget(MONITOR_DICT, monitor_key) m = jsonAcs.decode(redis_monitor) # destroy the monitor m.destroy() print key + " monitor removed." # remove the monitor key from the hset in redis self.r.hdel(MONITOR_DICT, monitor_key) if local_queue[key] > 0: # split the key to check what kind of monitoring is needed key_prefix = key.split(':', 1)[0] key = key.split(':', 1)[1] # dict used for saving data to redis tel_prop_dict = dict() # when there are 0 subscribers to a key check if monitor exists if key_prefix == "Monitor": tel_prop_dict[key] = "" # Build local component key rkey_local = COMPONENT_PREFIX_LOCAL + ':Monitor:%s' % key set_name = DICT_SET_NAME + ':%s' % key # check the redis hset if the monitor exists if not self.r.hexists(MONITOR_DICT, monitor_key): self.create_monitor(rkey_local, key) print "Added monitor for property " + key + "." # check if the value in redis different if self.r.sadd(set_name, json.dumps(tel_prop_dict)): # recreate the set self.r.delete(set_name) self.r.sadd(set_name, json.dumps(tel_prop_dict)) # Push to local component key; TTL 10sec self.r.setex(rkey_local, 10, json.dumps(tel_prop_dict)) else: continue elif key_prefix == "Polling": # if a thread for the current property doesn't exist, create it if key not in polling_threads.keys(): # create a polling thread t = PollingThread( target=self.sub_polling, args=( props[key]["component_name"], props[key]["Polling"]["additional_parameters"], key, props[key]["Polling"]["polling_command"])) polling_threads[key] = t t.start() # todo: not implemented yet elif key_prefix == "Mongo": print "DB not supported yet" else: print "unsupported monitoring"
class PropertyMonitorQueue: def __init__(self, site_type): self.log = HMILog(title=__name__) self.log.info([['y', " - PropertyMonitor Queue - "], ['g', site_type]]) self.site_type = site_type self.tel_ids = tel_ids[site_type] self.r = redis.StrictRedis(host='localhost', port=redis_port[site_type], db=0) # to be on a safe side, clean the counter self.r.delete(DICT_SET_NAME) t = threading.Thread(target=self.queue_parser_loop) # set the thread to daemon for quicker termination t.daemon = True t.start() return def queue_parser_loop(self): """ this method calls the queue_parser method every second """ while True: self.queue_parser() time.sleep(QUEUE_FREQ / 10000000) def queue_parser(self): """ This method gets the list from the monitoring queue in redis. The values # get parsed and put into a counter dict. :return: a counter dict of components subscriptions """ ##### # Monitoring queue ##### for _ in self.r.lrange(QUEUE_NAME, 0, 1000): # Pop the element from queue and parse it pop = self.r.rpop(QUEUE_NAME) # split the pop'ed string pop_command = pop.split(':', 1)[0] pop_tel = pop.split(':', 1)[1] pop_tel = "Monitor:" + pop_tel # Depending on the prefix increment or decrement the counter if pop_command == 'UNSUB': self.log.info( [['g', " - PropertyMonitorQueue.queue_parser.UNSUB - "], ['p', pop_tel]]) # Check if key value is lower than 0 if local_queue[pop_tel] <= 0: local_queue.pop(pop_tel, None) else: local_queue[pop_tel] -= 1 else: self.log.info( [['g', " - PropertyMonitorQueue.queue_parser.SUB - "], ['p', pop_tel]]) local_queue[pop_tel] += 1 print local_queue ##### # Polling queue ##### for _ in self.r.lrange(POLLING_QUEUE, 0, 1000): # Pop the element from queue and parse it pop = self.r.rpop(POLLING_QUEUE) pop_command = pop.split(':', 1)[0] pop_tel = pop.split(':', 1)[1] pop_tel = "Polling:" + pop_tel # Depending on the prefix increment or decrement the counter if pop_command == 'UNSUB': # Check if key value is lower than 0 if local_queue[pop_tel] <= 0: local_queue.pop(pop_tel, None) else: local_queue[pop_tel] -= 1 else: local_queue[pop_tel] += 1 print local_queue ##### # Database queue ##### for _ in self.r.lrange(MONGO_QUEUE, 0, 1000): # Pop the element from queue and parse it pop = self.r.rpop(MONGO_QUEUE) pop_command = pop.split(':', 1)[0] pop_tel = pop.split(':', 1)[1] pop_tel = "Mongo:" + pop_tel # Depending on the prefix increment or decrement the counter if pop_command == 'UNSUB': # Check if key value is lower than 0 if local_queue[pop_tel] <= 0: local_queue.pop(pop_tel, None) else: local_queue[pop_tel] -= 1 else: local_queue[pop_tel] += 1 print local_queue