class GeventedHTTPTransport(HTTPTransport): scheme = ['gevent+http', 'gevent+https'] def __init__(self, parsed_url, maximum_outstanding_requests=100): if not gevented: raise ImportError('GeventedHTTPTransport requires gevent.') self._lock = Semaphore(maximum_outstanding_requests) super(GeventedHTTPTransport, self).__init__(parsed_url) # remove the gevent+ from the protocol, as it is not a real protocol self._url = self._url.split('+', 1)[-1] def send(self, data, headers): """ Spawn an async request to a remote webserver. """ # this can be optimized by making a custom self.send that does not # read the response since we don't use it. self._lock.acquire() return spawn(super(GeventedHTTPTransport, self).send, data, headers).link(self._done, self) def _done(self, *args): self._lock.release()
class Pool(Group): def __init__(self, size=None, greenlet_class=None): if size is not None and size < 0: raise ValueError('size must not be negative: %r' % (size, )) Group.__init__(self) self.size = size if greenlet_class is not None: self.greenlet_class = greenlet_class if size is None: self._semaphore = DummySemaphore() else: self._semaphore = Semaphore(size) def wait_available(self): self._semaphore.wait() def full(self): return self.free_count() <= 0 def free_count(self): if self.size is None: return 1 return max(0, self.size - len(self)) def add(self, greenlet): self._semaphore.acquire() try: Group.add(self, greenlet) except: self._semaphore.release() raise def _discard(self, greenlet): Group._discard(self, greenlet) self._semaphore.release()
class imap_manager(object): def __init__(self): from gevent.coros import Semaphore self.client = None self.sem = Semaphore(1) self.count = 0 def close(self, current): import gevent gevent.sleep(360) self.sem.acquire() if self.client is not None and self.count == current: self.client.close() self.client = None self.sem.release() @contextmanager def get(self): import gevent self.count += 1 self.sem.acquire() self.count += 1 if self.client is None: from rbit import config from rbit import backend from rbit import imap cfg = config.Config('config', backend.create_session) self.client = imap.IMAPClient.from_config(cfg) yield self.client self.sem.release() gevent.spawn(lambda : self.close(self.count))
class SessionPool(object): """ This class is intended to allow for the model to be used in a threaded environment. """ def __init__(self, engine, min_sessions=10, max_sessions=25): self.engine = engine sm = sessionmaker(autoflush=False, autocommit=False, bind=engine) self.session = scoped_session(sm) self.min_sessions = min_sessions self.max_sessions = max_sessions self.session_pool = [] self.available = [] self.checkouts = {} self.sessions = local() self.lock = Semaphore() def checkin(self): self.lock.acquire() try: session = self.sessions.session if session: session.close() finally: self.lock.release() def checkout(self): self.lock.acquire() try: session = self.sessions.session = self.session() return session finally: self.lock.release()
class MediaCodec_Base: def __init__(self,svc): #maxsize - 最大缓存,超过限制将视为无效解码 self.buf ='' self.conn = None self.svc = svc self.mtx = Semaphore() self.msgcnt=0 #接收消息计数 def parseMessage(self,s): pass #数据置入队列,应检查队列长度和数据合法性 def queueIn(self,s,conn): self.mtx.acquire() # self.buflist.append(s) self.buf+=s self.mtx.release() return True def decode(self): #@return (packets,retry) retry表示解码遇到错误或者数据非法 return (),False #根据传入的消息分拣出gps,alarm消息 MsgAoModule_Alarm(), MsgAoModule_GpsData() def filter_msg(self,m,aom): # GSP,ALARM 消息是关心的消息,需要反馈到客户端 return (m,) def command(self,aom,m): pass def save(self,aom,m): ''' 保存设备所有信息,不区分是何类型 ''' try: cm = aom.ao.cm params= m.params if isinstance(m.params,dict): params = json.dumps(m.params) log = cm.AO_ModuleLog() log.ao = aom.ao.r log.module = aom.r log.type = ModuleMsgType.DEV2SYS log.time = datetime.datetime.now() log.msgtype = 'N/A' log.params = params log.rawmsg = m.rawmsg log.seq = 0 log.save() return True except: traceback.print_exc() return False
class Client(object): px_per_tick = 100 def __init__(self, canvas): self.canvas = canvas self.socket = None self.connect_ts = time.time() # This buffer discards all but the newest 1024 messages self.sendbuffer = deque([], 1024) # And this is used to limit clients to X messages per tick # We start at 0 (instead of x) to add a reconnect-penalty. self.limit = Semaphore(0) self.lock = RLock() def send(self, line): self.sendbuffer.append(line.strip() + '\n') def nospam(self, line): if not self.sendbuffer: self.sendbuffer.append(line.strip() + '\n') def disconnect(self): with self.lock: if self.socket: socket = self.socket self.socket = None socket.close() log.info('Disconnect') def serve(self, socket): with self.lock: self.socket = socket sendall = self.socket.sendall readline = self.socket.makefile().readline try: while self.socket: self.limit.acquire() # Idea: Send first, receive later. If the client is to # slow to get the send-buffer empty, he cannot send. while self.sendbuffer: sendall(self.sendbuffer.popleft()) line = readline().strip() if not line: break arguments = line.split() command = arguments.pop(0) try: self.canvas.fire('COMMAND-%s' % command.upper(), self, *arguments) except Exception, e: socket.send('ERROR %r :(' % e) break finally: self.disconnect() def tick(self): while self.limit.counter <= self.px_per_tick: self.limit.release()
def test_release_twice(self): s = Semaphore() result = [] s.rawlink(lambda s: result.append('a')) s.release() s.rawlink(lambda s: result.append('b')) s.release() gevent.sleep(0.001) self.assertEqual(result, ['a', 'b'])
class GeventSemaphore(BaseSemaphore): def __init__(self): self.__semaphore = Semaphore() def acquire(self): self.__semaphore.acquire() def release(self): self.__semaphore.release()
def test(): s = Semaphore(0) future1 = yield batchy_gevent.greenlet_future(gevent.spawn(acq, s)) future2 = yield batchy_gevent.greenlet_future(gevent.spawn(acq, s)) s.release() yield future1 s.release() yield future2
def handle_socket(sock, address): semaphore = Semaphore() while 1: sock.setblocking(0) semaphore.acquire() sockfd = sock.makefile() chunk_size = 1024 head_packet_format = "!LL128s128sL" head_packet_size = struct.calcsize(head_packet_format) data = sockfd.read(head_packet_size) if not data or len(data) != head_packet_size: return filepath_len, filename_len, filepath,filename, filesize = struct.unpack(head_packet_format,data) filepath = filepath[:filepath_len] filename = filename[:filename_len] #logger.debug("update file: %s" % filepath + '/' + filename) fd = open(filename,'wb') fcntl.flock(fd,fcntl.LOCK_EX) print "File %s size: %s" % (filename, filesize) print 111111111111111 writen_size = 0 if filesize > chunk_size: times = filesize / chunk_size first_part_size = times * chunk_size second_part_size = filesize % chunk_size print "times: %s first_part_size:%s second_part_size:%s" % (times,first_part_size,second_part_size) print 22222222222222222222 #receive first part packets while 1: data = sockfd.read(chunk_size) fd.write(data) fd.flush() writen_size += len(data) if writen_size == first_part_size: break print "writen_size in first_par: %s" % writen_size print 333333333333333333333 if second_part_size: #receive the packet at last data = sockfd.read(second_part_size) fd.write(data) fd.flush() writen_size += len(data) print 4444444444444444444444 else: data = sockfd.read(filesize) fd.write(data) fd.flush() writen_size += len(data) fcntl.flock(fd,fcntl.LOCK_UN) fd.close() print '555555555555555555555' print "File %s size: %s\n" % (filename, writen_size) semaphore.release()
class MediaCodec_Base: def __init__(self, svc): #maxsize - 最大缓存,超过限制将视为无效解码 self.buf = '' self.conn = None self.svc = svc self.mtx = Semaphore() self.msgcnt = 0 #接收消息计数 def parseMessage(self, s): pass #数据置入队列,应检查队列长度和数据合法性 def queueIn(self, s, conn): self.mtx.acquire() # self.buflist.append(s) self.buf += s self.mtx.release() return True def decode(self): #@return (packets,retry) retry表示解码遇到错误或者数据非法 return (), False #根据传入的消息分拣出gps,alarm消息 MsgAoModule_Alarm(), MsgAoModule_GpsData() def filter_msg(self, m, aom): # GSP,ALARM 消息是关心的消息,需要反馈到客户端 return (m, ) def command(self, aom, m): pass def save(self, aom, m): ''' 保存设备所有信息,不区分是何类型 ''' try: cm = aom.ao.cm params = m.params if isinstance(m.params, dict): params = json.dumps(m.params) log = cm.AO_ModuleLog() log.ao = aom.ao.r log.module = aom.r log.type = ModuleMsgType.DEV2SYS log.time = datetime.datetime.now() log.msgtype = 'N/A' log.params = params log.rawmsg = m.rawmsg log.seq = 0 log.save() return True except: traceback.print_exc() return False
class BaseSensor(Greenlet, Model): """ Base class for sensors. Each sensor is associated with a Docker container by its cid (or name, if you prefer). A sensor has to be started with its `start` method, eventually, stopped with its `kill` method. Every sensor provides collected data by means of `get_data` method which returns a generator. Each call to `next` value is blocking for a time declared on `__init__` of the sensor through `spacing` parameter. """ cid = StringType(required=True) spacing = FloatType(default=0.1) @serializable def uid(self): return get_uid(self.__class__, self.cid) def __init__(self, *args, **kwargs): Greenlet.__init__(self) Model.__init__(self, *args, **kwargs) docker_url = config.get('docker_url') self.container = api.DockerAPI(self.cid, docker_url) self._lock = Semaphore() self._lock.acquire() # locking semaphore self._new_data = None @raise_connection_error def _run(self): while True: self._new_data = self._get() LOG.debug("{} got {}".format(self.uid, self._new_data)) self._store(self._new_data) self._lock.release() sleep(self.spacing) def _get(self): """Override""" return None def _store(self, data): """Override""" pass def get_data(self): while True: if not self.started or self.dead: raise SensorError("Start the sensor before getting data.") self._lock.acquire() yield self._new_data
class Sema: def __init__(self): self.sem = Semaphore() self.timestamp = datetime.datetime.now() self.count = 1 self.sem.acquire(blocking=True) def join(self): self.count += 1 self.sem.acquire(blocking=True) def release(self): for ii in xrange(self.count): self.sem.release()
class ConnectionPool(object): def __init__(self, dsn, max_con=10, max_idle=3, connection_factory=RealDictConnection): self.dsn = dsn self.max_con = max_con self.max_idle = max_idle self.connection_factory = connection_factory self._sem = Semaphore(max_con) self._free = [] self._local = gevent_local() def __enter__(self): self._sem.acquire() try: if getattr(self._local, 'con', None) is not None: raise RuntimeError("Attempting to re-enter connection pool?") if self._free: con = self._free.pop() else: con = psycopg2.connect( dsn=self.dsn, connection_factory=self.connection_factory) self._local.con = con return con except StandardError: self._sem.release() raise def __exit__(self, exc_type, exc_value, traceback): try: if self._local.con is None: raise RuntimeError("Exit connection pool with no connection?") if exc_type is not None: self.rollback() else: self.commit() if len(self._free) < self.max_idle: self._free.append(self._local.con) self._local.con = None finally: self._sem.release() def commit(self): self._local.con.commit() def rollback(self): self._local.con.rollback()
class HitCounter(object): def __init__(self): self.value = 0 self.lock = Semaphore(1) def __str__(self): return str(self.value) def increment(self): try: self.lock.acquire() self.value += 1 finally: self.lock.release()
class BlockingDeque(deque): def __init__(self, *args, **kwargs): super(BlockingDeque, self).__init__(*args, **kwargs) self.sema = Semaphore(len(self)) def append(self, *args, **kwargs): ret = super(BlockingDeque, self).append(*args, **kwargs) self.sema.release() return ret def appendleft(self, *args, **kwargs): ret = super(BlockingDeque, self).appendleft(*args, **kwargs) self.sema.release() return ret def clear(self, *args, **kwargs): ret = super(BlockingDeque, self).clear(*args, **kwargs) while not self.sema.locked(): self.sema.acquire(blocking=False) return ret def extend(self, *args, **kwargs): pre_n = len(self) ret = super(BlockingDeque, self).extend(*args, **kwargs) post_n = len(self) for i in xrange(pre_n, post_n): self.sema.release() return ret def extendleft(self, *args, **kwargs): pre_n = len(self) ret = super(BlockingDeque, self).extendleft(*args, **kwargs) post_n = len(self) for i in xrange(pre_n, post_n): self.sema.release() return ret def pop(self, *args, **kwargs): self.sema.acquire() return super(BlockingDeque, self).pop(*args, **kwargs) def popleft(self, *args, **kwargs): self.sema.acquire() return super(BlockingDeque, self).popleft(*args, **kwargs) def remove(self, *args, **kwargs): ret = super(BlockingDeque, self).remove(*args, **kwargs) self.sema.acquire() return ret
def test_transactional(): """Test that a transaction is atomic.""" # Start a transaction in another thread that will block until we signal it sem = Semaphore(0) greenlet = gevent.spawn(slow_insert, sem) # Add something that will cause the transaction to fail with db.get_pool().connection() as conn: c = conn.cursor() c.execute("INSERT INTO test(id, name) VALUES(7, 'zonk')") conn.commit() # Let the transaction continue sem.release() # Wait for it to finish greenlet.join() # Check that the database does not contain the values from the transaction names = select_names().flat assert 'zonk' in names assert 'five' not in names assert 'seven' not in names
class RateLimiter(object): def __init__(self, cps=None, cpm=None, cph=None, cpd=None, max_wait=None): self.__max_wait = max_wait self.__limits = {} if cps is not None and cps != 0: self.__limits['calls per second'] = RateLimit(cps, 1) if cpm is not None and cpm != 0: self.__limits['calls per minute'] = RateLimit(cpm, 60) if cph is not None and cph != 0: self.__limits['calls per hour'] = RateLimit(cph, 60*60) if cpd is not None and cpd != 0: self.__limits['calls per day'] = RateLimit(cpd, 60*60*24) self.__semaphore = Semaphore() def __enter__(self): begin = time.time() locked = self.__semaphore.acquire(timeout=self.__max_wait) if locked: try: max_wait = 0 max_name = None now = time.time() for name, limit in self.__limits.items(): wait = limit.wait(now) if wait > max_wait: max_name = name max_wait = wait if self.__max_wait is not None and max_wait > self.__max_wait: raise RateException('Rate limit exceeded for %s' % max_name) if max_wait > 0: end = now + max_wait while True: cur = time.time() if cur >= end: break else: sleep( end - cur) for name, limit in self.__limits.items(): wait = limit.wait() if wait > 0: logs.warning('%s wait should be 0 is %s' % (name, wait)) limit.call() finally: self.__semaphore.release() else: raise RateException('Max wait (%s) exceeded trying to get slot' % self.__max_wait) def __exit__(self, exctype, value, trace): return False
class GeventTransport(SocketTransport): ''' Transport using gevent backend. It relies on gevent's implementation of sendall to send whole frames at a time. On the input side, it uses a gevent semaphore to ensure exclusive access to the socket and input buffer. NOTE: This is new to haigha and there may be integration issues with some versions of RabbitMQ. In particular, what may happen when a blocking call to send a frame allows another thread to queue another frame, but the frames can't be interlaced due to how the protocol is defined or implemented. A lot of the 'synchronous' calls have very specific expectations. If this becomes a problem then the quickest way to solve it would be to switch to non-blocking so that the same thread of execution is allowed to send all of its frames in sequence. In that case, the gevent implementation may be pushed into the EventSocket as another supported concurrency lib. Note also that the blocking nature of the sockets means that the any threads running IO should actively yield with a sleep(0) to ensure other threads are serviced. In a saturated environment, failure to do so will lead to significant lags in signal handling or other IO. In practice, a typical client is attaching to other data stores so those could be enough to yield. ''' def __init__(self, *args, **kwargs): super(GeventTransport,self).__init__(*args) self._synchronous = kwargs.get('synchronous',False) self._read_lock = Semaphore() self._write_lock = Semaphore() ### ### Transport API ### # nothing to overload with connect() def read(self, timeout=None): ''' Read from the transport. If no data is available, should return None. If timeout>0, will only block for `timeout` seconds. ''' self._read_lock.acquire() try: return super(GeventTransport,self).read(timeout=timeout) finally: self._read_lock.release() def buffer(self, data): ''' Buffer unused bytes from the input stream. ''' self._read_lock.acquire() try: return super(GeventTransport,self).buffer(data) finally: self._read_lock.release() def write(self, data): ''' Write some bytes to the transport. ''' # MUST use a lock here else gevent could raise an exception if 2 greenlets # try to write at the same time. I was hoping that sendall() would do that # blocking for me, but I guess not. May require an eventsocket-like buffer # to speed up under high load. self._write_lock.acquire() try: return super(GeventTransport,self).write(data) finally: self._write_lock.release()
class SchedValve(SchedCommon): """Mirrors (and monitors) a valve.""" locked = False # external command, don't change sched = None sched_ts = None sched_job = None sched_lock = None on = False on_ts = None flow = 0 _flow_check = None def __new__(cls,v): if v.id in valves: return valves[v.id] self = object.__new__(cls) valves[v.id] = self self.v = v self.site = SchedSite(self.v.controller.site) self.env = EnvGroup(self.v.envgroup) self.controller = SchedController(self.v.controller) self.sched_lock = Semaphore() if self.site.ci: try: self.site.send_command("set","output","off",*(self.v.var.split())) except NotConnected: pass return self def __init__(self,v): pass def _on(self,sched=None,duration=None): print >>sys.stderr,"Open",self.v.var self.site.delay_on() if self.controller.has_max_on(): print >>sys.stderr,"… but too many:", " ".join(str(v) for v in self.controller.c.valves.all() if SchedValve(v).on) if sched: sched.update(seen = False) self.on = False self.log("NOT running for %s: too many"%(duration,)) raise TooManyOn(self) if duration is None and sched is not None: duration = sched.duration if duration is None: self.log("Run (indefinitely)") self.site.send_command("set","output","on",*(self.v.var.split())) else: self.log("Run for %s"%(duration,)) if not isinstance(duration,(int,long)): duration = duration.total_seconds() self.site.send_command("set","output","on",*(self.v.var.split()), sub=(("for",duration),("async",))) if sched is not None: if self.v.verbose: self.log("Opened for %s"%(sched,)) self.sched = sched if not sched.seen: sched.update(start=now(), seen = True) sched.refresh() #Save(sched) else: if self.v.verbose: self.log("Opened for %s"%(duration,)) def _off(self): if self.on: if self.v.verbose: self.log("Closing") print >>sys.stderr,"Close",self.v.var try: self.site.send_command("set","output","off",*(self.v.var.split())) except NotConnected: pass def shutdown(self): if self._flow_check is not None: self._flow_check.dead() def run_schedule(self): if not self.sched_lock.acquire(blocking=False): if self.v.verbose: print >>sys.stderr,"SCHED LOCKED1 %s" % (self.v.name,) return try: self._run_schedule() except Exception: self.log(format_exc()) finally: self.sched_lock.release() def _run_schedule(self): if self.sched_job is not None: self.sched_job.kill() self.sched_job = None if self.locked: if self.v.verbose: print >>sys.stderr,"SCHED LOCKED2 %s" % (self.v.name,) return n = now() try: if self.sched is not None: self.sched.refresh() if self.sched.start+self.sched.duration <= n: self._off() self.sched = None else: self.sched_job = gevent.spawn_later((self.sched.start+self.sched.duration-n).total_seconds(),self.run_sched_task,reason="_run_schedule 1") if self.v.verbose: print >>sys.stderr,"SCHED LATER %s: %s" % (self.v.name,humandelta(self.sched.start+self.sched.duration-n)) return except ObjectDoesNotExist: pass # somebody deleted it *shrug* sched = None if self.sched_ts is None: try: sched = self.v.schedules.filter(start__lt=n).order_by("-start")[0] except IndexError: self.sched_ts = n-timedelta(1,0) else: self.sched_ts = sched.start+sched.duration if sched.start+sched.duration > n: # still running if self.v.verbose: print >>sys.stderr,"SCHED RUNNING %s: %s" % (self.v.name,humandelta(sched.start+sched.duration-n)) try: self._on(sched, sched.start+sched.duration-n) except TooManyOn: self.log("Could not schedule: too many open valves") except NotConnected: self.log("Could not schedule: connection to HomEvenT failed") return try: sched = self.v.schedules.filter(start__gte=self.sched_ts).order_by("start")[0] except IndexError: if self.v.verbose: print >>sys.stderr,"SCHED EMPTY %s: %s" % (self.v.name,str_tz(self.sched_ts)) self._off() return if sched.start > n: if self.v.verbose: print >>sys.stderr,"SCHED %s: sched %d in %s" % (self.v.name,sched.id,humandelta(sched.start-n)) self._off() self.sched_job = gevent.spawn_later((sched.start-n).total_seconds(),self.run_sched_task,reason="_run_schedule 2") return try: self._on(sched) except TooManyOn: self.log("Could not schedule: too many open valves") except NotConnected: self.log("Could not schedule: connection to HomEvenT failed") def run_sched_task(self,reason="valve"): self.sched_job = None self.site.run_sched_task(reason=reason) def add_flow(self, val): if self._flow_check is not None: if self._flow_check.add_flow(val): return self.flow += val def check_flow(self,**k): cf = None try: cf = FlowCheck(self) cf.run() except Exception as ex: log_error(self.v) if cf is not None: cf._unlock() def refresh(self): self.v.refresh() # if self.sched is not None: # self.sched.refresh() def connect_monitors(self): if self.site.ci is None: return self.mon = self.site.ci.root.monitor(self.watch_state,"output","set","*","*",*(self.v.var.split())) self.ckf = self.site.ci.root.monitor(self.check_flow,"check","flow",*self.v.var.split()) def watch_state(self,event=None,**k): """output set OLD NEW NAME""" on = (str(event[3]).lower() in ("1","true","on")) if self._flow_check is not None: # TODO self.on = on self._flow_check.state(on) return if self.locked: self.on = on return try: if on != self.on: print >>sys.stderr,"Report %s" % ("ON" if on else "OFF"),self.v.var n=now() if self.sched is not None and self.sched.start+self.sched.duration <= n: self.sched.update(db_duration=(n-self.sched.start).total_seconds()) self.sched.refresh() self.sched_ts = self.sched.start+self.sched.duration self.sched = None flow,self.flow = self.flow,0 # If nothing happened, calculate. if not on: duration = n-self.on_ts maxflow = self.v.flow * duration.total_seconds() if (not flow and not self.v.feed.var) or flow > maxflow: flow = maxflow self.new_level_entry(flow) if not on: if self.v.level > self.v.stop_level + (self.v.start_level-self.v.stop_level)/5: self.v.update(priority=True) self.log("Done for %s, level is now %s"%(duration,self.v.level)) self.on = on self.on_ts = n except Exception: print_exc() def sync(self): flow,self.flow = self.flow,0 self.new_level_entry(flow) def sync_history(self): n=now() try: lv = self.v.levels.order_by("-time")[0] except IndexError: pass else: if self.v.time > lv.time: self.log("Timestamp downdate: %s %s" % (self.v.time,lv.time)) self.v.update(time = lv.time) self.v.refresh() #Save(self.v) if (n-self.v.time).total_seconds() > 3500: self.new_level_entry() def new_level_entry(self,flow=0): self.site.current_history_entry() n=now() self.v.refresh() hts = None try: lv = self.v.levels.order_by("-time")[0] except IndexError: ts = n-timedelta(1,0) else: ts = lv.time sum_f = 0 sum_r = 0 for h in self.site.s.history.filter(time__gt=ts).order_by("time"): if self.v.verbose>2: self.log("Env factor for %s: T=%s W=%s S=%s"%(h,h.temp,h.wind,h.sun)) f = self.env.env_factor(h, logger=self.log if self.v.verbose>2 else None)*self.v.adj if self.v.verbose>1: self.log("Env factor for %s is %s"%(h,f)) sum_f += self.site.s.db_rate * self.v.do_shade(self.env.eg.factor*f) * (h.time-ts).total_seconds() sum_r += self.v.runoff*h.rain ts=h.time if self.v.verbose: self.log("Apply env %f, rain %r"%(sum_f,sum_r)) if self.v.time == ts: return if self.v.level < 0: level = 0 else: level = F('level') level += sum_f if flow > 0 and self.v.level > self.v.max_level: level = self.v.max_level level -= flow/self.v.area+sum_r #if level < 0: # self.log("Level %s ?!?"%(self.v.level,)) self.v.update(time=ts, level=level) self.v.refresh() lv = Level(valve=self.v,time=ts,level=self.v.level,flow=flow) lv.save() def log(self,txt): log(self.v,txt)
class Queue(Greenlet): """Manages the queue of |Envelope| objects waiting for delivery. This is not a standard FIFO queue, a message's place in the queue depends entirely on the timestamp of its next delivery attempt. :param store: Object implementing :class:`QueueStorage`. :param relay: |Relay| object used to attempt message deliveries. :param backoff: Function that, given an |Envelope| and number of delivery attempts, will return the number of seconds before the next attempt. If it returns ``None``, the message will be permanently failed. The default backoff function simply returns ``None`` and messages are never retried. :param bounce_factory: Function that produces a |Bounce| object given the same parameters as the |Bounce| constructor. If the function returns ``None``, no bounce is delivered. By default, a new |Bounce| is created in every case. :param store_pool: Number of simultaneous operations performable against the ``store`` object. Default is unlimited. :param relay_pool: Number of simultaneous operations performable against the ``relay`` object. Default is unlimited. """ def __init__(self, store, relay, backoff=None, bounce_factory=None, store_pool=None, relay_pool=None): super(Queue, self).__init__() self.store = store self.relay = relay self.backoff = backoff or self._default_backoff self.bounce_factory = bounce_factory or Bounce self.wake = Event() self.queued = [] self.queued_lock = Semaphore(1) self.queue_policies = [] self._use_pool('store_pool', store_pool) self._use_pool('relay_pool', relay_pool) def add_policy(self, policy): """Adds a |QueuePolicy| to be executed before messages are persisted to storage. :param policy: |QueuePolicy| object to execute. """ if isinstance(policy, QueuePolicy): self.queue_policies.append(policy) else: raise TypeError('Argument not a QueuePolicy.') @staticmethod def _default_backoff(envelope, attempts): pass def _run_policies(self, envelope): results = [envelope] def recurse(current, i): try: policy = self.queue_policies[i] except IndexError: return ret = policy.apply(current) if ret: results.remove(current) results.extend(ret) for env in ret: recurse(env, i + 1) else: recurse(current, i + 1) recurse(envelope, 0) return results def _use_pool(self, attr, pool): if pool is None: pass elif isinstance(pool, Pool): setattr(self, attr, pool) else: setattr(self, attr, Pool(pool)) def _pool_run(self, which, func, *args, **kwargs): pool = getattr(self, which + '_pool', None) if pool: ret = pool.spawn(func, *args, **kwargs) return ret.get() else: return func(*args, **kwargs) def _pool_imap(self, which, func, *iterables): pool = getattr(self, which + '_pool', gevent) threads = imap(pool.spawn, repeat(func), *iterables) ret = [] for thread in threads: thread.join() ret.append(thread.exception or thread.value) return ret def _pool_spawn(self, which, func, *args, **kwargs): pool = getattr(self, which + '_pool', gevent) return pool.spawn(func, *args, **kwargs) def _add_queued(self, entry): for i, info in enumerate(self.queued): if info[0] > entry[0]: # [0] is the timestamp. self.queued.insert(i, entry) break else: self.queued.append(entry) self.wake.set() def enqueue(self, envelope): """Drops a new message in the queue for delivery. The first delivery attempt is made immediately (depending on relay pool availability). This method is not typically called directly, |Edge| objects use it when they receive new messages. :param envelope: |Envelope| object to enqueue. :returns: Zipped list of envelopes and their respective queue IDs (or thrown :exc:`QueueError` objects). """ now = time.time() envelopes = self._run_policies(envelope) ids = self._pool_imap('store', self.store.write, envelopes, repeat(now)) results = zip(envelopes, ids) for env, id in results: if not isinstance(id, BaseException): self._pool_spawn('relay', self._attempt, id, env, 0) elif not isinstance(id, QueueError): raise id # Re-raise exceptions that are not QueueError. return results def _load_all(self): for entry in self.store.load(): self._add_queued(entry) def _bounce(self, envelope, reply): bounce = self.bounce_factory(envelope, reply) if bounce: return self.enqueue(bounce) def _perm_fail(self, id, envelope, reply): self._pool_spawn('store', self.store.remove, id) if envelope.sender: # Can't bounce to null-sender. self._pool_spawn('bounce', self._bounce, envelope, reply) def _retry_later(self, id, envelope, reply): attempts = self.store.increment_attempts(id) wait = self.backoff(envelope, attempts) if wait is None: reply.message += ' (Too many retries)' self._perm_fail(id, envelope, reply) else: when = time.time() + wait self.store.set_timestamp(id, when) self._add_queued((when, id)) def _attempt(self, id, envelope, attempts): try: self.relay._attempt(envelope, attempts) except TransientRelayError as e: self._pool_spawn('store', self._retry_later, id, envelope, e.reply) except PermanentRelayError as e: self._perm_fail(id, envelope, e.reply) except Exception as e: log_exception(__name__) reply = Reply('450', '4.0.0 Unhandled delivery error: ' + str(e)) self._pool_spawn('store', self._retry_later, id, envelope, reply) raise else: self._pool_spawn('store', self.store.remove, id) def _dequeue(self, id): envelope, attempts = self.store.get(id) self._pool_spawn('relay', self._attempt, id, envelope, attempts) def _check_ready(self, now): last_i = 0 for i, entry in enumerate(self.queued): timestamp, id = entry if now >= timestamp: self._pool_spawn('store', self._dequeue, id) last_i = i + 1 else: break if last_i > 0: self.queued = self.queued[last_i:] def _wait_ready(self, now): try: first = self.queued[0] except IndexError: self.wake.wait() self.wake.clear() return first_timestamp = first[0] if first_timestamp > now: self.wake.wait(first_timestamp - now) self.wake.clear() def flush(self): """Attempts to immediately flush all messages waiting in the queue, regardless of their retry timers. ***Note:*** This can be a very expensive operation, use with care. """ self.wake.set() self.wake.clear() self.queued_lock.acquire() try: for entry in self.queued: self._pool_spawn('store', self._dequeue, entry[1]) self.queued = [] finally: self.queued_lock.release() def _run(self): self._pool_spawn('store', self._load_all) while True: self.queued_lock.acquire() try: now = time.time() self._check_ready(now) self._wait_ready(now) finally: self.queued_lock.release()
class DataFrameIO(io.RawIOBase): """Raw I/O implementation for stream sockets. This class supports the makefile() method on sockets. It provides the raw I/O interface on top of a socket object. """ def __init__(self): io.RawIOBase.__init__(self) self.buf = bytearray() self.target_size = 0 self.semaphore = Semaphore() self._closed = False @property def closed(self): return self._closed def readinto(self, b): """Read up to len(b) bytes into the writable buffer *b* and return the number of bytes read. If the socket is non-blocking and no bytes are available, None is returned. If *b* is non-empty, a 0 return value indicates that the connection was shutdown at the other end. """ self.target_size = len(b) if self.target_size > len(self.buf): self.semaphore.acquire() b[0 : self.target_size] = self.buf[0 : self.target_size] del self.buf[0 : self.target_size] return self.target_size def write(self, b): """Write the given bytes or bytearray object *b* to the socket and return the number of bytes written. This can be less than len(b) if not all data could be written. If the socket is non-blocking and no bytes could be written None is returned. """ self.buf += b if self.target_size <= len(self.buf): self.semaphore.release() return len(b) def readable(self): return True def writable(self): return True def seekable(self): return False def fileno(self): return None def close(self): io.RawIOBase.close(self) self._closed = True self.buf = bytearray()
class RateLimiterState(object): def __init__(self, fail_limit, fail_period, blackout_wait): self.__local_rlservice = None self.__request_fails = 0 self.__fails = deque() self.__fail_limit = fail_limit self.__fail_period = fail_period self.__blackout_start = None self.__blackout_wait = blackout_wait self.__is_ec2 = utils.is_ec2() self.__service_init_semaphore = Semaphore() stack_info = libs.ec2_utils.get_stack() self.__stack_name = 'localhost' self.__node_name = 'localhost' if stack_info is not None: self.__stack_name = stack_info.instance.stack self.__node_name = stack_info.instance.name self.__last_email_time = 0 self.__emails = deque() # determine the private ip address of the ratelimiter instance for this stack self._getHost() print('### host: %s' % self.__host) class FailLog(object): def __init__(self, exception): self.timestamp = time.time() self.exception = exception @property def _local_rlservice(self): if self.__local_rlservice is None: # use a semaphore here because if two requests come in immediately, we might instantiate two services self.__service_init_semaphore.acquire() if self.__local_rlservice is None: if self.__is_ec2: self.__local_rlservice = StampedRateLimiterService(throttle=True) else: self.__local_rlservice = StampedRateLimiterService(throttle=False) self.__service_init_semaphore.release() return self.__local_rlservice def _getHost(self): ratelimiter_nodes = None try: ratelimiter_nodes = libs.ec2_utils.get_nodes('ratelimiter') except: logs.warning("Could not find a node with tag 'ratelimiter' on same stack") if ratelimiter_nodes is None: self.__host = 'localhost' else: self.__host = ratelimiter_nodes[0]['private_ip_address'] self.__port = 18861 def sendFailLogEmail(self): if len(self.__fails) == 0: return output = '<html>' output += "<h3>RateLimiter RPC Server Failure on %s</h3>" % self.__stack_name output += "<p>On stack '%s' instance '%s'.</p>" % (self.__stack_name, self.__node_name) output += "<p><i>There were %s failed requests to the rpc server within the last %s seconds</i></p>" %\ (self.__fail_limit, self.__fail_period) back_online = time.strftime('%m/%d/%Y %H:%M:%S', time.localtime(self.__blackout_start + self.__blackout_wait)) # Timestamp output += "<p>Waiting for %s seconds. Will use local Rate Limiter service until: %s</p>" % (self.__blackout_wait, back_online) output += '<h3>Fail Log</h3>' output += '<table border=1 cellpadding=5>' output += '<tr>' labels = ['Timestamp', 'Exception'] for label in labels: output += '<td style="font-weight:bold">%s</td>' % label output += '</tr>' for fail in self.__fails: output += '<tr>' output += '<td valign=top>%s</td>' % time.strftime('%m/%d/%Y %H:%M:%S', time.localtime(fail.timestamp)) # Timestamp output += '<td valign=top>%s</td>' % fail.exception output += '</tr>' output += '</table>' output += '</html>' try: email = {} email['from'] = 'Stamped <*****@*****.**>' email['to'] = '*****@*****.**' email['subject'] = "%s RateLimiter RPC server failure" % self.__stack_name email['body'] = output utils.sendEmail(email, format='html') except Exception as e: print('UNABLE TO SEND EMAIL: %s' % e) return output def _fail(self, exception): try: del localData.rateLimiter except: pass self._getHost() if self.__blackout_start is not None: return self.__fails.append(self.FailLog(exception)) now = time.time() cutoff = now - self.__fail_period count = 0 while len(self.__fails) > 0: if self.__fails[0].timestamp < cutoff: self.__fails.popleft() else: break count = len(self.__fails) if count >= self.__fail_limit: print('### RPC server fail threshold reached') self.__blackout_start = time.time() #self.__local_rlservice.loadDbLog() # update the local call log from the db logs.error('RPC server request FAIL THRESHOLD REACHED') # Email dev if a fail limit was reached if self.__is_ec2: if self.__last_email_time is not None and (time.time() - self.__last_email_time) > EMAIL_WAIT: self.sendFailLogEmail() self.__last_email_time = time.time() def _isBlackout(self): if self.__blackout_start is None: return False if self.__blackout_start + self.__blackout_wait > time.time(): return True else: self.__blackout_start = None self.__request_fails = 0 # self.__local_rlservice.shutdown() # self.__local_rlservice = None return False @property def _rpc_service_connection(self): try: return localData.rateLimiter except AttributeError: config = { 'allow_pickle' : True, 'allow_all_attrs' : True, 'instantiate_custom_exceptions' : True, 'import_custom_exceptions' : True, } localData.rateLimiter = rpyc.connect(self.__host, self.__port, config=config) return localData.rateLimiter def _rpc_service_request(self, service, method, url, body, header, priority, timeout): async_request = rpyc.async(self._rpc_service_connection.root.request) asyncresult = async_request(service, priority, timeout, method, url, pickle.dumps(body), pickle.dumps(header)) asyncresult.set_expiry(timeout) response, content = asyncresult.value return pickle.loads(response), content def _local_service_request(self, service, method, url, body, header, priority, timeout): response, content = self._local_rlservice.handleRequest(service, priority, timeout, method, url, body, header) return response, content def request(self, service, method, url, body, header, priority, timeout): if not self.__is_ec2 or self._isBlackout(): return self._local_service_request(service, method.upper(), url, body, header, priority, timeout) try: logs.info('### attempting rpc service request') return self._rpc_service_request(service, method.upper(), url, body, header, priority, timeout) except DailyLimitException as e: raise StampedThirdPartyRequestFailError("Hit daily rate limit for service: '%s'" % service) except WaitTooLongException as e: raise StampedThirdPartyRequestFailError("'%s' request estimated wait time longer than timeout" % service) except TimeoutException as e: raise StampedThirdPartyRequestFailError("'%s' request timed out." % service) except TooManyFailedRequestsException as e: raise StampedThirdPartyRequestFailError("%s" % e) except Exception as e: import traceback print('### caught exception type: %s e: %s' % (type(e), e)) logs.info("RPC Service Request fail." "service: %s method: %s url: %s body: %s header: %s" "priority: %s timeout: %s Exception: %s Stack: %s" % (service, method, url, body, header, priority, timeout, e, traceback.format_exc())) self._fail(e) logs.info('### Falling back to local rate limiter request') return self._local_service_request(service, method.upper(), url, body, header, priority, timeout)
class Pool(Group): def __init__(self, size=None, greenlet_class=None): if size is not None and size < 0: raise ValueError('Invalid size for pool (positive integer or None required): %r' % (size, )) Group.__init__(self) self.size = size if greenlet_class is not None: self.greenlet_class = greenlet_class if size is None: self._semaphore = DummySemaphore() else: self._semaphore = Semaphore(size) def wait_available(self): self._semaphore.wait() def full(self): return self.free_count() <= 0 def free_count(self): if self.size is None: return 1 return max(0, self.size - len(self)) def start(self, greenlet): self._semaphore.acquire() try: self.add(greenlet) except: self._semaphore.release() raise greenlet.start() def spawn(self, *args, **kwargs): self._semaphore.acquire() try: greenlet = self.greenlet_class.spawn(*args, **kwargs) self.add(greenlet) except: self._semaphore.release() raise return greenlet def spawn_link(self, *args, **kwargs): self._semaphore.acquire() try: greenlet = self.greenlet_class.spawn_link(*args, **kwargs) self.add(greenlet) except: self._semaphore.release() raise return greenlet def spawn_link_value(self, *args, **kwargs): self._semaphore.acquire() try: greenlet = self.greenlet_class.spawn_link_value(*args, **kwargs) self.add(greenlet) except: self._semaphore.release() raise return greenlet def spawn_link_exception(self, *args, **kwargs): self._semaphore.acquire() try: greenlet = self.greenlet_class.spawn_link_exception(*args, **kwargs) self.add(greenlet) except: self._semaphore.release() raise return greenlet def discard(self, greenlet): Group.discard(self, greenlet) self._semaphore.release()
class ThreadPool(object): def __init__(self, maxsize, hub=None): if hub is None: hub = get_hub() self.hub = hub self._maxsize = 0 self.manager = None self.pid = os.getpid() self.fork_watcher = hub.loop.fork(ref=False) self._init(maxsize) def _set_maxsize(self, maxsize): if not isinstance(maxsize, integer_types): raise TypeError("maxsize must be integer: %r" % (maxsize,)) if maxsize < 0: raise ValueError("maxsize must not be negative: %r" % (maxsize,)) difference = maxsize - self._maxsize self._semaphore.counter += difference self._maxsize = maxsize self.adjust() # make sure all currently blocking spawn() start unlocking if maxsize increased self._semaphore._start_notify() def _get_maxsize(self): return self._maxsize maxsize = property(_get_maxsize, _set_maxsize) def __repr__(self): return "<%s at 0x%x %s/%s/%s>" % (self.__class__.__name__, id(self), len(self), self.size, self.maxsize) def __len__(self): return self.task_queue.unfinished_tasks @property def size(self): return self._size def _init(self, maxsize): self._size = 0 self._semaphore = Semaphore(1) self._lock = Lock() self.task_queue = Queue() self._set_maxsize(maxsize) def _on_fork(self): # fork() only leaves one thread; also screws up locks; # let's re-create locks and threads pid = os.getpid() if pid != self.pid: self.pid = pid if self._size > 0: # Do not mix fork() and threads; since fork() only copies one thread # all objects referenced by other threads has refcount that will never # go down to 0. sys.stderr.write("WARNING: Mixing fork() and threads detected; memory leaked.\n") self._init(self._maxsize) def join(self): delay = 0.0005 while self.task_queue.unfinished_tasks > 0: sleep(delay) delay = min(delay * 2, 0.05) def kill(self): if self.manager: self.manager.kill() self._manage(0) def _adjust(self, maxsize): if maxsize is None: maxsize = self._maxsize while self.task_queue.unfinished_tasks > self._size and self._size < maxsize: self._add_thread() while self._size - maxsize > self.task_queue.unfinished_tasks: self.task_queue.put(None) if self._size: self.fork_watcher.start(self._on_fork) else: self.fork_watcher.stop() def _manage(self, maxsize=None): if maxsize is None: maxsize = self._maxsize delay = 0.0001 while True: self._adjust(maxsize) if self._size <= maxsize: return sleep(delay) delay = min(delay * 2, 0.05) def adjust(self): if self.manager: return if self._adjust(self.maxsize): return self.manager = Greenlet.spawn(self._manage) def _add_thread(self): with self._lock: self._size += 1 try: start_new_thread(self._worker, ()) except: with self._lock: self._size -= 1 raise def spawn(self, func, *args, **kwargs): while True: semaphore = self._semaphore semaphore.acquire() if semaphore is self._semaphore: break try: task_queue = self.task_queue result = AsyncResult() thread_result = ThreadResult(result, hub=self.hub) task_queue.put((func, args, kwargs, thread_result)) self.adjust() # rawlink() must be the last call result.rawlink(lambda *args: self._semaphore.release()) except: semaphore.release() raise return result def _decrease_size(self): if sys is None: return _lock = getattr(self, "_lock", None) if _lock is not None: with _lock: self._size -= 1 def _worker(self): need_decrease = True try: while True: task_queue = self.task_queue task = task_queue.get() try: if task is None: need_decrease = False self._decrease_size() # we want first to decrease size, then decrease unfinished_tasks # otherwise, _adjust might think there's one more idle thread that # needs to be killed return func, args, kwargs, result = task try: value = func(*args, **kwargs) except: exc_info = getattr(sys, "exc_info", None) if exc_info is None: return result.handle_error(func, exc_info()) else: if sys is None: return result.set(value) finally: if sys is None: return task_queue.task_done() finally: if need_decrease: self._decrease_size() def apply_e(self, expected_errors, function, args=None, kwargs=None): if args is None: args = () if kwargs is None: kwargs = {} success, result = self.spawn(wrap_errors, expected_errors, function, args, kwargs).get() if success: return result raise result def apply(self, func, args=None, kwds=None): """Equivalent of the apply() builtin function. It blocks till the result is ready.""" if args is None: args = () if kwds is None: kwds = {} return self.spawn(func, *args, **kwds).get() def apply_cb(self, func, args=None, kwds=None, callback=None): result = self.apply(func, args, kwds) if callback is not None: callback(result) return result def apply_async(self, func, args=None, kwds=None, callback=None): """A variant of the apply() method which returns a Greenlet object. If callback is specified then it should be a callable which accepts a single argument. When the result becomes ready callback is applied to it (unless the call failed).""" if args is None: args = () if kwds is None: kwds = {} return Greenlet.spawn(self.apply_cb, func, args, kwds, callback) def map(self, func, iterable): return list(self.imap(func, iterable)) def map_cb(self, func, iterable, callback=None): result = self.map(func, iterable) if callback is not None: callback(result) return result def map_async(self, func, iterable, callback=None): """ A variant of the map() method which returns a Greenlet object. If callback is specified then it should be a callable which accepts a single argument. """ return Greenlet.spawn(self.map_cb, func, iterable, callback) def imap(self, func, iterable): """An equivalent of itertools.imap()""" return IMap.spawn(func, iterable, spawn=self.spawn) def imap_unordered(self, func, iterable): """The same as imap() except that the ordering of the results from the returned iterator should be considered in arbitrary order.""" return IMapUnordered.spawn(func, iterable, spawn=self.spawn)
class BaseDataHandler(object): def __init__(self, dh_config): ''' Constructor for all data handlers. @param dh_config: Dictionary containing configuration parameters for the data handler ''' self._polling = False # Moved these four variables so they're instance variables, not class variables self._polling_glet = None self._dh_config = {} self._terminate_polling = None self._acquiring_data = None self._params = { 'POLLING_INTERVAL': 3600, 'PATCHABLE_CONFIG_KEYS': ['stream_id', 'constraints', 'stream_route'] } self._dh_config = dh_config self._semaphore = Semaphore() def set_event_callback(self, evt_callback): """ Sets a callback function to be triggered on events @param evt_callback: """ self._event_callback = evt_callback def _dh_event(self, type, value): event = { 'type': type, 'value': value, 'time': time.time() } self._event_callback(event) def _poll(self): """ Internal polling method, run inside a greenlet, that triggers execute_acquire_sample without configuration mods The polling interval (in seconds) is retrieved from the POLLING_INTERVAL parameter """ self._polling = True self._terminate_polling = Event() interval = get_safe(self._params, 'POLLING_INTERVAL', 3600) log.debug('Polling interval: {0}'.format(interval)) while not self._terminate_polling.wait(timeout=interval): self.execute_acquire_sample() def cmd_dvr(self, cmd, *args, **kwargs): """ Command a DataHandler by request-reply messaging. Package command message and send on blocking command socket. Block on same socket to receive the reply. Return the driver reply. @param cmd The DataHandler command identifier. @param args Positional arguments of the command. @param kwargs Keyword arguments of the command. @throws InstrumentCommandException if the command is not recognized @retval Command result. """ # Package command dictionary. #need to account for observatory_execute_resource commands #connect -> Not used #get_current_state -> Not used #discover -> Not used #disconnect -> Not used log.debug('cmd_dvr received command \'{0}\' with: args={1} kwargs={2}'.format(cmd, args, kwargs)) reply = None if cmd == 'initialize': # Delegate to BaseDataHandler.initialize() reply = self.initialize(*args, **kwargs) elif cmd == 'get_resource': # Delegate to BaseDataHandler.get() reply = self.get(*args, **kwargs) elif cmd == 'set_resource': # Delegate to BaseDataHandler.set() reply = self.set(*args, **kwargs) elif cmd == 'get_resource_params': # Delegate to BaseDataHandler.get_resource_params() reply = self.get_resource_params(*args, **kwargs) elif cmd == 'get_resource_commands': # Delegate to BaseDataHandler.get_resource_commands() reply = self.get_resource_commands(*args, **kwargs) elif cmd == 'get_resource_capabilities': # Delegate to BaseDataHandler.get_resource_commands() reply = self.get_resource_capabilities(*args, **kwargs) elif cmd == 'execute_acquire_sample': # Delegate to BaseDataHandler.execute_acquire_sample() reply = self.execute_acquire_sample(*args, **kwargs) elif cmd == 'execute_start_autosample': # Delegate to BaseDataHandler.execute_start_autosample() reply = self.execute_start_autosample(*args, **kwargs) elif cmd == 'execute_stop_autosample': # Delegate to BaseDataHandler.execute_stop_autosample() reply = self.execute_stop_autosample(*args, **kwargs) elif cmd == 'execute_resource': reply = self.execute_resource(*args, **kwargs) elif cmd in ['configure', 'connect', 'disconnect', 'get_current_state', 'discover', 'execute_acquire_data']: # Disregard log.info('Command \'{0}\' not used by DataHandler'.format(cmd)) pass else: desc = 'Command \'{0}\' unknown by DataHandler'.format(cmd) log.info(desc) raise InstrumentCommandException(desc) return reply def initialize(self, *args, **kwargs): #TODO: Add documentation #TODO: This should put the DataHandler back into an 'unconfigured' state """ Called from: InstrumentAgent._handler_idle_reset InstrumentAgent._handler_idle_go_inactive InstrumentAgent._handler_stopped_reset InstrumentAgent._handler_stopped_go_inactive InstrumentAgent._handler_observatory_reset InstrumentAgent._handler_observatory_go_inactive InstrumentAgent._handler_uninitialized_initialize |--> ExternalDataAgent._start_driver @param args Positional arguments of the command. @param kwargs Keyword arguments of the command. """ log.debug('Initializing DataHandler...') self._glet_queue = [] return None def configure(self, *args, **kwargs): #TODO: Add documentation #TODO: This should configure the DataHandler for the particular dataset """ Called from: InstrumentAgent._handler_inactive_go_active @param args First argument should be a config dictionary @param kwargs Keyword arguments of the command. @throws InstrumentParameterException if the first argument isn't a config dictionary """ log.debug('Configuring DataHandler: args = {0}'.format(args)) try: self._dh_config = args[0] except IndexError: raise InstrumentParameterException('\'acquire_sample\' command requires a config dict as the first argument') return def execute_acquire_sample(self, *args): """ Creates a copy of self._dh_config, creates a publisher, and spawns a greenlet to perform a data acquisition cycle If the args[0] is a dict, any entries keyed with one of the 'PATCHABLE_CONFIG_KEYS' are used to patch the config Greenlet binds to BaseDataHandler._acquire_sample and passes the publisher and config Disallows multiple "new data" (unconstrained) requests using BaseDataHandler._semaphore lock Called from: InstrumentAgent._handler_observatory_execute_resource |--> ExternalDataAgent._handler_streaming_execute_resource @parameter args First argument can be a config dictionary @throws IndexError if first argument is not a dictionary @throws ConfigurationError if required members aren't present @retval New ResourceAgentState (COMMAND) """ log.debug('Executing acquire_sample: args = {0}'.format(args)) # Make a copy of the config to ensure no cross-pollution config = self._dh_config.copy() # Patch the config if mods are passed in try: config_mods = args[0] if not isinstance(config_mods, dict): raise IndexError() log.debug('Configuration modifications provided: {0}'.format(config_mods)) for k in self._params['PATCHABLE_CONFIG_KEYS']: p = get_safe(config_mods, k) if not p is None: config[k] = p except IndexError: log.info('No configuration modifications were provided') # Verify that there is a stream_id member in the config stream_id = get_safe(config, 'stream_id', None) if not stream_id: raise ConfigurationError('Configuration does not contain required \'stream_id\' member') stream_route = get_safe(config, 'stream_route', None) if not stream_route: raise ConfigurationError('Configuration does not contain required \'stream_route\' member') isNew = get_safe(config, 'constraints') is None if isNew and not self._semaphore.acquire(blocking=False): log.warn('Already acquiring new data - action not duplicated') return ndc = None if isNew: # Get the NewDataCheck attachment and add it's content to the config ext_ds_id = get_safe(config, 'external_dataset_res_id') if ext_ds_id: ndc = self._find_new_data_check_attachment(ext_ds_id) config['new_data_check'] = ndc # Create a publisher to pass into the greenlet publisher = StandaloneStreamPublisher(stream_id=stream_id, stream_route=stream_route) # Spawn a greenlet to do the data acquisition and publishing g = spawn(self._acquire_sample, config, publisher, self._unlock_new_data_callback, self._update_new_data_check_attachment) log.debug('** Spawned {0}'.format(g)) self._glet_queue.append(g) return ResourceAgentState.COMMAND, None def execute_resource(self, *args, **kwargs): """ Function to acquire data and start/stop autosample @param args: first argument should be the command to run, second optional argument is a config dictionary @param kwargs: Keyword arguments of the command @retval Next ResourceAgentState """ cmd = args[0] if cmd == DriverEvent.ACQUIRE_SAMPLE: if len(args) == 1: return self.execute_acquire_sample() else: return self.execute_acquire_sample(args[1]) elif cmd == DriverEvent.START_AUTOSAMPLE: if len(args) == 1: return self.execute_start_autosample() else: return self.execute_start_autosample(args[1]) elif cmd == DriverEvent.STOP_AUTOSAMPLE: if len(args) == 1: return self.execute_stop_autosample() else: return self.execute_stop_autosample(args[1]) def execute_start_autosample(self, *args, **kwargs): #TODO: Add documentation #TODO: Fix raises statements """ Put the DataHandler into streaming mode and start polling for new data Called from: InstrumentAgent._handler_observatory_go_streaming @raises InstrumentTimeoutException: @raises InstrumentProtocolException: @raises NotImplementedException: @raises InstrumentParameterException: @retval Next ResourceAgentState (STREAMING) """ log.debug('Entered execute_start_autosample with args={0} & kwargs={1}'.format(args, kwargs)) if not self._polling and self._polling_glet is None: self._polling_glet = spawn(self._poll) return ResourceAgentState.STREAMING, None def execute_stop_autosample(self, *args, **kwargs): #TODO: Add documentation #TODO: Fix raises statements """ Stop polling for new data and put the DataHandler into observatory mode Called from: InstrumentAgent._handler_streaming_go_observatory @raises InstrumentTimeoutException: @raises InstrumentProtocolException: @raises NotImplementedException: @raises InstrumentParameterException: @retval Next ResourceAgentState (COMMAND) """ log.debug('Entered execute_stop_autosample with args={0} & kwargs={1}'.format(args, kwargs)) if self._polling and not self._polling_glet is None: log.debug("Terminating polling") self._terminate_polling.set() self._polling_glet.join(timeout=30) log.debug("Polling terminated") self._polling = False self._polling_glet = None return ResourceAgentState.COMMAND, None def get(self, *args, **kwargs): #TODO: Add documentation #TODO: Fix raises statements """ Retrieve required parameter Called from: InstrumentAgent._handler_get_params @throws InstrumentTimeoutException: @throws InstrumentProtocolException: @throws NotImplementedException: @throws InstrumentParameterException: if paramter is not a list or not found @retval Parameter value """ try: pnames = args[0] except IndexError: log.warn("No argument provided to get, return all parameters") pnames = [DataHandlerParameter.ALL] result = None if DataHandlerParameter.ALL in pnames: result = self._params else: if not isinstance(pnames, (list, tuple)): raise InstrumentParameterException('Get argument not a list or tuple: {0}'.format(pnames)) result = {} for pn in pnames: try: log.debug('Get parameter with key: {0}'.format(pn)) result[pn] = self._params[pn] except KeyError: log.debug('\'{0}\' not found in self._params'.format(pn)) raise InstrumentParameterException('{0} is not a valid parameter for this DataHandler.'.format(pn)) return result def set(self, *args, **kwargs): #TODO: Add documentation #TODO: Fix raises statements """ Set required parameter. Called from: InstrumentAgent._handler_observatory_set_params @throws InstrumentTimeoutException: @throws InstrumentProtocolException: @throws NotImplementedException: @throws InstrumentParameterException: Invalid parameter or no parameter dictionary provided @retval None """ try: params = args[0] except IndexError: raise InstrumentParameterException('Set command requires a parameter dict.') to_raise = [] if not isinstance(params, dict): raise InstrumentParameterException('Set parameters not a dict.') else: for (key, val) in params.iteritems(): if key in self._params: log.debug('Set parameter \'{0}\' = {1}'.format(key, val)) self._params[key] = val else: log.debug('Parameter \'{0}\' not in self._params and cannot be set'.format(key)) to_raise.append(key) if len(to_raise) > 0: log.debug('Raise InstrumentParameterException for un-set parameters: {0}'.format(to_raise)) raise InstrumentParameterException('Invalid parameter(s) could not be set: {0}'.format(to_raise)) def get_resource_params(self, *args, **kwargs): """ Return list of resource parameters. Implemented in specific DataHandlers Called from: InstrumentAgent._handler_get_resource_params @retval list of resource parameters """ return self._params.keys() def get_resource_capabilities(self, *args, **kwargs): """ Return list of DataHandler execute commands available. Called from: InstrumentAgent._handler_get_resource_capabilities @retval list of available execute commands and parameters """ res_cmds = [cmd.replace('execute_', '') for cmd in dir(self) if cmd.startswith('execute_')] res_params = ['POLLING_INTERVAL', 'PATCHABLE_CONFIG_KEYS'] result = [res_cmds, res_params] return result def get_resource_commands(self, *args, **kwargs): """ Return list of DataHandler execute commands available. Called from: InstrumentAgent._handler_get_resource_commands @retval list of available execute commands """ cmds = [cmd.replace('execute_', '') for cmd in dir(self) if cmd.startswith('execute_')] return cmds def _unlock_new_data_callback(self, caller): """ Release the polling semaphore @retval None """ log.debug('** Release {0}'.format(caller)) self._semaphore.release() def _find_new_data_check_attachment(self, res_id): """ Returns a list of the last data files that were found @param res_id The resource ID of the external dataset resource @throws InstrumentException if no attachment is found @retval list of data file names and sizes """ rr_cli = ResourceRegistryServiceClient() try: attachment_objs = rr_cli.find_attachments(resource_id=res_id, include_content=False, id_only=False) for attachment_obj in attachment_objs: kwds = set(attachment_obj.keywords) if 'NewDataCheck' in kwds: log.debug('Found NewDataCheck attachment: {0}'.format(attachment_obj._id)) return msgpack.unpackb(attachment_obj.content) else: log.debug('Found attachment: {0}'.format(attachment_obj)) except NotFound: raise InstrumentException('ExternalDatasetResource \'{0}\' not found'.format(res_id)) @classmethod def _update_new_data_check_attachment(cls, res_id, new_content): """ Update the list of data files for the external dataset resource @param res_id the ID of the external dataset resource @param new_content list of new files found on data host @throws InstrumentException if external dataset resource can't be found @retval the attachment ID """ rr_cli = ResourceRegistryServiceClient() try: # Delete any attachments with the "NewDataCheck" keyword attachment_objs = rr_cli.find_attachments(resource_id=res_id, include_content=False, id_only=False) for attachment_obj in attachment_objs: kwds = set(attachment_obj.keywords) if 'NewDataCheck' in kwds: log.debug('Delete NewDataCheck attachment: {0}'.format(attachment_obj._id)) rr_cli.delete_attachment(attachment_obj._id) else: log.debug('Found attachment: {0}'.format(attachment_obj)) # Create the new attachment att = Attachment(name='new_data_check', attachment_type=AttachmentType.ASCII, keywords=['NewDataCheck', ], content_type='text/plain', content=msgpack.packb(new_content)) att_id = rr_cli.create_attachment(resource_id=res_id, attachment=att) except NotFound: raise InstrumentException('ExternalDatasetResource \'{0}\' not found'.format(res_id)) return att_id @classmethod def _acquire_sample(cls, config, publisher, unlock_new_data_callback, update_new_data_check_attachment): """ Ensures required keys (such as stream_id) are available from config, configures the publisher and then calls: BaseDataHandler._constraints_for_new_request (only if config does not contain 'constraints') BaseDataHandler._publish_data passing BaseDataHandler._get_data as a parameter @param config Dict containing configuration parameters, may include constraints, formatters, etc @param publisher the publisher used to publish data @param unlock_new_data_callback BaseDataHandler callback function to allow conditional unlocking of the BaseDataHandler._semaphore @param update_new_data_check_attachment classmethod to update the external dataset resources file list attachment @throws InstrumentParameterException if the data constraints are not a dictionary @retval None """ log.debug('start _acquire_sample: config={0}'.format(config)) cls._init_acquisition_cycle(config) constraints = get_safe(config, 'constraints') if not constraints: gevent.getcurrent().link(unlock_new_data_callback) try: constraints = cls._constraints_for_new_request(config) except NoNewDataWarning: #log.info(nndw.message) if get_safe(config, 'TESTING'): #log.debug('Publish TestingFinished event') pub = EventPublisher('DeviceCommonLifecycleEvent') pub.publish_event(origin='BaseDataHandler._acquire_sample', description='TestingFinished') return if constraints is None: raise InstrumentParameterException("Data constraints returned from _constraints_for_new_request cannot be None") config['constraints'] = constraints elif isinstance(constraints, dict): addnl_constr = cls._constraints_for_historical_request(config) if not addnl_constr is None and isinstance(addnl_constr, dict): constraints.update(addnl_constr) else: raise InstrumentParameterException('Data constraints must be of type \'dict\': {0}'.format(constraints)) cls._publish_data(publisher, cls._get_data(config)) if 'set_new_data_check' in config: update_new_data_check_attachment(config['external_dataset_res_id'], config['set_new_data_check']) # Publish a 'TestFinished' event if get_safe(config, 'TESTING'): #log.debug('Publish TestingFinished event') pub = EventPublisher('DeviceCommonLifecycleEvent') pub.publish_event(origin='BaseDataHandler._acquire_sample', description='TestingFinished') @classmethod def _constraints_for_historical_request(cls, config): """ Determines any constraints that must be added to the constraints configuration. This should present a uniform constraints configuration to be sent to _get_data @param config Dict containing configuration parameters, may include constraints, formatters, etc @retval dictionary containing the restraints """ raise NotImplementedException('{0}.{1} must implement \'_constraints_for_historical_request\''.format(cls.__module__, cls.__name__)) @classmethod def _init_acquisition_cycle(cls, config): """ Allows the concrete implementation to initialize/prepare objects the data handler will use repeatedly (such as a dataset object) in cls._constraints_for_new_request and/or cls._get_data Objects should be added to the config so they are available later in the workflow @param config Dict containing configuration parameters, may include constraints, formatters, etc """ raise NotImplementedException('{0}.{1} must implement \'_init_acquisition_cycle\''.format(cls.__module__, cls.__name__)) @classmethod def _constraints_for_new_request(cls, config): #TODO: Document what "constraints" looks like (yml)!! """ Determines the appropriate constraints for acquiring any "new data" from the external dataset Returned value cannot be None and is assigned to config['constraints'] The format of the constraints are documented: @param config dict of configuration parameters - may be used to generate the returned 'constraints' dict @retval dict that contains the constraints for retrieval of new data from the external dataset """ raise NotImplementedException('{0}.{1} must implement \'_constraints_for_new_request\''.format(cls.__module__, cls.__name__)) @classmethod def _get_data(cls, config): """ Iterable function that acquires data from a source iteratively based on constraints provided by config Passed into BaseDataHandler._publish_data and iterated to publish samples. @param config dict containing configuration parameters, may include constraints, formatters, etc @retval an iterable that returns well-formed Granule objects on each iteration """ raise NotImplementedException('{0}.{1} must implement \'_get_data\''.format(cls.__module__, cls.__name__)) @classmethod def _publish_data(cls, publisher, data_generator): """ Iterates over the data_generator and publishes granules to the stream indicated in stream_id @param publisher to publish the data with @param data_generator enumerator to cycle through the data @throws InstrumentDataException if data_generator isn't an enumerator """ if data_generator is None or not hasattr(data_generator, '__iter__'): raise InstrumentDataException('Invalid object returned from _get_data: returned object cannot be None and must have \'__iter__\' attribute') for count, gran in enumerate(data_generator): if isinstance(gran, Granule): #log.warn('_publish_data: {0}\n{1}'.format(count, gran)) publisher.publish(gran) else: log.warn('Could not publish object of {0} returned by _get_data: {1}'.format(type(gran), gran)) publisher.close()
class ThreadPool(object): def __init__(self, maxsize, hub=None): if hub is None: hub = get_hub() self.hub = hub self._maxsize = 0 self._init(maxsize) self.pid = os.getpid() self.fork_watcher = hub.loop.fork(ref=False) self.fork_watcher.start(self._on_fork) def _set_maxsize(self, maxsize): if not isinstance(maxsize, integer_types): raise TypeError('maxsize must be integer: %r' % (maxsize, )) if maxsize < 0: raise ValueError('maxsize must not be negative: %r' % (maxsize, )) difference = maxsize - self._maxsize self._semaphore.counter += difference self._maxsize = maxsize self._remove_threads() self._add_threads() # make sure all currently blocking spawn() start unlocking if maxsize increased self._semaphore._start_notify() def _get_maxsize(self): return self._maxsize maxsize = property(_get_maxsize, _set_maxsize) def __repr__(self): return '<%s at 0x%x %s/%s/%s>' % (self.__class__.__name__, id(self), len(self), self.size, self.maxsize) def __len__(self): return self.task_queue.unfinished_tasks @property def size(self): return self._size def _init(self, maxsize): self._size = 0 self._semaphore = Semaphore(1) self._lock = Lock() self.task_queue = Queue() self._set_maxsize(maxsize) def _on_fork(self): # fork() only leaves one thread; also screws up locks; # let's re-create locks and threads pid = os.getpid() if pid != self.pid: self.pid = pid if self._size > 0: # Do not mix fork() and threads; since fork() only copies one thread # all objects referenced by other threads has refcount that will never # go down to 0. sys.stderr.write( "WARNING: Mixing fork() and threads detected; memory leaked.\n" ) self._init(self._maxsize) def join(self): delay = 0.0005 while self.task_queue.unfinished_tasks > 0: sleep(delay) delay = min(delay * 2, .05) def kill(self): delay = 0.0005 while self._size > 0: self._remove_threads(0) sleep(delay) delay = min(delay * 2, .05) def _add_threads(self): while self.task_queue.unfinished_tasks > self._size: if self._size >= self.maxsize: break self._add_thread() def _remove_threads(self, maxsize=None): if maxsize is None: maxsize = self._maxsize excess = self._size - maxsize if excess > 0: while excess > self.task_queue.qsize(): self.task_queue.put(None) def _add_thread(self): with self._lock: self._size += 1 try: start_new_thread(self._worker, ()) except: with self._lock: self._size -= 1 raise def spawn(self, func, *args, **kwargs): while True: semaphore = self._semaphore semaphore.acquire() if semaphore is self._semaphore: break try: task_queue = self.task_queue result = AsyncResult() tr = ThreadResult(result, hub=self.hub) self._remove_threads() task_queue.put((func, args, kwargs, tr)) self._add_threads() result.rawlink(lambda *args: self._semaphore.release()) except: semaphore.release() raise return result def _worker(self): try: while True: task_queue = self.task_queue task = task_queue.get() try: if task is None: return func, args, kwargs, result = task try: value = func(*args, **kwargs) except: exc_info = getattr(sys, 'exc_info', None) if exc_info is None: return result.set_exception(exc_info()[1]) else: if sys is None: return result.set(value) finally: if sys is None: return task_queue.task_done() finally: if sys is None: return _lock = getattr(self, '_lock', None) if _lock is not None: with _lock: self._size -= 1 def apply(self, func, args=None, kwds=None): """Equivalent of the apply() builtin function. It blocks till the result is ready.""" if args is None: args = () if kwds is None: kwds = {} return self.spawn(func, *args, **kwds).get() def apply_cb(self, func, args=None, kwds=None, callback=None): result = self.apply(func, args, kwds) if callback is not None: callback(result) return result def apply_async(self, func, args=None, kwds=None, callback=None): """A variant of the apply() method which returns a Greenlet object. If callback is specified then it should be a callable which accepts a single argument. When the result becomes ready callback is applied to it (unless the call failed).""" if args is None: args = () if kwds is None: kwds = {} return Greenlet.spawn(self.apply_cb, func, args, kwds, callback) def map(self, func, iterable): return list(self.imap(func, iterable)) def map_cb(self, func, iterable, callback=None): result = self.map(func, iterable) if callback is not None: callback(result) return result def map_async(self, func, iterable, callback=None): """ A variant of the map() method which returns a Greenlet object. If callback is specified then it should be a callable which accepts a single argument. """ return Greenlet.spawn(self.map_cb, func, iterable, callback) def imap(self, func, iterable): """An equivalent of itertools.imap()""" return IMap.spawn(func, iterable, spawn=self.spawn) def imap_unordered(self, func, iterable): """The same as imap() except that the ordering of the results from the returned iterator should be considered in arbitrary order.""" return IMapUnordered.spawn(func, iterable, spawn=self.spawn)
class VBucketAwareCouchbaseClient(object): #poll server every few seconds to see if the vbucket-map #has changes def __init__(self, url, bucket, password="", verbose=False): self.log = logger.logger("VBucketAwareMemcachedClient") self.bucket = bucket self.rest_username = bucket self.rest_password = password self._memcacheds = {} self._vBucketMap = {} self._vBucketMap_lock = Lock() self._vBucketMapFastForward = {} self._vBucketMapFastForward_lock = Lock() #TODO: use regular expressions to parse the url server = {} if not bucket: raise InvalidArgumentException("bucket can not be an empty string", parameters="bucket") if not url: raise InvalidArgumentException("url can not be an empty string", parameters="url") if url.find("http://") != -1 and url.rfind(":") != -1 and url.find( "/pools/default") != -1: server["ip"] = url[url.find("http://") + len("http://"):url.rfind(":")] server["port"] = url[url.rfind(":") + 1:url.find("/pools/default")] server["username"] = self.rest_username server["password"] = self.rest_password self.servers = [server] self.servers_lock = Lock() self.rest = RestConnection(server) self.reconfig_vbucket_map() self.init_vbucket_connections() self.dispatcher = CommandDispatcher(self) self.dispatcher_thread = Process(name="dispatcher-thread", target=self._start_dispatcher) self.dispatcher_thread.daemon = True self.dispatcher_thread.start() self.streaming_thread = Process(name="streaming", target=self._start_streaming, args=()) self.streaming_thread.daemon = True self.streaming_thread.start() self.verbose = verbose def _start_dispatcher(self): self.dispatcher.dispatch() def _start_streaming(self): # this will dynamically update vBucketMap, vBucketMapFastForward, servers urlopener = urllib.FancyURLopener() urlopener.prompt_user_passwd = lambda host, realm: (self.rest_username, self.rest_password) current_servers = True while current_servers: self.servers_lock.acquire() current_servers = deepcopy(self.servers) self.servers_lock.release() for server in current_servers: response = urlopener.open( "http://{0}:{1}/pools/default/bucketsStreaming/{2}".format( server["ip"], server["port"], self.bucket)) while response: try: line = response.readline() if not line: # try next server if we get an EOF response.close() break except: # try next server if we fail to read response.close() break try: data = json.loads(line) except: continue serverlist = data['vBucketServerMap']['serverList'] vbucketmapfastforward = {} index = 0 if 'vBucketMapForward' in data['vBucketServerMap']: for vbucket in data['vBucketServerMap'][ 'vBucketMapForward']: vbucketmapfastforward[index] = serverlist[ vbucket[0]] index += 1 self._vBucketMapFastForward_lock.acquire() self._vBucketMapFastForward = deepcopy( vbucketmapfastforward) self._vBucketMapFastForward_lock.release() vbucketmap = {} index = 0 for vbucket in data['vBucketServerMap']['vBucketMap']: vbucketmap[index] = serverlist[vbucket[0]] index += 1 # only update vBucketMap if we don't have a fastforward # on a not_mb_vbucket error, we already update the # vBucketMap from the fastforward map if not vbucketmapfastforward: self._vBucketMap_lock.acquire() self._vBucketMap = deepcopy(vbucketmap) self._vBucketMap_lock.release() new_servers = [] nodes = data["nodes"] for node in nodes: if node["clusterMembership"] == "active" and node[ "status"] == "healthy": hostport = node["hostname"] new_servers.append({ "ip": hostport.split(":")[0], "port": int(hostport.split(":")[1]), "username": self.rest_username, "password": self.rest_password }) new_servers.sort() self.servers_lock.acquire() self.servers = deepcopy(new_servers) self.servers_lock.release() def init_vbucket_connections(self): # start up all vbucket connections self._vBucketMap_lock.acquire() vbucketcount = len(self._vBucketMap) self._vBucketMap_lock.release() for i in range(vbucketcount): self.start_vbucket_connection(i) def start_vbucket_connection(self, vbucket): self._vBucketMap_lock.acquire() server = deepcopy(self._vBucketMap[vbucket]) self._vBucketMap_lock.release() serverIp, serverPort = server.split(":") if not server in self._memcacheds: self._memcacheds[server] = MemcachedClientHelper.direct_client( self.rest, serverIp, serverPort, self.bucket) def start_vbucket_fastforward_connection(self, vbucket): self._vBucketMapFastForward_lock.acquire() if not vbucket in self._vBucketMapFastForward: self._vBucketMapFastForward_lock.release() return server = deepcopy(self._vBucketMapFastForward[vbucket]) self._vBucketMapFastForward_lock.release() serverIp, serverPort = server.split(":") if not server in self._memcacheds: self._memcacheds[server] = MemcachedClientHelper.direct_client( self.rest, serverIp, serverPort, self.bucket) def restart_vbucket_connection(self, vbucket): self._vBucketMap_lock.acquire() server = deepcopy(self._vBucketMap[vbucket]) self._vBucketMap_lock.release() serverIp, serverPort = server.split(":") if server in self._memcacheds: self._memcacheds[server].close() self._memcacheds[server] = MemcachedClientHelper.direct_client( self.rest, serverIp, serverPort, self.bucket) def reconfig_vbucket_map(self, vbucket=-1): vb_ready = RestHelper(self.rest).vbucket_map_ready(self.bucket, 60) if not vb_ready: raise Exception("vbucket map is not ready for bucket {0}".format( self.bucket)) vBuckets = self.rest.get_vbuckets(self.bucket) self.vbucket_count = len(vBuckets) bucket_info = self.rest.get_bucket(self.bucket) nodes = bucket_info.nodes self._vBucketMap_lock.acquire() for vBucket in vBuckets: if vBucket.id == vbucket or vbucket == -1: self._vBucketMap[vBucket.id] = vBucket.master self._vBucketMap_lock.release() def memcached(self, key, fastforward=False): self._vBucketMap_lock.acquire() self._vBucketMapFastForward_lock.acquire() vBucketId = (zlib.crc32(key) >> 16) & (len(self._vBucketMap) - 1) if fastforward and vBucketId in self._vBucketMapFastForward: # only try the fastforward if we have an entry # otherwise we just wait for the main map to update self.start_vbucket_fastforward_connection(vBucketId) self._vBucketMap[vBucketId] = self._vBucketMapFastForward[ vBucketId] if vBucketId not in self._vBucketMap: msg = "vbucket map does not have an entry for vb : {0}" self._vBucketMapFastForward_lock.release() self._vBucketMap_lock.release() raise Exception(msg.format(vBucketId)) if self._vBucketMap[vBucketId] not in self._memcacheds: msg = "smart client does not have a mc connection for server : {0}" self._vBucketMapFastForward_lock.release() self._vBucketMap_lock.release() raise Exception(msg.format(self._vBucketMap[vBucketId])) r = self._memcacheds[self._vBucketMap[vBucketId]] self._vBucketMapFastForward_lock.release() self._vBucketMap_lock.release() return r def vbucketid(self, key): self._vBucketMap_lock.acquire() r = (zlib.crc32(key) >> 16) & (len(self._vBucketMap) - 1) self._vBucketMap_lock.release() return r def done(self): if self.dispatcher: self.dispatcher.shutdown() if self.verbose: self.log.info("dispatcher shutdown invoked") [self._memcacheds[ip].close() for ip in self._memcacheds] if self.verbose: self.log.info("closed all memcached open connections") self.dispatcher = None def _respond(self, item, event): timeout = 30 event.wait(timeout) if not event.is_set(): # if we timeout, then try to reconnect to the server # responsible for this vbucket self.restart_vbucket_connection(self.vbucketid(item['key'])) raise MemcachedTimeoutException(item, timeout) if "error" in item["response"]: raise item["response"]["error"] return item["response"]["return"] def get(self, key): event = Event() item = {"operation": "get", "key": key, "event": event, "response": {}} self.dispatcher.put(item) return self._respond(item, event) def gat(self, key, expiry): event = Event() item = { "operation": "gat", "key": key, "expiry": expiry, "event": event, "response": {} } self.dispatcher.put(item) return self._respond(item, event) def touch(self, key, expiry): event = Event() item = { "operation": "touch", "key": key, "expiry": expiry, "event": event, "response": {} } self.dispatcher.put(item) return self._respond(item, event) def cas(self, key, expiry, flags, old_value, value): event = Event() item = { "operation": "cas", "key": key, "expiry": expiry, "flags": flags, "old_value": old_value, "value": value, "event": event, "response": {} } self.dispatcher.put(item) return self._respond(item, event) def decr(self, key, amount=1, init=0, expiry=0): event = Event() item = { "operation": "decr", "key": key, "amount": amount, "init": init, "expiry": expiry, "event": event, "response": {} } self.dispatcher.put(item) return self._respond(item, event) def set(self, key, expiry, flags, value): event = Event() item = { "operation": "set", "key": key, "expiry": expiry, "flags": flags, "value": value, "event": event, "response": {} } self.dispatcher.put(item) return self._respond(item, event) def add(self, key, expiry, flags, value): event = Event() item = { "operation": "add", "key": key, "expiry": expiry, "flags": flags, "value": value, "event": event, "response": {} } self.dispatcher.put(item) return self._respond(item, event) def append(self, key, value, cas=0): event = Event() item = { "operation": "append", "key": key, "cas": cas, "value": value, "event": event, "response": {} } self.dispatcher.put(item) return self._respond(item, event) def delete(self, key, cas=0): event = Event() item = { "operation": "delete", "key": key, "cas": cas, "event": event, "response": {} } self.dispatcher.put(item) return self._respond(item, event) def prepend(self, key, value, cas=0): event = Event() item = { "operation": "prepend", "key": key, "cas": cas, "value": value, "event": event, "response": {} } self.dispatcher.put(item) return self._respond(item, event) def getl(self, key, expiry=15): event = Event() item = { "operation": "getl", "key": key, "expiry": expiry, "event": event, "response": {} } self.dispatcher.put(item) return self._respond(item, event) def replace(self, key, expiry, flags, value): event = Event() item = { "operation": "replace", "key": key, "expiry": expiry, "flags": flags, "value": value, "event": event, "response": {} } self.dispatcher.put(item) return self._respond(item, event) def incr(self, key, amount=1, init=0, expiry=0): event = Event() item = { "operation": "incr", "key": key, "amount": amount, "init": init, "expiry": expiry, "event": event, "response": {} } self.dispatcher.put(item) return self._respond(item, event)
class DMServerCore(OperationRequest): def __init__(self, server_ip, server_port, client_ip, client_port): super(DMServerCore, self).__init__() self.lwm2m_dm_server_ip = server_ip self.lwm2m_dm_server_port = server_port self.local_client_ip_ = client_ip self.local_client_port_ = client_port self.sem = Semaphore() self.sem_counter = 0 self.lwm2m_resources = LWM2MResourceTree() self.registration = Registration(self.lwm2m_resources) self.execution = Execution(self.lwm2m_resources) self.discover = Discovery(lwm2m_resources=self.lwm2m_resources) self.observation = ObservationNotificationEngine(self.lwm2m_resources) self.read = Read(self.lwm2m_resources) self.write = Write(self.lwm2m_resources) self.create_object_instance = Create(self.lwm2m_resources) self.write_attributes = WriteAttributes(self.lwm2m_resources) def create_server(self, ): """ Creates and starts a LWM2M DM Server using Gevent DatagramServer. The server listens at the ip and port specified below. A handler is used to entertain the requests coming at that port """ self.dm_server = DatagramServer((self.lwm2m_dm_server_ip, \ self.lwm2m_dm_server_port), self.handle_request) self.dm_server.start() def stop_server(self, ): """ Stops the LWM2M DM Server """ self.dm_server.stop() def handle_request(self, message, remote): """ Handles the requests coming at the specified ip and port """ rx_record = connection.ReceptionRecord(None, message, remote) msg = rx_record.message uri_query = msg.findOption(options.UriQuery) self.process(rx_record, remote, uri_query) def handle_lwm2m_put(self, msg, uri_query, remote, rx_record): """ It consists of Normal Update, Write Operation, Write Attribute Operation. Write Operation is used to update the resource(s) as per the request. Write Attributes operation is used to update the attributes of the object, object instance or resource. """ method = None try: method = uri_query[0].value.split("=")[1] except: pass if method == "write": path = msg.findOption(URI_PATH_VALUE) content_type_number = msg.findOption(options.ContentType) if content_type_number is None: content_type = "text/plain" else: content_type = constants.media_types[content_type_number.value] self.write.write_resource(msg.payload, path, content_type) payload_forward = msg.payload msg = connection.Message(connection.Message.ACK, code=constants.CHANGED, payload="Resource Updated") self.dm_server.sendto(msg._pack(rx_record.transaction_id), remote) client_port = self.generate_client_port() self.write.forward_write_request(path, payload_forward, \ content_type, remote, client_port) elif method == "write_attributes": path = msg.findOption(URI_PATH_VALUE) content_type_number = msg.findOption(options.ContentType) if content_type_number is None: content_type = "text/plain" else: content_type = constants.media_types[content_type_number.value] payload = loads(msg.payload) self.write_attributes.set_attributes(path, remote, payload) msg = connection.Message(connection.Message.ACK, code=constants.CHANGED, payload="Resource Attributes Updated") self.dm_server.sendto(msg._pack(rx_record.transaction_id), remote) client_port = self.generate_client_port() self.write_attributes.forward_request(path, remote, payload, content_type, client_port) else: endpoint_location = msg.findOption(URI_PATH_VALUE)[0].value if msg.payload == "": self.logger.info("Updating the Registration Params") endpoint_object = self.lwm2m_resources.return_endpoint_object( endpoint_location=endpoint_location) endpoint_object.listener_ip = uri_query[6].value.split("=")[1] endpoint_object.local_ip = uri_query[6].value.split("=")[1] msg = connection.Message(connection.Message.ACK, code=constants.CHANGED, payload="Resource Updated") self.dm_server.sendto(msg._pack(rx_record.transaction_id), remote) else: self.logger.info("Adding/Updating the Resources") payload = self.update_resource( loads(msg.payload), endpoint_location=endpoint_location) msg = connection.Message(connection.Message.ACK, code=constants.CHANGED, payload="Resource Updated") self.dm_server.sendto(msg._pack(rx_record.transaction_id), remote) self.logger.info("Forwarding the Notification") request = lwm2m_api() client_port = self.generate_client_port() response = request.send_notification(self.general_observation.listener_ip, self.general_observation.listener_port, \ self.general_observation.token_id, payload, content_type="application/json", client_port=client_port) def update_resource(self, res_payload, endpoint_location=None, endpoint_name=None): total_res_dict = {} total_object_info = {} payload = res_payload endpoint_object = self.registration.handle_put_resource_updates( res_payload, endpoint_location=endpoint_location, endpoint_name=endpoint_name) for item, value in payload.iteritems(): resources_dict = endpoint_object.objects_dict[item][ "object"].resources_id_dict res_dict = {} for item1, value1 in resources_dict.iteritems(): res_dict.update({item1: value1["object"].res_value}) total_res_dict.update({item: {"resources": res_dict}}) total_object_info = {endpoint_object.endpoint_name: total_res_dict} return total_object_info def process(self, rx_record, remote, uri_query): """ Processes various requests like CON (POST, PUT, GET) or NON. POST requests : Generally used for Registration and Execution PUT requests : Generally used for Updating the resources GET requests : Generally used for Discovery, Observation, Cancel Observation """ msg = rx_record.message self.uri_query = uri_query if msg.transaction_type == connection.Message.CON: if constants.POST == msg.code: method = None try: method = uri_query[0].value.split("=")[1] except: pass if method == "create": path = msg.findOption(URI_PATH_VALUE) content_type_number = msg.findOption(options.ContentType) if content_type_number is None: content_type = "text/plain" else: content_type = constants.media_types[ content_type_number.value] self.create_object_instance.create_instance( path, remote, content_type, loads(msg.payload)) resources = loads(msg.payload) msg = connection.Message(connection.Message.ACK, code=constants.CREATED, payload="Resource Created") self.dm_server.sendto(msg._pack(rx_record.transaction_id), remote) client_port = self.generate_client_port() self.create_object_instance.forward_request( path, remote, resources, content_type, client_port) elif method == "execute": path = msg.findOption(URI_PATH_VALUE) content_type_number = msg.findOption(options.ContentType) if content_type_number is None: content_type = "text/plain" else: content_type = constants.media_types[ content_type_number.value] self.execution.execute_resource(path, remote, msg.payload) execute_payload = msg.payload msg = connection.Message(connection.Message.ACK, code=constants.CHANGED, payload="Resource Executed") self.dm_server.sendto(msg._pack(rx_record.transaction_id), remote) client_port = self.generate_client_port() self.execution.forward_request(path, remote, execute_payload, client_port) elif method == "notify": self.logger.info("Notification Received") client_port = self.generate_client_port() for item1, item2 in loads(msg.payload).iteritems(): if item1 == "observer_ip": observer_ip = item2 elif item1 == "observer_port": observer_port = item2 elif item1 != "observer_ip" and item1 != "observer_port": endpoint_name = item1 for item3, item4 in item2.iteritems(): for item5, item6 in item4[ "resources"].iteritems(): pass res = { item3: { "resources": { item5.split("_")[0]: { "res_value": item6, "res_inst_id": item5.split("_")[1] } } } } payload = {} payload = self.update_resource(res, endpoint_name=endpoint_name) payload["observer_ip"] = observer_ip payload["observer_port"] = observer_port token_id = msg.token observe_value = msg.findOption(options.Observe).value self.logger.info("Forwarding Notification") content_type = "application/json" request = lwm2m_api() response = request.send_notification(self.general_observation.listener_ip, self.general_observation.listener_port, token_id, payload, \ content_type=content_type, time_elapse=observe_value, client_port=client_port) msg = connection.Message(connection.Message.ACK, code=constants.CREATED, payload="Notification Received") self.dm_server.sendto(msg._pack(rx_record.transaction_id), remote) else: """ Handles the Client Registration Request """ self.logger.info( "Registering Client Endpoint in the LWM2M DM Server") endpoint = self.registration.process_registration( msg, uri_query) response = self.registration.register_client(endpoint) registered_client_location = response if registered_client_location is not None: self.logger.info( "Client Endpoint Registration Successful for Endpoint : %s", endpoint.endpoint_name) self.logger.info("The registered location is %s", registered_client_location) payload = self.set_general_observation_params() else: self.logger.info("Client Endpoint Registration Failed") msg = connection.Message( connection.Message.ACK, code=constants.CREATED, location=registered_client_location) self.dm_server.sendto(msg._pack(rx_record.transaction_id), remote) #Send the General Observation to the Registered Client #self.send_general_observation(registered_client_location) elif constants.PUT == msg.code: """ It consists of Normal Update, Write Operation, Write Attribute Operation. Write Operation is used to update the resource(s) as per the request. Write Attributes operation is used to update the attributes of the object, object instance or resource. """ self.handle_lwm2m_put(msg, uri_query, remote, rx_record) elif constants.GET == msg.code: """ Handles Requests like Discovery, Observation """ try: observe_value = msg.findOption(options.Observe).value except: observe_value = "" if observe_value == OBSERVE_OPTION_VALUE_OBSERVATION: """ Sets the Observation. Two types of observations. General Observation and Specific Observation. General Observation is used for anything that is not observed and updates are sent as general notifications using a general token. Specific observation is implicitly defined by the observer(as request) and handled as specific notification with a specific token """ path = msg.findOption(URI_PATH_VALUE) if len(path) == 1: self.set_m2m_server_adapter_params(rx_record, remote) else: self.logger.info( "Specific Observation Request Received") content_type_number = msg.findOption( options.ContentType) if content_type_number is None: content_type = "text/plain" else: content_type = constants.media_types[ content_type_number.value] token_id = msg.token app_ip = loads(msg.payload)["app_ip"] app_port = loads(msg.payload)["app_port"] client_port = self.generate_client_port() response = self.observation.forward_request(path, remote, observe_value, \ self.lwm2m_dm_server_ip, self.lwm2m_dm_server_port, \ app_ip, app_port, token_id, client_port) msg = connection.Message(connection.Message.ACK, code=constants.CONTENT, \ payload="test") #todo: payload to be replaced self.dm_server.sendto( msg._pack(rx_record.transaction_id, token_id), remote) elif observe_value == OBSERVE_OPTION_VALUE_CANCEL_OBSERVATION: """ Removes the observation from the List """ self.logger.info("Cancel Observation Request Received") path = msg.findOption(URI_PATH_VALUE) token_id = msg.token app_ip = loads(msg.payload)["app_ip"] app_port = loads(msg.payload)["app_port"] client_port = self.generate_client_port() response = self.observation.forward_request(path, remote, observe_value, \ self.lwm2m_dm_server_ip, self.lwm2m_dm_server_port, \ app_ip, app_port, token_id, client_port) def _handle_response(response): msg = connection.Message(connection.Message.ACK, code=constants.CONTENT, payload=response) self.dm_server.sendto( msg._pack(rx_record.transaction_id), remote) response.then(_handle_response) else: method = None try: method = uri_query[0].value.split("=")[1] except: pass if method == "read": path = msg.findOption(URI_PATH_VALUE) self.read.read_resource(path, remote) msg = connection.Message(connection.Message.ACK, code=constants.CONTENT, \ payload="info read", content_type="text/plain") self.dm_server.sendto( msg._pack(rx_record.transaction_id), remote) elif method == "discover": if msg.payload == "/.well-known/core": payload = dumps(self.discover.get_all_resources()) else: path = msg.findOption(URI_PATH_VALUE) client_port = self.generate_client_port() payload = self.discover.forward_request( path, remote, client_port) def _handle_response(payload): msg = connection.Message(connection.Message.ACK, code=constants.CONTENT, \ payload=payload, content_type="application/json") self.dm_server.sendto( msg._pack(rx_record.transaction_id), remote) if payload is Promise: payload.then(_handle_response) else: _handle_response(payload) elif msg.transaction_type == connection.Message.NON: print "reached msg non" payload = msg.payload print payload def set_general_observation_params(self, ): return { "listener_ip": self.lwm2m_dm_server_ip, "listener_port": self.lwm2m_dm_server_port } def send_general_observation(self, registered_client_location): if registered_client_location is not None: payload = dumps(self.set_general_observation_params()) endpoint_object = self.lwm2m_resources.return_endpoint_object( endpoint_location=registered_client_location) client_listener_ip = endpoint_object.listener_ip client_listener_port = endpoint_object.listener_port request = lwm2m_api() response = request.observe_resource(client_listener_ip, client_listener_port, \ payload=payload, client_port=self.generate_client_port()) def set_m2m_server_adapter_params(self, rx_record, remote): msg = rx_record.message #content_type is application/json listener_ip = loads(msg.payload)["listener_ip"] listener_port = loads(msg.payload)["listener_port"] token_id = msg.token self.general_observation = GeneralObservationInformation( listener_ip, listener_port, token_id) response = "Observation Started on the LWM2M Server" msg = connection.Message(connection.Message.ACK, code=constants.CONTENT, payload=response) self.dm_server.sendto(msg._pack(rx_record.transaction_id, token_id), remote) def generate_client_port(self, ): if self.sem_counter >= 1000: self.sem_counter = 0 self.sem.acquire() self.sem_counter += 1 sem_counter = self.sem_counter self.sem.release() client_port = self.local_client_port_ + sem_counter return client_port
class BaseDataHandler(object): _params = { 'POLLING_INTERVAL': 3600, 'PATCHABLE_CONFIG_KEYS': ['stream_id', 'constraints'] } _polling = False _polling_glet = None _dh_config = {} _rr_cli = None def __init__(self, rr_cli, stream_registrar, dh_config): self._dh_config = dh_config self._stream_registrar = stream_registrar self._rr_cli = rr_cli def set_event_callback(self, evt_callback): self._event_callback = evt_callback def _dh_event(self, type, value): event = {'type': type, 'value': value, 'time': time.time()} self._event_callback(event) def _poll(self): """ Internal polling method, run inside a greenlet, that triggers execute_acquire_data without configuration mods The polling interval (in seconds) is retrieved from the POLLING_INTERVAL parameter """ self._polling = True interval = get_safe(self._params, 'POLLING_INTERVAL', 3600) log.debug('Polling interval: {0}'.format(interval)) while self._polling: self.execute_acquire_data() time.sleep(interval) def cmd_dvr(self, cmd, *args, **kwargs): """ Command a DataHandler by request-reply messaging. Package command message and send on blocking command socket. Block on same socket to receive the reply. Return the driver reply. @param cmd The DataHandler command identifier. @param args Positional arguments of the command. @param kwargs Keyword arguments of the command. @retval Command result. """ # Package command dictionary. #need to account for observatory_execute_resource commands #connect -> Not used #get_current_state -> Not used #discover -> Not used #disconnect -> Not used log.debug('cmd_dvr received command \'{0}\' with: args={1} kwargs={2}'. format(cmd, args, kwargs)) reply = None if cmd == 'initialize': # Delegate to BaseDataHandler.initialize() reply = self.initialize(*args, **kwargs) elif cmd == 'get': # Delegate to BaseDataHandler.get() reply = self.get(*args, **kwargs) elif cmd == 'set': # Delegate to BaseDataHandler.set() reply = self.set(*args, **kwargs) elif cmd == 'get_resource_params': # Delegate to BaseDataHandler.get_resource_params() reply = self.get_resource_params(*args, **kwargs) elif cmd == 'get_resource_commands': # Delegate to BaseDataHandler.get_resource_commands() reply = self.get_resource_commands(*args, **kwargs) elif cmd == 'execute_acquire_data': # Delegate to BaseDataHandler.execute_acquire_data() reply = self.execute_acquire_data(*args, **kwargs) elif cmd == 'execute_start_autosample': # Delegate to BaseDataHandler.execute_start_autosample() reply = self.execute_start_autosample(*args, **kwargs) elif cmd == 'execute_stop_autosample': # Delegate to BaseDataHandler.execute_stop_autosample() reply = self.execute_stop_autosample(*args, **kwargs) elif cmd in [ 'configure', 'connect', 'disconnect', 'get_current_state', 'discover', 'execute_acquire_sample' ]: # Disregard log.info('Command \'{0}\' not used by DataHandler'.format(cmd)) pass else: desc = 'Command \'{0}\' unknown by DataHandler'.format(cmd) log.info(desc) raise InstrumentCommandException(desc) return reply def initialize(self, *args, **kwargs): #TODO: Add documentation #TODO: This should put the DataHandler back into an 'unconfigured' state """ Called from: InstrumentAgent._handler_idle_reset InstrumentAgent._handler_idle_go_inactive InstrumentAgent._handler_stopped_reset InstrumentAgent._handler_stopped_go_inactive InstrumentAgent._handler_observatory_reset InstrumentAgent._handler_observatory_go_inactive InstrumentAgent._handler_uninitialized_initialize |--> ExternalDataAgent._start_driver """ log.debug('Initializing DataHandler...') self._glet_queue = [] self._semaphore = Semaphore() return None def configure(self, *args, **kwargs): #TODO: Add documentation #TODO: This should configure the DataHandler for the particular dataset """ Called from: InstrumentAgent._handler_inactive_go_active """ log.debug('Configuring DataHandler: args = {0}'.format(args)) try: self._dh_config = args[0] except IndexError: raise InstrumentParameterException( '\'acquire_data\' command requires a config dict as the first argument' ) return def execute_acquire_data(self, *args): """ Creates a copy of self._dh_config, creates a publisher, and spawns a greenlet to perform a data acquisition cycle If the args[0] is a dict, any entries keyed with one of the 'PATCHABLE_CONFIG_KEYS' are used to patch the config Greenlet binds to BaseDataHandler._acquire_data and passes the publisher and config Disallows multiple "new data" (unconstrained) requests using BaseDataHandler._semaphore lock Called from: InstrumentAgent._handler_observatory_execute_resource |--> ExternalDataAgent._handler_streaming_execute_resource @parameter args First argument can be a config dictionary """ log.debug('Executing acquire_data: args = {0}'.format(args)) # Make a copy of the config to ensure no cross-pollution config = self._dh_config.copy() # Patch the config if mods are passed in try: config_mods = args[0] if not isinstance(config_mods, dict): raise IndexError() log.debug('Configuration modifications provided: {0}'.format( config_mods)) for k in self._params['PATCHABLE_CONFIG_KEYS']: if get_safe(config_mods, k): config[k] = config_mods[k] except IndexError: log.info('No configuration modifications were provided') # Verify that there is a stream_id member in the config stream_id = get_safe(config, 'stream_id') if not stream_id: raise ConfigurationError( 'Configuration does not contain required \'stream_id\' member') isNew = get_safe(config, 'constraints') is None if isNew and not self._semaphore.acquire(blocking=False): log.warn('Already acquiring new data - action not duplicated') return if isNew: # Get any NewDataCheck attachments and add them to the config ext_ds_id = get_safe(config, 'external_dataset_res_id') if ext_ds_id: try: attachment_objs, _ = self._rr_cli.find_objects( ext_ds_id, PRED.hasAttachment, RT.Attachment, False) for attachment_obj in attachment_objs: kwds = set(attachment_obj.keywords) if 'NewDataCheck' in kwds: log.debug( 'Found NewDataCheck attachment: {0}'.format( attachment_obj)) config['new_data_check'] = attachment_obj.content break else: log.debug( 'Found attachment: {0}'.format(attachment_obj)) except NotFound: raise InstrumentException( 'ExternalDatasetResource \'{0}\' not found'.format( ext_ds_id)) if not get_safe(config, 'new_data_check'): config['new_data_check'] = None # Create a publisher to pass into the greenlet publisher = self._stream_registrar.create_publisher( stream_id=stream_id) # Spawn a greenlet to do the data acquisition and publishing g = spawn(self._acquire_data, config, publisher, self._unlock_new_data_callback) log.debug('** Spawned {0}'.format(g)) self._glet_queue.append(g) def execute_start_autosample(self, *args, **kwargs): #TODO: Add documentation #TODO: Fix raises statements """ Put the DataHandler into streaming mode and start polling for new data Called from: InstrumentAgent._handler_observatory_go_streaming @raises InstrumentTimeoutException: @raises InstrumentProtocolException: @raises NotImplementedException: @raises InstrumentParameterException: """ log.debug( 'Entered execute_start_autosample with args={0} & kwargs={1}'. format(args, kwargs)) if not self._polling and self._polling_glet is None: self._polling_glet = spawn(self._poll) return None def execute_stop_autosample(self, *args, **kwargs): #TODO: Add documentation #TODO: Fix raises statements """ Stop polling for new data and put the DataHandler into observatory mode Called from: InstrumentAgent._handler_streaming_go_observatory @raises InstrumentTimeoutException: @raises InstrumentProtocolException: @raises NotImplementedException: @raises InstrumentParameterException: """ log.debug('Entered execute_stop_autosample with args={0} & kwargs={1}'. format(args, kwargs)) if self._polling and not self._polling_glet is None: self._polling_glet.kill() self._polling = False self._polling_glet = None return None def get(self, *args, **kwargs): #TODO: Add documentation #TODO: Fix raises statements """ Called from: InstrumentAgent._handler_get_params @raises InstrumentTimeoutException: @raises InstrumentProtocolException: @raises NotImplementedException: @raises InstrumentParameterException: """ try: pnames = args[0] except IndexError: log.warn("No argument provided to get, return all parameters") pnames = [DataHandlerParameter.ALL] result = None if DataHandlerParameter.ALL in pnames: result = self._params else: if not isinstance(pnames, (list, tuple)): raise InstrumentParameterException( 'Get argument not a list or tuple: {0}'.format(pnames)) result = {} for pn in pnames: try: log.debug('Get parameter with key: {0}'.format(pn)) result[pn] = self._params[pn] except KeyError: log.debug('\'{0}\' not found in self._params'.format(pn)) raise InstrumentParameterException( '{0} is not a valid parameter for this DataHandler.'. format(pn)) return result def set(self, *args, **kwargs): #TODO: Add documentation #TODO: Fix raises statements """ Called from: InstrumentAgent._handler_observatory_set_params @raises InstrumentTimeoutException: @raises InstrumentProtocolException: @raises NotImplementedException: @raises InstrumentParameterException: """ # Retrieve required parameter. # Raise if no parameter provided, or not a dict. try: params = args[0] except IndexError: raise InstrumentParameterException( 'Set command requires a parameter dict.') to_raise = [] if not isinstance(params, dict): raise InstrumentParameterException('Set parameters not a dict.') else: for (key, val) in params.iteritems(): if key in self._params: log.debug('Set parameter \'{0}\' = {1}'.format(key, val)) self._params[key] = val else: log.debug( 'Parameter \'{0}\' not in self._params and cannot be set' .format(key)) to_raise.append(key) if len(to_raise) > 0: log.debug( 'Raise InstrumentParameterException for un-set parameters: {0}' .format(to_raise)) raise InstrumentParameterException( 'Invalid parameter(s) could not be set: {0}'.format(to_raise)) def get_resource_params(self, *args, **kwargs): """ Return list of resource parameters. Implemented in specific DataHandlers Called from: InstrumentAgent._handler_get_resource_params """ return self._params.keys() def get_resource_commands(self, *args, **kwargs): """ Return list of DataHandler execute commands available. Called from: InstrumentAgent._handler_get_resource_commands """ cmds = [ cmd.replace('execute_', '') for cmd in dir(self) if cmd.startswith('execute_') ] return cmds def _unlock_new_data_callback(self, caller): log.debug('** Release {0}'.format(caller)) self._semaphore.release() @classmethod def _acquire_data(cls, config, publisher, unlock_new_data_callback): """ Ensures required keys (such as stream_id) are available from config, configures the publisher and then calls: BaseDataHandler._new_data_constraints (only if config does not contain 'constraints') BaseDataHandler._publish_data passing BaseDataHandler._get_data as a parameter @param config Dict containing configuration parameters, may include constraints, formatters, etc @param unlock_new_data_callback BaseDataHandler callback function to allow conditional unlocking of the BaseDataHandler._semaphore """ log.debug('start _acquire_data: config={0}'.format(config)) cls._init_acquisition_cycle(config) constraints = get_safe(config, 'constraints') if not constraints: gevent.getcurrent().link(unlock_new_data_callback) constraints = cls._new_data_constraints(config) if constraints is None: raise InstrumentParameterException( "Data constraints returned from _new_data_constraints cannot be None" ) config['constraints'] = constraints cls._publish_data(publisher, cls._get_data(config)) # Publish a 'TestFinished' event if get_safe(config, 'TESTING'): log.debug('Publish TestingFinished event') pub = EventPublisher('DeviceCommonLifecycleEvent') pub.publish_event(origin='BaseDataHandler._acquire_data', description='TestingFinished') @classmethod def _init_acquisition_cycle(cls, config): """ Allows the concrete implementation to initialize/prepare objects the data handler will use repeatedly (such as a dataset object) in cls._new_data_constraints and/or cls._get_data Objects should be added to the config so they are available later in the workflow """ raise NotImplementedException( '{0}.{1} must implement \'_init_acquisition_cycle\''.format( cls.__module__, cls.__name__)) @classmethod def _new_data_constraints(cls, config): #TODO: Document what "constraints" looks like (yml)!! """ Determines the appropriate constraints for acquiring any "new data" from the external dataset Returned value cannot be None and is assigned to config['constraints'] The format of the constraints are documented: @param config dict of configuration parameters - may be used to generate the returned 'constraints' dict @retval dict that contains the constraints for retrieval of new data from the external dataset """ raise NotImplementedException( '{0}.{1} must implement \'_new_data_constraints\''.format( cls.__module__, cls.__name__)) @classmethod def _get_data(cls, config): """ Iterable function that acquires data from a source iteratively based on constraints provided by config Passed into BaseDataHandler._publish_data and iterated to publish samples. @param config dict containing configuration parameters, may include constraints, formatters, etc @return an iterable that returns well-formed Granule objects on each iteration """ raise NotImplementedException( '{0}.{1} must implement \'_get_data\''.format( cls.__module__, cls.__name__)) @classmethod def _publish_data(cls, publisher, data_generator): """ Iterates over the data_generator and publishes granules to the stream indicated in stream_id """ if data_generator is None or not hasattr(data_generator, '__iter__'): raise InstrumentDataException( 'Invalid object returned from _get_data: returned object cannot be None and must have \'__iter__\' attribute' ) for count, gran in enumerate(data_generator): if isinstance(gran, Granule): publisher.publish(gran) else: log.warn('Could not publish object returned by _get_data: {0}'. format(gran)) #TODO: Persist the 'state' of this operation so that it can be re-established in case of failure #TODO: When finished publishing, update (either directly, or via an event callback to the agent) the UpdateDescription @classmethod def _calc_iter_cnt(cls, total_recs, max_rec): """ Given the total number of records and the maximum records allowed in a granule, calculates the number of iterations required to traverse the entire array in chunks of size max_rec @param total_recs The total number of records @param max_rec The maximum number of records allowed in a granule """ cnt = total_recs / max_rec if total_recs % max_rec > 0: cnt += 1 return cnt
class BooleanBlindTechnique: def __init__(self, query, requester): self.query = query self.requester = requester self.rungl = None def _reset(self): ''' reset all the variables used for keeping track of internal state ''' #an list of Character()s self.results = [] #an list of strings self.str_results = [] #character generators take care of building the Character objects. we need one per row self.char_gens = [] #a queue for communications between Character()s and request_makers self.q = Queue() #"threads" that run the Character()s self.character_pool = Pool(self.concurrency) #"threads" that make requests self.request_makers = [gevent.spawn(self._request_maker) for i in range(self.concurrency)] #fire this event when shutting down self.shutting_down = Event() #do we need to add more rows? self.need_more_rows = True #use this as a lock to know when not to mess with self.results self.results_lock = Semaphore(1) #request_count is the number of requests made on the current run self.request_count = 0 #failure_count is the number of requests made on the current run self.failure_count = 0 def _request_maker(self): ''' this runs in a gevent "thread". It is a worker ''' #keep going until we shut down the technique while not self.shutting_down.is_set(): #pull the info needed to make a request from the queue row_index,char_index,char_val,comparator,char_asyncresult = self.q.get() #build out our query object query = copy(self.query) query.set_option('row_index',str(row_index)) query.set_option('char_index',str(char_index)) query.set_option('char_val',str(ord(char_val))) query.set_option('comparator',comparator) query_string = query.render() self.request_count += 1 count = 0 response = None while response == None: try: response = self.requester.make_request(query_string) except SendRequestFailed: self.failure_count += 1 response = None gevent.sleep(.01 * 2 ** count) if count == 10: raise SendRequestFailed('cant request') count += 1 char_asyncresult.set(response) def _character_generator(self,row_index): ''' creates a Character object for us. this generator is useful just because it keeps track of the char_index ''' char_index = 1 row_die_event = AsyncResult() while not self.shutting_down.is_set(): c = BlindCharacter(\ row_index = row_index,\ char_index = char_index,\ queue = self.q,\ row_die = row_die_event) char_index += 1 #fire off the Character within our Pool. self.character_pool.spawn(c.run) yield c def _adjust_row_lengths(self): ''' if a row is full of "success", but we havent reached the end yet (the last elt isnt "error") then we need to increase the row_len. ''' while not self.shutting_down.is_set(): self.results_lock.acquire() if self.row_len is not None: unused_threads = self.concurrency - reduce(lambda x,row: x + row.count('working'),self.results,0) rows_working = len(filter(lambda row: 'working' in row,self.results)) if rows_working == 0: add_to_rows = self.row_len else: add_to_rows = unused_threads//rows_working add_to_rows = [add_to_rows,1][add_to_rows==0] else: add_to_rows = 1 for row_index in range(len(self.results)): #if the row isn't finished or hasn't been started yet, we add Character()s to the row if 'error' not in self.results[row_index]: self.results[row_index] += [self.char_gens[row_index].next() for i in range(add_to_rows)] self.results_lock.release() gevent.sleep(.3) def _add_rows(self): ''' look at how many gevent "threads" are being used and add more rows to correct this ''' if self.row_len is not None: rows_to_work_on = self.concurrency // self.row_len else: rows_to_work_on = self.concurrency rows_to_work_on = [rows_to_work_on,1][rows_to_work_on == 0] row_index = 0 # keep adding new rows until we dont need any more while self.need_more_rows: working_rows = len(filter(lambda row: 'working' in row,self.results)) for row in range(rows_to_work_on - working_rows): self.char_gens.append(self._character_generator(row_index)) self.results.append([]) row_index += 1 gevent.sleep(.3) self.need_more_rows = not(len(self.results) and filter(lambda row: len(row) and row[0] == 'error',self.results)) # delete any extra rows. while not self.shutting_down.is_set(): self.results_lock.acquire() # delete any rows that shouldn't have been added in the first place errored = filter(lambda ri: len(self.results[ri]) and self.results[ri][0] == 'error',range(len(self.results))) if errored: end = min(errored) for ri in xrange(len(self.results)-1,end-1,-1): del(self.results[ri]) self.results_lock.release() #if there aren't going to be any more rows in need of deletion we can stop this nonsense if self.results and self.results[-1][0] == 'success': break gevent.sleep(.3) def _keep_going(self): ''' Look at the results gathered so far and determine if we should keep going. we want to keep going until we have an empty row ''' # chill out until we don't need any more rows while self.need_more_rows: gevent.sleep(1) # chill out untill all the rows have finished working while filter(lambda row:'error' not in row or 'working' in row[:row.index('error')],self.results): gevent.sleep(.5) # call it quits self.shutting_down.set() def _run(self): self.kg_gl = gevent.spawn(self._keep_going) self.ar_gl = gevent.spawn(self._add_rows) self.arl_gl = gevent.spawn(self._adjust_row_lengths) self.kg_gl.join() self.ar_gl.join() self.arl_gl.join() self.character_pool.join() gevent.killall(self.request_makers) gevent.joinall(self.request_makers) def run(self,row_len=None,concurrency=20): ''' run the exploit. returns the data retreived. :concurrency how many gevent "threads" to use. This is useful for throttling the attack. :row_len An estimated starting point for the length of rows. This will get adjusted as the attack goes on. ''' self.run_start_time = time() self.row_len = row_len self.concurrency = concurrency #start fresh self._reset() self.rungl = gevent.spawn(self._run) return self.rungl def get_results(self,color=False): if not color: return filter(lambda row: row != '',[''.join([str(x) for x in row]) for row in self.results]) rval = [] running_status = "unknown" for row in self.results: if len(row): running_status = "unknown" row_string = "" for c in row: cstatus = c.get_status() if cstatus != running_status: row_string += COLORS[cstatus] running_status = cstatus row_string += str(c) rval.append(row_string + COLORS['endc']) return rval #return filter(lambda row: row != '',[''.join([COLORS[x.get_status()] + str(x) + COLORS['endc'] for x in row]) for row in self.results]) def get_status(self): status = "" status += "requests: %d\t" % self.request_count status += "failures: %d\t" % self.failure_count status += "rows: %d\t" % reduce(lambda x,row: ('success' in row)+x,self.results,0) status += "working threads: %d\t" % reduce(lambda x,row: x + row.count('working'),self.results,0) chars = reduce(lambda x,row: row.count('success') + x,self.results,0) status += "chars: %d\t" % chars if self.run_start_time: run_time = time() - self.run_start_time status += "time: %f\t" % run_time status += "char/sec: %f\t" % (chars/run_time) if chars: rc = float(self.request_count) / chars else: rc = 0.0 status += "req/char: %f\t" % rc return status
class AsyncWorker(object): def __init__(self, worker_id): # used in process of selecting jobs self.own_async_id = worker_id # database setup dbb = settings.ASYNC_DB_BACKEND if dbb=="sqlite": from db.sqlite import SQLiteDatabase self.DB = SQLiteDatabase( worker_id ) else: raise Exception("Unknown database backend defined in configuration: %r" % dbb) # serializer / deserializer self.serializer = Serializer() # caller self.PROXY = RawProxy() self._processing = True self.SEMA = Semaphore() def get_database_id(self): return self.DB.CONF['databaseid'] def close(self): self._processing = False self.DB.close() # task processing loop def task_eater(self): rectime = time.time() while self._processing: taskproc = self.process_next_job() if taskproc: gevent.sleep() else: gevent.sleep(2) if rectime<time.time(): g = gevent.Greenlet(self.check_lost_tasks) g.start() rectime = time.time() + settings.ASYNC_RECOVERY_TIME gevent.sleep() def start_eat(self): g = gevent.Greenlet(self.task_eater) g.start() # task scheduling def task_add_new(self, task, context, args, kwargs, ign_result=False): """ Register task in database. task - task name with worker context - context data args, kwargs - function arguments ign_result - ignore result flag (True / False) """ args = (args, kwargs) taskid = self.DB.task_add( taskname = task, time = time.time(), args = self.serializer.data_2_bin(args), context = self.serializer.data_2_bin(context), ign_result = ign_result, ) #self._check_next() return taskid def process_task(self, taskid): """ Process job with given ID """ LOG.debug("Processing task %i" % taskid) # get task from database data = self.DB.task_start_process(taskid) if data is None: return # unpack data data['args'],data['kwargs'] = self.serializer.bin_2_data(data['args']) data['context'] = self.serializer.bin_2_data(data['context']) # send task to realize try: result = self.PROXY.sync_call( data['task'], data['context'], data['args'], data['kwargs'], ) except exceptions.SerializationError: # Exception raised when serialization or deserialization fails. # If this exception occurs here, we can't try to run this task again, because # data stored in async database are probably screwed up (for example async daemon # died or was stopped, but current configuration is different and uses other # serialization methods). We mark this task as permanently broken (and newer will be repeated). self.DB.task_fail_permanently(taskid) return except exceptions.ServiceNotFound: # Worker not found occurs when destination service is currently not # available. Task will be repeated in future. self.DB.task_error_and_delay(taskid, settings.ASYNC_ERROR_TASK_DELAY) return except exceptions.ServiceBusException: # Any other internal exception # will bump error counter self.DB.task_error_and_delay(taskid, settings.ASYNC_ERROR_TASK_DELAY) return if result['message'] == messages.ERROR: # task prodoced error # not kasaya exception, but task's own error # it's not our fault ;-) # get task context or create it if not exist ctx = data['context'] if ctx is None: ctx = {} # increace error count errcnt = ctx.get("err_count", 0) + 1 ctx['err_count'] = errcnt # if error counter is limited, is that limit reached? maxerr = ctx.get("err_max", None) if not maxerr is None: no_retry = errcnt>=maxerr else: no_retry = False data['context'] = ctx if no_retry: self.DB.task_fail_permanently( taskid, settings.ASYNC_ERROR_TASK_DELAY, self.serializer.data_2_bin(result) ) self.DB.task_store_context( taskid, self.serializer.data_2_bin(ctx) ) else: self.DB.task_error_and_delay(taskid, settings.ASYNC_ERROR_TASK_DELAY, self.serializer.data_2_bin(result) ) self.DB.task_store_context( taskid, self.serializer.data_2_bin(ctx) ) return # task is processed succesfully self.DB.task_finished_ok(taskid, self.serializer.data_2_bin( result ) ) def process_next_job(self): """ Check if is any job waiting and start processing it. """ self.SEMA.acquire() try: taskid = self.DB.task_choose_for_process() finally: self.SEMA.release() if taskid is None: # nothing is waiting return False else: self.process_task( taskid ) return True def check_lost_tasks(self): """ Find tasks assigned to unexisting async workers and reassign them to self. once - if true, then not register task to do it again in future also: Check database for dead tasks: task waiting long with status=1 (selected to process, but unprocessed) also: Find tasks with status=2 - processing, but without asyncid (after asyncid died). """ # get all tasks belonging to dead async workers lost_asyncd = 0 for asyncid in self.DB.async_list(): if control.worker.exists( asyncid ): # worker exists continue # found lost async daemon tasks, # reassign them to self rc = self.DB.unlock_lost_tasks(asyncid) lost_asyncd =+ 1 # process all tasks with status 1 -> selected for process but unprocessed long time self.DB.recover_unprocessed_tasks( settings.ASYNC_DEAD_TASK_TIME_LIMIT ) # process all tasks with status 2 -> processing started, but async daemon died before receiving result self.DB.recover_unfinished_tasks()
class MemoryDB(BaseDB): def __init__(self): self.__db = SQ.connect(":memory:") self.cur = self.__db.cursor() self.cur.execute( "CREATE TABLE hosts (id TEXT, addr TEXT, hostname TEXT)") self.cur.execute( "CREATE TABLE workers (id TEXT, host_id TEXT, service TEXT, addr TEXT, pid INT, online BOOLEAN)" ) self.cur.execute("CREATE TABLE services (host_id TEXT, name TEXT)") self.SEMA = Semaphore() def close(self): self.__db.close() # hosts # ------------------------- def host_add(self, ID, address): #, hostname): self.SEMA.acquire() try: self.cur.execute("INSERT INTO hosts ('id','addr') VALUES (?,?)", (ID, address)) self.__db.commit() finally: self.SEMA.release() def host_get(self, ID): self.SEMA.acquire() try: self.cur.execute("SELECT ID,addr,hostname FROM hosts WHERE id=?", (ID, )) res = self.cur.fetchone() finally: self.SEMA.release() if res is None: return None return {'id': res[0], 'addr': res[1], 'hostname': res[2]} def host_del(self, ID): self.SEMA.acquire() try: self.cur.execute("DELETE FROM hosts WHERE id=?", (ID, )) self.__db.commit() finally: self.SEMA.release() def host_list(self): self.SEMA.acquire() try: self.cur.execute("SELECT id,addr,hostname FROM hosts") res = self.cur.fetchall() finally: self.SEMA.release() for u, a, h in res: yield {'id': u, 'addr': a, 'hostname': h} def service_add(self, host_id, name): """ Dodanie serwisu do hosta """ self.SEMA.acquire() try: self.cur.execute( "INSERT INTO services ('host_id','name') VALUES (?,?)", (host_id, name)) self.__db.commit() finally: self.SEMA.release() def service_del(self, host_id, name): """ Usunięcie serwisu z hosta """ self.SEMA.acquire() try: self.cur.execute("DELETE FROM services WHERE host_id=? AND name=?", (host_id, name)) self.__db.commit() finally: self.SEMA.release() def service_list(self, host_id): """ List of services on host """ self.SEMA.acquire() try: self.cur.execute("SELECT name FROM services WHERE host_id=?", (host_id, )) res = self.cur.fetchall() finally: self.SEMA.release() for s in res: yield {'service': s[0]} def worker_add(self, host_id, worker_id, name, address, pid, online): self.SEMA.acquire() try: self.cur.execute( "INSERT INTO workers ('id', 'host_id', 'addr', 'service','pid', 'online') VALUES (?,?,?,?,?,?)", (worker_id, host_id, address, name, pid, online)) self.__db.commit() finally: self.SEMA.release() def worker_set_state(self, worker_id, is_online): self.SEMA.acquire() try: self.cur.execute("UPDATE workers SET online=? WHERE id=?;", (is_online, worker_id)) self.__db.commit() finally: self.SEMA.release() def worker_get(self, worker_id): self.SEMA.acquire() try: self.cur.execute( "SELECT id,host_id,service,addr,pid, online FROM workers WHERE id=?", [ worker_id, ]) res = self.cur.fetchone() finally: self.SEMA.release() if res is None: return return { 'id': res[0], 'host_id': res[1], 'service': res[2], 'addr': res[3], 'pid': res[4], 'online': res[5] > 0 } def worker_del_by_addr(self, addr): """ Delete worker from database identified by address """ self.SEMA.acquire() try: self.cur.execute("DELETE FROM workers WHERE addr=?", (addr, )) self.__db.commit() finally: self.SEMA.release() def worker_del_by_id(self, id): """ Delete worker from database identified by worker id """ self.SEMA.acquire() try: self.cur.execute("DELETE FROM workers WHERE id=?", (id, )) self.__db.commit() finally: self.SEMA.release() def worker_list(self, host_id): ''' list all workers on given host ''' self.SEMA.acquire() try: self.cur.execute( "SELECT id,service,addr,pid, online FROM workers WHERE host_id=?", (host_id, )) res = self.cur.fetchall() finally: self.SEMA.release() if res is None: return for i, s, a, pid, o in res: yield { 'id': i, 'service': s, 'addr': a, 'pid': pid, 'online': o > 0 } def workers_for_service(self, service, only_online=True): """ List of workers for service """ self.SEMA.acquire() try: if only_online: self.cur.execute( "SELECT id,addr FROM workers WHERE service=? AND online=?", (service, 1)) else: self.cur.execute("SELECT id,addr FROM workers WHERE service=?", (service, )) res = self.cur.fetchall() finally: self.SEMA.release() for i, a in res: yield {'id': i, 'addr': a}
class VBucketAwareCouchbaseClient(object): #poll server every few seconds to see if the vbucket-map #has changes def __init__(self, url, bucket, password="", verbose=False): self.log = logger.logger("VBucketAwareMemcachedClient") self.bucket = bucket self.rest_username = bucket self.rest_password = password self._memcacheds = {} self._vBucketMap = {} self._vBucketMap_lock = Lock() self._vBucketMapFastForward = {} self._vBucketMapFastForward_lock = Lock() #TODO: use regular expressions to parse the url server = {} if not bucket: raise InvalidArgumentException("bucket can not be an empty string", parameters="bucket") if not url: raise InvalidArgumentException("url can not be an empty string", parameters="url") if url.find("http://") != -1 and url.rfind(":") != -1 and url.find("/pools/default") != -1: server["ip"] = url[url.find("http://") + len("http://"):url.rfind(":")] server["port"] = url[url.rfind(":") + 1:url.find("/pools/default")] server["username"] = self.rest_username server["password"] = self.rest_password self.servers = [server] self.servers_lock = Lock() self.rest = RestConnection(server) self.reconfig_vbucket_map() self.init_vbucket_connections() self.dispatcher = CommandDispatcher(self) self.dispatcher_thread = Process(name="dispatcher-thread", target=self._start_dispatcher) self.dispatcher_thread.daemon = True self.dispatcher_thread.start() self.streaming_thread = Process(name="streaming", target=self._start_streaming, args=()) self.streaming_thread.daemon = True self.streaming_thread.start() self.verbose = verbose def _start_dispatcher(self): self.dispatcher.dispatch() def _start_streaming(self): # this will dynamically update vBucketMap, vBucketMapFastForward, servers urlopener = urllib.FancyURLopener() urlopener.prompt_user_passwd = lambda host, realm: (self.rest_username, self.rest_password) current_servers = True while current_servers: self.servers_lock.acquire() current_servers = deepcopy(self.servers) self.servers_lock.release() for server in current_servers: response = urlopener.open("http://{0}:{1}/pools/default/bucketsStreaming/{2}".format(server["ip"], server["port"], self.bucket)) while response: try: line = response.readline() if not line: # try next server if we get an EOF response.close() break except: # try next server if we fail to read response.close() break try: data = json.loads(line) except: continue serverlist = data['vBucketServerMap']['serverList'] vbucketmapfastforward = {} index = 0 if 'vBucketMapForward' in data['vBucketServerMap']: for vbucket in data['vBucketServerMap']['vBucketMapForward']: vbucketmapfastforward[index] = serverlist[vbucket[0]] index += 1 self._vBucketMapFastForward_lock.acquire() self._vBucketMapFastForward = deepcopy(vbucketmapfastforward) self._vBucketMapFastForward_lock.release() vbucketmap = {} index = 0 for vbucket in data['vBucketServerMap']['vBucketMap']: vbucketmap[index] = serverlist[vbucket[0]] index += 1 # only update vBucketMap if we don't have a fastforward # on a not_mb_vbucket error, we already update the # vBucketMap from the fastforward map if not vbucketmapfastforward: self._vBucketMap_lock.acquire() self._vBucketMap = deepcopy(vbucketmap) self._vBucketMap_lock.release() new_servers = [] nodes = data["nodes"] for node in nodes: if node["clusterMembership"] == "active" and node["status"] == "healthy": hostport = node["hostname"] new_servers.append({"ip":hostport.split(":")[0], "port":int(hostport.split(":")[1]), "username":self.rest_username, "password":self.rest_password}) new_servers.sort() self.servers_lock.acquire() self.servers = deepcopy(new_servers) self.servers_lock.release() def init_vbucket_connections(self): # start up all vbucket connections self._vBucketMap_lock.acquire() vbucketcount = len(self._vBucketMap) self._vBucketMap_lock.release() for i in range(vbucketcount): self.start_vbucket_connection(i) def start_vbucket_connection(self,vbucket): self._vBucketMap_lock.acquire() server = deepcopy(self._vBucketMap[vbucket]) self._vBucketMap_lock.release() serverIp, serverPort = server.split(":") if not server in self._memcacheds: self._memcacheds[server] = MemcachedClientHelper.direct_client(self.rest, serverIp, serverPort, self.bucket) def start_vbucket_fastforward_connection(self,vbucket): self._vBucketMapFastForward_lock.acquire() if not vbucket in self._vBucketMapFastForward: self._vBucketMapFastForward_lock.release() return server = deepcopy(self._vBucketMapFastForward[vbucket]) self._vBucketMapFastForward_lock.release() serverIp, serverPort = server.split(":") if not server in self._memcacheds: self._memcacheds[server] = MemcachedClientHelper.direct_client(self.rest, serverIp, serverPort, self.bucket) def restart_vbucket_connection(self,vbucket): self._vBucketMap_lock.acquire() server = deepcopy(self._vBucketMap[vbucket]) self._vBucketMap_lock.release() serverIp, serverPort = server.split(":") if server in self._memcacheds: self._memcacheds[server].close() self._memcacheds[server] = MemcachedClientHelper.direct_client(self.rest, serverIp, serverPort, self.bucket) def reconfig_vbucket_map(self, vbucket=-1): vb_ready = RestHelper(self.rest).vbucket_map_ready(self.bucket, 60) if not vb_ready: raise Exception("vbucket map is not ready for bucket {0}".format(self.bucket)) vBuckets = self.rest.get_vbuckets(self.bucket) self.vbucket_count = len(vBuckets) bucket_info = self.rest.get_bucket(self.bucket) nodes = bucket_info.nodes self._vBucketMap_lock.acquire() for vBucket in vBuckets: if vBucket.id == vbucket or vbucket == -1: self._vBucketMap[vBucket.id] = vBucket.master self._vBucketMap_lock.release() def memcached(self, key, fastforward=False): self._vBucketMap_lock.acquire() self._vBucketMapFastForward_lock.acquire() vBucketId = (zlib.crc32(key) >> 16) & (len(self._vBucketMap) - 1) if fastforward and vBucketId in self._vBucketMapFastForward: # only try the fastforward if we have an entry # otherwise we just wait for the main map to update self.start_vbucket_fastforward_connection(vBucketId) self._vBucketMap[vBucketId] = self._vBucketMapFastForward[vBucketId] if vBucketId not in self._vBucketMap: msg = "vbucket map does not have an entry for vb : {0}" self._vBucketMapFastForward_lock.release() self._vBucketMap_lock.release() raise Exception(msg.format(vBucketId)) if self._vBucketMap[vBucketId] not in self._memcacheds: msg = "smart client does not have a mc connection for server : {0}" self._vBucketMapFastForward_lock.release() self._vBucketMap_lock.release() raise Exception(msg.format(self._vBucketMap[vBucketId])) r = self._memcacheds[self._vBucketMap[vBucketId]] self._vBucketMapFastForward_lock.release() self._vBucketMap_lock.release() return r def vbucketid(self, key): self._vBucketMap_lock.acquire() r = (zlib.crc32(key) >> 16) & (len(self._vBucketMap) - 1) self._vBucketMap_lock.release() return r def done(self): if self.dispatcher: self.dispatcher.shutdown() if self.verbose: self.log.info("dispatcher shutdown invoked") [self._memcacheds[ip].close() for ip in self._memcacheds] if self.verbose: self.log.info("closed all memcached open connections") self.dispatcher = None def _respond(self, item, event): timeout = 30 event.wait(timeout) if not event.is_set(): # if we timeout, then try to reconnect to the server # responsible for this vbucket self.restart_vbucket_connection(self.vbucketid(item['key'])) raise MemcachedTimeoutException(item, timeout) if "error" in item["response"]: raise item["response"]["error"] return item["response"]["return"] def get(self, key): event = Event() item = {"operation": "get", "key": key, "event": event, "response": {}} self.dispatcher.put(item) return self._respond(item, event) def gat(self, key, expiry): event = Event() item = {"operation": "gat", "key": key, "expiry": expiry, "event": event, "response": {}} self.dispatcher.put(item) return self._respond(item, event) def touch(self, key, expiry): event = Event() item = {"operation": "touch", "key": key, "expiry": expiry, "event": event, "response": {}} self.dispatcher.put(item) return self._respond(item, event) def cas(self, key, expiry, flags, old_value, value): event = Event() item = {"operation": "cas", "key": key, "expiry": expiry, "flags": flags, "old_value": old_value, "value": value , "event": event, "response": {}} self.dispatcher.put(item) return self._respond(item, event) def decr(self, key, amount=1, init=0, expiry=0): event = Event() item = {"operation": "decr", "key": key, "amount": amount, "init": init, "expiry": expiry, "event": event, "response": {}} self.dispatcher.put(item) return self._respond(item, event) def set(self, key, expiry, flags, value): event = Event() item = {"operation": "set", "key": key, "expiry": expiry, "flags": flags, "value": value, "event": event, "response": {}} self.dispatcher.put(item) return self._respond(item, event) def add(self, key, expiry, flags, value): event = Event() item = {"operation": "add", "key": key, "expiry": expiry, "flags": flags, "value": value, "event": event, "response": {}} self.dispatcher.put(item) return self._respond(item, event) def append(self, key, value, cas=0): event = Event() item = {"operation": "append", "key": key, "cas": cas, "value": value, "event": event, "response": {}} self.dispatcher.put(item) return self._respond(item, event) def delete(self, key, cas=0): event = Event() item = {"operation": "delete", "key": key, "cas": cas, "event": event, "response": {}} self.dispatcher.put(item) return self._respond(item, event) def prepend(self, key, value, cas=0): event = Event() item = {"operation": "prepend", "key": key, "cas": cas, "value": value, "event": event, "response": {}} self.dispatcher.put(item) return self._respond(item, event) def getl(self, key, expiry=15): event = Event() item = {"operation": "getl", "key": key, "expiry": expiry, "event": event, "response": {}} self.dispatcher.put(item) return self._respond(item, event) def replace(self, key, expiry, flags, value): event = Event() item = {"operation": "replace", "key": key, "expiry": expiry, "flags": flags, "value": value, "event": event, "response": {}} self.dispatcher.put(item) return self._respond(item, event) def incr(self, key, amount=1, init=0, expiry=0): event = Event() item = {"operation": "incr", "key": key, "amount": amount, "init": init, "expiry": expiry, "event": event, "response": {}} self.dispatcher.put(item) return self._respond(item, event)
class Client(object): px_per_tick = 10 def __init__(self, canvas, socket, address): self.canvas = canvas self.socket = socket self.address = address self.connect_ts = time.time() # This buffer discards all but the newest 1024 messages self.sendbuffer = deque([], 1024) # And this is used to limit clients to X messages per tick # We start at 0 (instead of x) to add a reconnect-penalty. self.limit = Semaphore(0) print 'CONNECT', address def send(self, line): self.sendbuffer.append(line.strip() + '\n') def disconnect(self): print 'DISCONNECT', self.address self.socket.close() del self.canvas.clients[self.address] def serve(self): sendall = self.socket.sendall readline = self.socket.makefile().readline try: while True: # Idea: Sand first, recieve later. If the client is to # slow to get the sendbuffer empty, he cannot send. while self.sendbuffer: sendall(self.sendbuffer.popleft()) line = readline() if not line: break arguments = line.split() command = arguments.pop(0) if command == 'PX': self.on_PX(arguments) elif command == 'SIZE': self.on_SIZE(arguments) finally: self.disconnect() def on_SIZE(self, args): self.send('SIZE %d %d' % self.canvas.get_size()) def on_PX(self, args): self.limit.acquire() x,y,color = args x,y = int(x), int(y) c = int(color, 16) if c <= 16777215: r = (c & 0xff0000) >> 16 g = (c & 0x00ff00) >> 8 b = c & 0x0000ff a = 0xff else: r = (c & 0xff000000) >> 24 g = (c & 0x00ff0000) >> 16 b = (c & 0x0000ff00) >> 8 a = c & 0x000000ff self.canvas.set_pixel(x, y, r, g, b, a) def tick(self): while self.limit.counter <= self.px_per_tick: self.limit.release()
class SchedSite(SchedCommon): """Mirrors a site""" rain_timer = None rain_counter = 0 _run_delay = None _sched = None _sched_running = None _delay_on = None running = False def __new__(cls,s): if s.id in sites: return sites[s.id] self = object.__new__(cls) sites[s.id] = self self.s = s self.connect() self._delay_on = Semaphore() self.controllers = set() self.envgroups = set() self.meters = {} for M in METERS: ml = set() self.meters[M.meter_type] = ml for d in getattr(self.s,M.meter_type+"_meters").all(): ml.add(M(d)) self.log("Startup") self.connect_monitors(do_controllers=False) signal.signal(signal.SIGINT,self.do_shutdown) signal.signal(signal.SIGTERM,self.do_shutdown) signal.signal(signal.SIGHUP,self.do_syncsched) self.running = True return self def __init__(self,s): pass def do_shutdown(self,x,y,**k): gevent.spawn_later(0.1,connwrap,self.shutdown) def do_syncsched(self,x,y): gevent.spawn_later(0.1,connwrap,self.syncsched) def syncsched(self): print("Sync+Sched", file=sys.stderr) self.sync() self.refresh() self.run_sched_task(reason="Sync+Sched") def delay_on(self): self._delay_on.acquire() gevent.spawn_later(1,self._delay_on.release) @async_gevent def check_flow(self,**k): for c in self.controllers: c.check_flow(**k) def connect(self): d = dict() if self.s.port: d['port'] = self.s.port if self.s.username: d['login'] = self.s.username if self.s.password: d['password'] = self.s.password if self.s.virtualhost: d['virtualhost'] = self.s.virtualhost try: self.qb = qbroker.make_unit_gevent("moat.rain.runschedule", amqp=dict(server=dict(host=self.s.host, **d))) except Exception: print("Could not connect:",self.s.host, file=sys.stderr) raise def maybe_restart(self): self.log("reconnecting") try: self.connect() except Exception: print_exc() gevent.spawn_later(100,connwrap,self.maybe_restart) else: self.connect_monitors() def connect_monitors(self,do_controllers=True): if self.qb is None: return if do_controllers: for c in self.controllers: c.connect_monitors() for mm in self.meters.values(): for m in mm: m.connect_monitors() n = self.s.var.replace(' ','.') self.ckf = self.qb.register_rpc_gevent("rain.check.flow."+n, self.check_flow) self.cks = self.qb.register_rpc_gevent("rain.read.schedule."+n, partial(self.run_sched_ext,reason="read schedule")) self.ckt = self.qb.register_rpc_gevent("rain.sync."+n, self.sync_ext) self.cku = self.qb.register_rpc_gevent("rain.shutdown."+n, self.do_shutdown) def sync(self,**k): print("Sync", file=sys.stderr) for c in self.controllers: c.sync() for eg in self.envgroups: eg.sync() for mm in self.meters.values(): for m in mm: m.sync() self.run_main_task() #Save(None) print("Sync end", file=sys.stderr) sync_ext = async_gevent(sync) def shutdown(self,**k): print("Shutdown", file=sys.stderr) signal.signal(signal.SIGINT,signal.SIG_DFL) signal.signal(signal.SIGTERM,signal.SIG_DFL) if self.running: self.running = False self.sync() for eg in self.envgroups: eg.sync() for c in self.controllers: c.shutdown() for mm in self.meters.values(): for m in mm: m.shutdown() #Save(None) sys.exit(0) def run_schedule(self): for c in self.controllers: c.run_schedule() def refresh(self): self.s.refresh() for eg in self.envgroups: eg.refresh() for c in self.controllers: c.refresh() for mm in self.meters.values(): for m in mm: m.refresh() def log(self,txt): log(self.s,txt) def add_controller(self,controller): self.controllers.add(controller) def no_rain(self): """Rain has stopped.""" # called by timer self.rain_timer = None self.log("Stopped raining") self.run_main_task() def has_rain(self): """Some monitor told us that it started raining""" r,self.rain_timer = self.rain_timer,gevent.spawn_later(self.s.db_rain_delay,connwrap,self.no_rain) if r: r.kill() return self.log("Started raining") self.rain = True #for v in self.s.valves.all(): vo = Valve.objects.filter(controller__site=self.s, runoff__gt=0) for v in vo.all(): valve = SchedValve(v) if valve.locked: continue try: valve._off(1) except NotConnected: pass except Exception: self.log_error(v) Schedule.objects.filter(valve__in=vo, start__gte=now()-timedelta(1),seen=False).delete() self.run_main_task() def send_command(self,*a,**k): # TODO: return a sensible error and handle that correctly if self.qb is None: raise NotConnected try: self.qb.rpc_gevent("moat.cmd",args=a, _dest="moat.main",**k) except asyncio.TimeoutError as e: print("Timeout sending %s %s" % (repr(a),repr(k))) def run_every(self,delay): """Initiate running the calculation and scheduling loop every @delay seconds.""" if self._run_delay is not None: self._run_delay = delay # just update return self._run_delay = delay self._run_last = now() self._running = Semaphore() self._run_result = None sd = self._run_delay.total_seconds()/10 if sd < 66: sd = 66 self._run = gevent.spawn_later(sd, connwrap,self.run_main_task, kill=False) if self._sched is not None: self._sched.kill() self._sched = gevent.spawn_later(2, connwrap,self.run_sched_task, kill=False, reason="run_every") def run_main_task(self, kill=True): """Run the calculation loop.""" res = None if not self._running.acquire(blocking=False): return self._run_result.get() try: self._run_result = AsyncResult() if kill: self._run.kill() n = now() ts = (n-self._run_last).total_seconds() if ts < 5: try: res = self.s.history.order_by("-time")[0] except IndexError: return None else: return res self._run_last = n res = self.main_task() return res finally: self._run = gevent.spawn_later((self._run_last+self._run_delay-n).total_seconds(), connwrap,self.run_main_task, kill=False) r,self._run_result = self._run_result,None self._running.release() r.set(res) def current_history_entry(self,delta=15): # assure that the last history entry is reasonably current try: he = self.s.history.order_by("-time")[0] except IndexError: pass else: if (now()-he.time).total_seconds() < delta: return he return self.new_history_entry() def new_history_entry(self,rain=0): """Create a new history entry""" values = {} n = now() for t,ml in self.meters.items(): sum_it = False sum_val = 0 sum_f = 0 for m in ml: f = m.weight v = m.get_value() if m.last_time is None: f *= 0.01 else: s = (n - m.last_time).total_seconds() if s > METER_MAXTIME: f *= 0.01 elif s > METER_TIME: f *= METER_TIME/s if v is not None: sum_val += f*v sum_f += f if m.sum_it: sum_it = True if sum_f: if not sum_it: sum_val /= sum_f values[t] = sum_val print("Values:",values, file=sys.stderr) h = History(site=self.s,time=now(),**values) h.save() return h def sync_history(self): for c in self.controllers: c.sync_history() def main_task(self): print("MainTask", file=sys.stderr) self.refresh() h = self.current_history_entry(3) self.sync_history() gevent.spawn_later(2,connwrap,self.sched_task) print("MainTask end",h, file=sys.stderr) return h def run_sched_task(self,delayed=False,reason=None,kill=True, **k): if self._sched_running is not None: print("RunSched.running",reason, file=sys.stderr) return self._sched_running.get() if self._sched is not None: if kill: self._sched.kill() if delayed: print("RunSched.delay",reason, file=sys.stderr) self._sched = gevent.spawn_later(10,connwrap,self.run_sched_task,kill=False,reason="Timer 10") return print("RunSched",reason, file=sys.stderr) self._sched = None self._sched_running = AsyncResult() try: self.sched_task() except Exception: self.log(format_exc()) finally: r,self._sched_running = self._sched_running,None if self._sched is None: self._sched = gevent.spawn_later(600,connwrap,self.run_sched_task,kill=False,reason="Timer 600") if r is not None: r.set(None) print("RunSched end", file=sys.stderr) run_sched_ext = async_gevent(run_sched_task) def sched_task(self, kill=True): self.refresh() self.run_schedule()
class SchedValve(SchedCommon): """Mirrors (and monitors) a valve.""" locked = False # external command, don't change sched = None sched_ts = None sched_job = None sched_lock = None on = False on_ts = None flow = 0 _flow_check = None def __new__(cls,v): if v.id in valves: return valves[v.id] self = object.__new__(cls) valves[v.id] = self self.v = v self.site = SchedSite(self.v.controller.site) self.env = EnvGroup(self.v.envgroup) self.controller = SchedController(self.v.controller) self.sched_lock = Semaphore() if self.site.qb: try: self.site.send_command("set","output","off",*(self.v.var.split())) except NotConnected: pass except Exception as e: raise RuntimeError(self.v.var) from e return self def __init__(self,v): pass def _on(self,caller,sched=None,duration=None): print("Open",caller,self.v.var, file=sys.stderr) self.site.delay_on() if duration is None and sched is not None: duration = sched.duration if self.controller.has_max_on(): print("… but too many:", ", ".join(str(v) for v in self.controller.c.valves.all() if SchedValve(v).on), file=sys.stderr) if sched: sched.update(seen = False) self.log("NOT running %s for %s: too many"%(self.v,duration,)) raise TooManyOn(self) if duration is None: self.log("Run (indefinitely)") self.site.send_command("set","output","on",*(self.v.var.split())) else: self.log("Run for %s"%(duration,)) if not isinstance(duration,six.integer_types): duration = duration.total_seconds() try: self.site.send_command("set","output","on",*(self.v.var.split()), sub=(("for",duration),("async",))) except Exception: # Something broke. Try to turn this thing off. self.log(format_exc()) self.site.send_command("set","output","off",*(self.v.var.split())) raise RuntimeError("Could not start (logged)") if sched is not None: if self.v.verbose: self.log("Opened for %s"%(sched,)) self.sched = sched if not sched.seen: sched.update(start=now(), seen=True) sched.refresh() #Save(sched) else: if self.v.verbose: self.log("Opened for %s"%(duration,)) def _off(self, num): if self.on: if self.v.verbose: self.log("Closing "+str(num)) print("Close",self.v.var, file=sys.stderr) try: self.site.send_command("set","output","off",*(self.v.var.split())) except NotConnected: pass def shutdown(self): if self._flow_check is not None: self._flow_check.dead() def run_schedule(self): if not self.sched_lock.acquire(blocking=False): if self.v.verbose: print("SCHED LOCKED1 %s" % (self.v.name,), file=sys.stderr) return try: self._run_schedule() except Exception: self.log(format_exc()) finally: self.sched_lock.release() def _run_schedule(self): if self.sched_job is not None: self.sched_job.kill() self.sched_job = None if self.locked: if self.v.verbose: print("SCHED LOCKED2 %s" % (self.v.name,), file=sys.stderr) return n = now() try: if self.sched is not None: self.sched.refresh() if self.sched.end <= n: if self.v.verbose: print("Turn off: %s+%s <= %s" % (self.sched.start,self.sched.duration,n), file=sys.stderr) self._off(2) self.sched = None else: self.sched_job = gevent.spawn_later((self.sched.end-n).total_seconds(),connwrap,self.run_sched_task,reason="_run_schedule 1") if self.v.verbose: print("SCHED LATER %s: %s" % (self.v.name,humandelta(self.sched.end-n)), file=sys.stderr) return except ObjectDoesNotExist: pass # somebody deleted it *shrug* sched = None if self.sched_ts is None: try: sched = self.v.schedules.filter(start__lt=n).order_by("-start")[0] except IndexError: self.sched_ts = n-timedelta(1,0) else: self.sched_ts = sched.end if sched.end > n: # still running if self.v.verbose: print("SCHED RUNNING %s: %s" % (self.v.name,humandelta(sched.end-n)), file=sys.stderr) try: self._on(1,sched, sched.end-n) except TooManyOn: self.log("Could not schedule: too many open valves") except NotConnected: self.log("Could not schedule: connection to MoaT failed") return try: sched = self.v.schedules.filter(start__gte=self.sched_ts).order_by("start")[0] except IndexError: if self.v.verbose: print("SCHED EMPTY %s: %s" % (self.v.name,str_tz(self.sched_ts)), file=sys.stderr) self._off(3) return if sched.end <= n: if self.v.verbose: print("SCHED %s: sched %d done for %s" % (self.v.name,sched.id,humandelta(n-sched.end)), file=sys.stderr) self.sched_ts = None return if sched.start > n: if self.v.verbose: print("SCHED %s: sched %d in %s" % (self.v.name,sched.id,humandelta(sched.start-n)), file=sys.stderr) self._off(4) self.sched_job = gevent.spawn_later((sched.start-n).total_seconds(),connwrap,self.run_sched_task,reason="_run_schedule 2") return try: self._on(2,sched) except TooManyOn: self.log("Could not schedule: too many open valves") except NotConnected: self.log("Could not schedule: connection to MoaT failed") def run_sched_task(self,reason="valve"): self.sched_job = None self.site.run_sched_task(reason=reason) run_sched_ext = async_gevent(run_sched_task) def add_flow(self, val): if self._flow_check is not None: if self._flow_check.add_flow(val): return if self.v.verbose: print("FLOW %s: %s %s" % (self.v.name,self.flow,val), file=sys.stderr) self.flow += val @async_gevent def check_flow(self,**k): cf = None try: cf = FlowCheck(self) cf.run() except Exception as ex: log_error(self.v) if cf is not None: cf._unlock() def refresh(self): self.v.refresh() # if self.sched is not None: # self.sched.refresh() def connect_monitors(self): if self.site.qb is None: return n = self.v.var.replace(' ','.') self.mon = self.site.qb.register_alert_gevent("moat.event.output.change."+n, self.watch_state, call_conv=CC_DICT) self.ckf = self.site.qb.register_rpc_gevent("rain.check.flow."+n, self.check_flow, call_conv=CC_DICT) @async_gevent def watch_state(self,value=None,**kv): """output change NAME ::value ON""" on = (str(value).lower() in ("1","true","on")) if self._flow_check is not None: # TODO self.on = on self._flow_check.state(on) return if self.locked: self.on = on return try: if on != self.on: n=now() print("Report %s" % ("ON" if on else "OFF"),self.v.var,self.sched, file=sys.stderr) if self.sched is not None and not on: self.sched.update(db_duration=(n-self.sched.start).total_seconds()) self.sched.refresh() self.sched_ts = self.sched.end self.sched = None flow,self.flow = self.flow,0 # If nothing happened, calculate. if not on: duration = n-self.on_ts maxflow = self.v.flow * duration.total_seconds() if (not flow or not self.v.feed.var) or flow > 2*maxflow: flow = maxflow self.new_level_entry(flow) if not on: if self.v.level > self.v.stop_level + (self.v.start_level-self.v.stop_level)/5: self.v.update(priority=True) self.log("Done for %s, level is now %s"%(duration,self.v.level)) self.on = on self.on_ts = n except Exception: print_exc() def sync(self): flow,self.flow = self.flow,0 self.new_level_entry(flow) def sync_history(self): n=now() try: lv = self.v.levels.order_by("-time")[0] except IndexError: pass else: if self.v.time > lv.time: self.log("Timestamp downdate: %s %s" % (self.v.time,lv.time)) self.v.update(time = lv.time) self.v.refresh() #Save(self.v) if (n-self.v.time).total_seconds() >= 295: flow,self.flow = self.flow,0 self.new_level_entry(flow) def new_level_entry(self,flow=0): self.site.current_history_entry() n=now() self.v.refresh() hts = None try: lv = self.v.levels.order_by("-time")[0] except IndexError: ts = n-timedelta(1,0) else: ts = lv.time sum_f = 0 sum_r = 0 for h in self.site.s.history.filter(time__gt=ts).order_by("time"): if self.v.verbose>2: self.log("Env factor for %s: T=%s W=%s S=%s"%(h,h.temp,h.wind,h.sun)) f = self.env.env_factor(h, logger=self.log if self.v.verbose>2 else None)*self.v.adj if self.v.verbose>1: self.log("Env factor for %s is %s"%(h,f)) sum_f += self.site.s.db_rate * self.v.do_shade(self.env.eg.factor*f) * (h.time-ts).total_seconds() sum_r += self.v.runoff*h.rain ts=h.time if self.v.verbose: self.log("Apply env %f, rain %r,, flow %f = %f" % (sum_f,sum_r,flow,flow/self.v.area)) if self.v.time == ts: return if self.v.level < 0: level = 0 else: level = F('level') level += sum_f if (flow > 0 or sum_r > 0) and self.v.level > self.v.max_level: level = self.v.max_level level -= flow/self.v.area+sum_r #if level < 0: # self.log("Level %s ?!?"%(self.v.level,)) self.v.update(time=ts, level=level) self.v.refresh() lv = Level(valve=self.v,time=ts,level=self.v.level,flow=flow) lv.save() if self.on and not (self.sched and self.sched.forced) and self.v.level <= self.v.stop_level: self._off(5) def log(self,txt): log(self.v,txt)
class Pool(Group): def __init__(self, size=None, greenlet_class=None): if size is not None and size < 1: raise ValueError( 'Invalid size for pool (positive integer or None required): %r' % (size, )) Group.__init__(self) self.size = size if greenlet_class is not None: self.greenlet_class = greenlet_class if size is None: self._semaphore = DummySemaphore() else: self._semaphore = Semaphore(size) def wait_available(self): self._semaphore.wait() def full(self): return self.free_count() <= 0 def free_count(self): if self.size is None: return 1 return max(0, self.size - len(self)) def start(self, greenlet): self._semaphore.acquire() try: self.add(greenlet) except: self._semaphore.release() raise greenlet.start() def spawn(self, *args, **kwargs): self._semaphore.acquire() try: greenlet = self.greenlet_class.spawn(*args, **kwargs) self.add(greenlet) except: self._semaphore.release() raise return greenlet def spawn_link(self, *args, **kwargs): self._semaphore.acquire() try: greenlet = self.greenlet_class.spawn_link(*args, **kwargs) self.add(greenlet) except: self._semaphore.release() raise return greenlet def spawn_link_value(self, *args, **kwargs): self._semaphore.acquire() try: greenlet = self.greenlet_class.spawn_link_value(*args, **kwargs) self.add(greenlet) except: self._semaphore.release() raise return greenlet def spawn_link_exception(self, *args, **kwargs): self._semaphore.acquire() try: greenlet = self.greenlet_class.spawn_link_exception( *args, **kwargs) self.add(greenlet) except: self._semaphore.release() raise return greenlet def discard(self, greenlet): Group.discard(self, greenlet) self._semaphore.release()
class BaseDataHandler(object): _params = { 'POLLING_INTERVAL' : 3600, 'PATCHABLE_CONFIG_KEYS' : ['stream_id','constraints'] } _polling = False _polling_glet = None _dh_config = {} _rr_cli = None def __init__(self, rr_cli, stream_registrar, dh_config): self._dh_config=dh_config self._stream_registrar = stream_registrar self._rr_cli = rr_cli def set_event_callback(self, evt_callback): self._event_callback = evt_callback def _dh_event(self, type, value): event = { 'type' : type, 'value' : value, 'time' : time.time() } self._event_callback(event) def _poll(self): """ Internal polling method, run inside a greenlet, that triggers execute_acquire_data without configuration mods The polling interval (in seconds) is retrieved from the POLLING_INTERVAL parameter """ self._polling = True interval = get_safe(self._params, 'POLLING_INTERVAL', 3600) log.debug('Polling interval: {0}'.format(interval)) while self._polling: self.execute_acquire_data() time.sleep(interval) def cmd_dvr(self, cmd, *args, **kwargs): """ Command a DataHandler by request-reply messaging. Package command message and send on blocking command socket. Block on same socket to receive the reply. Return the driver reply. @param cmd The DataHandler command identifier. @param args Positional arguments of the command. @param kwargs Keyword arguments of the command. @retval Command result. """ # Package command dictionary. #need to account for observatory_execute_resource commands #connect -> Not used #get_current_state -> Not used #discover -> Not used #disconnect -> Not used log.debug('cmd_dvr received command \'{0}\' with: args={1} kwargs={2}'.format(cmd, args, kwargs)) reply = None if cmd == 'initialize': # Delegate to BaseDataHandler.initialize() reply = self.initialize(*args, **kwargs) elif cmd == 'get': # Delegate to BaseDataHandler.get() reply = self.get(*args, **kwargs) elif cmd == 'set': # Delegate to BaseDataHandler.set() reply = self.set(*args, **kwargs) elif cmd == 'get_resource_params': # Delegate to BaseDataHandler.get_resource_params() reply = self.get_resource_params(*args, **kwargs) elif cmd == 'get_resource_commands': # Delegate to BaseDataHandler.get_resource_commands() reply = self.get_resource_commands(*args, **kwargs) elif cmd == 'execute_acquire_data': # Delegate to BaseDataHandler.execute_acquire_data() reply = self.execute_acquire_data(*args, **kwargs) elif cmd == 'execute_start_autosample': # Delegate to BaseDataHandler.execute_start_autosample() reply = self.execute_start_autosample(*args, **kwargs) elif cmd == 'execute_stop_autosample': # Delegate to BaseDataHandler.execute_stop_autosample() reply = self.execute_stop_autosample(*args, **kwargs) elif cmd in ['configure','connect','disconnect','get_current_state','discover','execute_acquire_sample']: # Disregard log.info('Command \'{0}\' not used by DataHandler'.format(cmd)) pass else: desc='Command \'{0}\' unknown by DataHandler'.format(cmd) log.info(desc) raise InstrumentCommandException(desc) return reply def initialize(self, *args, **kwargs): #TODO: Add documentation #TODO: This should put the DataHandler back into an 'unconfigured' state """ Called from: InstrumentAgent._handler_idle_reset InstrumentAgent._handler_idle_go_inactive InstrumentAgent._handler_stopped_reset InstrumentAgent._handler_stopped_go_inactive InstrumentAgent._handler_observatory_reset InstrumentAgent._handler_observatory_go_inactive InstrumentAgent._handler_uninitialized_initialize |--> ExternalDataAgent._start_driver """ log.debug('Initializing DataHandler...') self._glet_queue = [] self._semaphore=Semaphore() return None def configure(self, *args, **kwargs): #TODO: Add documentation #TODO: This should configure the DataHandler for the particular dataset """ Called from: InstrumentAgent._handler_inactive_go_active """ log.debug('Configuring DataHandler: args = {0}'.format(args)) try: self._dh_config = args[0] except IndexError: raise InstrumentParameterException('\'acquire_data\' command requires a config dict as the first argument') return def execute_acquire_data(self, *args): """ Creates a copy of self._dh_config, creates a publisher, and spawns a greenlet to perform a data acquisition cycle If the args[0] is a dict, any entries keyed with one of the 'PATCHABLE_CONFIG_KEYS' are used to patch the config Greenlet binds to BaseDataHandler._acquire_data and passes the publisher and config Disallows multiple "new data" (unconstrained) requests using BaseDataHandler._semaphore lock Called from: InstrumentAgent._handler_observatory_execute_resource |--> ExternalDataAgent._handler_streaming_execute_resource @parameter args First argument can be a config dictionary """ log.debug('Executing acquire_data: args = {0}'.format(args)) # Make a copy of the config to ensure no cross-pollution config = self._dh_config.copy() # Patch the config if mods are passed in try: config_mods = args[0] if not isinstance(config_mods, dict): raise IndexError() log.debug('Configuration modifications provided: {0}'.format(config_mods)) for k in self._params['PATCHABLE_CONFIG_KEYS']: if get_safe(config_mods, k): config[k] = config_mods[k] except IndexError: log.info('No configuration modifications were provided') # Verify that there is a stream_id member in the config stream_id = get_safe(config, 'stream_id') if not stream_id: raise ConfigurationError('Configuration does not contain required \'stream_id\' member') isNew = get_safe(config, 'constraints') is None if isNew and not self._semaphore.acquire(blocking=False): log.warn('Already acquiring new data - action not duplicated') return if isNew: # Get any NewDataCheck attachments and add them to the config ext_ds_id = get_safe(config,'external_dataset_res_id') if ext_ds_id: try: attachment_objs, _ = self._rr_cli.find_objects(ext_ds_id, PRED.hasAttachment, RT.Attachment, False) for attachment_obj in attachment_objs: kwds = set(attachment_obj.keywords) if 'NewDataCheck' in kwds: log.debug('Found NewDataCheck attachment: {0}'.format(attachment_obj)) config['new_data_check'] = attachment_obj.content break else: log.debug('Found attachment: {0}'.format(attachment_obj)) except NotFound: raise InstrumentException('ExternalDatasetResource \'{0}\' not found'.format(ext_ds_id)) if not get_safe(config, 'new_data_check'): config['new_data_check'] = None # Create a publisher to pass into the greenlet publisher = self._stream_registrar.create_publisher(stream_id=stream_id) # Spawn a greenlet to do the data acquisition and publishing g = spawn(self._acquire_data, config, publisher, self._unlock_new_data_callback) log.debug('** Spawned {0}'.format(g)) self._glet_queue.append(g) def execute_start_autosample(self, *args, **kwargs): #TODO: Add documentation #TODO: Fix raises statements """ Put the DataHandler into streaming mode and start polling for new data Called from: InstrumentAgent._handler_observatory_go_streaming @raises InstrumentTimeoutException: @raises InstrumentProtocolException: @raises NotImplementedException: @raises InstrumentParameterException: """ log.debug('Entered execute_start_autosample with args={0} & kwargs={1}'.format(args, kwargs)) if not self._polling and self._polling_glet is None: self._polling_glet = spawn(self._poll) return None def execute_stop_autosample(self, *args, **kwargs): #TODO: Add documentation #TODO: Fix raises statements """ Stop polling for new data and put the DataHandler into observatory mode Called from: InstrumentAgent._handler_streaming_go_observatory @raises InstrumentTimeoutException: @raises InstrumentProtocolException: @raises NotImplementedException: @raises InstrumentParameterException: """ log.debug('Entered execute_stop_autosample with args={0} & kwargs={1}'.format(args, kwargs)) if self._polling and not self._polling_glet is None: self._polling_glet.kill() self._polling = False self._polling_glet = None return None def get(self, *args, **kwargs): #TODO: Add documentation #TODO: Fix raises statements """ Called from: InstrumentAgent._handler_get_params @raises InstrumentTimeoutException: @raises InstrumentProtocolException: @raises NotImplementedException: @raises InstrumentParameterException: """ try: pnames=args[0] except IndexError: log.warn("No argument provided to get, return all parameters") pnames = [DataHandlerParameter.ALL] result = None if DataHandlerParameter.ALL in pnames: result = self._params else: if not isinstance(pnames, (list,tuple)): raise InstrumentParameterException('Get argument not a list or tuple: {0}'.format(pnames)) result={} for pn in pnames: try: log.debug('Get parameter with key: {0}'.format(pn)) result[pn] = self._params[pn] except KeyError: log.debug('\'{0}\' not found in self._params'.format(pn)) raise InstrumentParameterException('{0} is not a valid parameter for this DataHandler.'.format(pn)) return result def set(self, *args, **kwargs): #TODO: Add documentation #TODO: Fix raises statements """ Called from: InstrumentAgent._handler_observatory_set_params @raises InstrumentTimeoutException: @raises InstrumentProtocolException: @raises NotImplementedException: @raises InstrumentParameterException: """ # Retrieve required parameter. # Raise if no parameter provided, or not a dict. try: params = args[0] except IndexError: raise InstrumentParameterException('Set command requires a parameter dict.') to_raise = [] if not isinstance(params, dict): raise InstrumentParameterException('Set parameters not a dict.') else: for (key, val) in params.iteritems(): if key in self._params: log.debug('Set parameter \'{0}\' = {1}'.format(key, val)) self._params[key] = val else: log.debug('Parameter \'{0}\' not in self._params and cannot be set'.format(key)) to_raise.append(key) if len(to_raise) > 0: log.debug('Raise InstrumentParameterException for un-set parameters: {0}'.format(to_raise)) raise InstrumentParameterException('Invalid parameter(s) could not be set: {0}'.format(to_raise)) def get_resource_params(self, *args, **kwargs): """ Return list of resource parameters. Implemented in specific DataHandlers Called from: InstrumentAgent._handler_get_resource_params """ return self._params.keys() def get_resource_commands(self, *args, **kwargs): """ Return list of DataHandler execute commands available. Called from: InstrumentAgent._handler_get_resource_commands """ cmds = [cmd.replace('execute_','') for cmd in dir(self) if cmd.startswith('execute_')] return cmds def _unlock_new_data_callback(self, caller): log.debug('** Release {0}'.format(caller)) self._semaphore.release() @classmethod def _acquire_data(cls, config, publisher, unlock_new_data_callback): """ Ensures required keys (such as stream_id) are available from config, configures the publisher and then calls: BaseDataHandler._new_data_constraints (only if config does not contain 'constraints') BaseDataHandler._publish_data passing BaseDataHandler._get_data as a parameter @param config Dict containing configuration parameters, may include constraints, formatters, etc @param unlock_new_data_callback BaseDataHandler callback function to allow conditional unlocking of the BaseDataHandler._semaphore """ log.debug('start _acquire_data: config={0}'.format(config)) cls._init_acquisition_cycle(config) constraints = get_safe(config,'constraints') if not constraints: gevent.getcurrent().link(unlock_new_data_callback) constraints = cls._new_data_constraints(config) if constraints is None: raise InstrumentParameterException("Data constraints returned from _new_data_constraints cannot be None") config['constraints'] = constraints cls._publish_data(publisher, cls._get_data(config)) # Publish a 'TestFinished' event if get_safe(config,'TESTING'): log.debug('Publish TestingFinished event') pub = EventPublisher('DeviceCommonLifecycleEvent') pub.publish_event(origin='BaseDataHandler._acquire_data', description='TestingFinished') @classmethod def _init_acquisition_cycle(cls, config): """ Allows the concrete implementation to initialize/prepare objects the data handler will use repeatedly (such as a dataset object) in cls._new_data_constraints and/or cls._get_data Objects should be added to the config so they are available later in the workflow """ raise NotImplementedException('{0}.{1} must implement \'_init_acquisition_cycle\''.format(cls.__module__,cls.__name__)) @classmethod def _new_data_constraints(cls, config): #TODO: Document what "constraints" looks like (yml)!! """ Determines the appropriate constraints for acquiring any "new data" from the external dataset Returned value cannot be None and is assigned to config['constraints'] The format of the constraints are documented: @param config dict of configuration parameters - may be used to generate the returned 'constraints' dict @retval dict that contains the constraints for retrieval of new data from the external dataset """ raise NotImplementedException('{0}.{1} must implement \'_new_data_constraints\''.format(cls.__module__,cls.__name__)) @classmethod def _get_data(cls, config): """ Iterable function that acquires data from a source iteratively based on constraints provided by config Passed into BaseDataHandler._publish_data and iterated to publish samples. @param config dict containing configuration parameters, may include constraints, formatters, etc @return an iterable that returns well-formed Granule objects on each iteration """ raise NotImplementedException('{0}.{1} must implement \'_get_data\''.format(cls.__module__,cls.__name__)) @classmethod def _publish_data(cls, publisher, data_generator): """ Iterates over the data_generator and publishes granules to the stream indicated in stream_id """ if data_generator is None or not hasattr(data_generator,'__iter__'): raise InstrumentDataException('Invalid object returned from _get_data: returned object cannot be None and must have \'__iter__\' attribute') for count, gran in enumerate(data_generator): if isinstance(gran, Granule): publisher.publish(gran) else: log.warn('Could not publish object returned by _get_data: {0}'.format(gran)) #TODO: Persist the 'state' of this operation so that it can be re-established in case of failure #TODO: When finished publishing, update (either directly, or via an event callback to the agent) the UpdateDescription @classmethod def _calc_iter_cnt(cls, total_recs, max_rec): """ Given the total number of records and the maximum records allowed in a granule, calculates the number of iterations required to traverse the entire array in chunks of size max_rec @param total_recs The total number of records @param max_rec The maximum number of records allowed in a granule """ cnt = total_recs / max_rec if total_recs % max_rec > 0: cnt += 1 return cnt
class BaseDataHandler(object): _params = {DataHandlerParameter.POLLING_INTERVAL : 3600} _polling = False _polling_glet = None _config = {} def set_event_callback(self, evt_callback): self._event_callback = evt_callback def _dh_event(self, type, value): event = { 'type' : type, 'value' : value, 'time' : time.time() } self._event_callback(event) def _poll(self): """ Internal polling method, run inside a greenlet, that triggers acquire_data for "new" data The polling interval (in seconds) is retrieved from the POLLING_INTERVAL parameter """ self._polling = True interval = get_safe(self._params, DataHandlerParameter.POLLING_INTERVAL, 3600) while True: self.execute_acquire_data({'stream_id':'first_new','TESTING':True}) time.sleep(interval) def cmd_dvr(self, cmd, *args, **kwargs): """ Command a DataHandler by request-reply messaging. Package command message and send on blocking command socket. Block on same socket to receive the reply. Return the driver reply. @param cmd The DataHandler command identifier. @param args Positional arguments of the command. @param kwargs Keyword arguments of the command. @retval Command result. """ # Package command dictionary. #need to account for observatory_execute_resource commands #connect -> Not used #get_current_state -> Not used #discover -> Not used #disconnect -> Not used log.debug('cmd_dvr received command \'{0}\' with: args={1} kwargs={2}'.format(cmd, args, kwargs)) reply = None if cmd == 'configure': # Delegate to BaseDataHandler.configure() reply = self.configure(*args, **kwargs) elif cmd == 'initialize': # Delegate to BaseDataHandler.initialize() reply = self.initialize(*args, **kwargs) elif cmd == 'get': # Delegate to BaseDataHandler.get() reply = self.get(*args, **kwargs) elif cmd == 'set': # Delegate to BaseDataHandler.set() reply = self.set(*args, **kwargs) elif cmd == 'get_resource_params': # Delegate to BaseDataHandler.get_resource_params() reply = self.get_resource_params(*args, **kwargs) elif cmd == 'get_resource_commands': # Delegate to BaseDataHandler.get_resource_commands() reply = self.get_resource_commands(*args, **kwargs) elif cmd == 'execute_acquire_data': # Delegate to BaseDataHandler.execute_acquire_data() reply = self.execute_acquire_data(*args, **kwargs) elif cmd == 'execute_acquire_sample': #TODO: Can we change these names? acquire_data would be a better name for EOI... # Delegate to BaseDataHandler.execute_acquire_sample() reply = self.execute_acquire_sample(*args, **kwargs) elif cmd == 'execute_start_autosample': #TODO: Can we change these names? stop_polling would be a better name for EOI... # Delegate to BaseDataHandler.execute_start_autosample() reply = self.execute_start_autosample(*args, **kwargs) elif cmd == 'execute_stop_autosample': #TODO: Can we change these names? stop_polling would be a better name for EOI... # Delegate to BaseDataHandler.execute_stop_autosample() reply = self.execute_stop_autosample(*args, **kwargs) elif cmd in ['connect','disconnect','get_current_state','discover']: # Disregard pass else: desc='Command unknown by DataHandler: {0}'.format(cmd) log.info(desc) raise UnknownCommandError(desc) return reply def initialize(self, *args, **kwargs): #TODO: Add documentation #TODO: This should put the DataHandler back into an 'unconfigured' state """ Called from: InstrumentAgent._handler_idle_reset InstrumentAgent._handler_idle_go_inactive InstrumentAgent._handler_stopped_reset InstrumentAgent._handler_stopped_go_inactive InstrumentAgent._handler_observatory_reset InstrumentAgent._handler_observatory_go_inactive InstrumentAgent._handler_uninitialized_initialize |--> ExternalDataAgent._start_driver """ log.debug('Initializing DataHandler...') self._glet_queue = [] self._semaphore=Semaphore() return None def configure(self, *args, **kwargs): #TODO: Add documentation #TODO: This should configure the DataHandler for the particular dataset """ Called from: InstrumentAgent._handler_inactive_go_active """ log.debug('Configuring DataHandler: args = {0}'.format(args)) try: self._config = args[0] except IndexError: raise ParameterError('\'acquire_data\' command requires a config dict as the first argument') return def execute_acquire_data(self, *args): """ Spawns a greenlet to perform a data acquisition Calls BaseDataHandler._acquire_data Disallows multiple "new data" (unconstrained) requests using BaseDataHandler._semaphore lock Called from: InstrumentAgent._handler_observatory_execute_resource |--> ExternalDataAgent._handler_streaming_execute_resource @parameter args First argument should be a config dictionary """ try: config = args[0] except IndexError: raise ParameterError('\'acquire_data\' command requires a config dict.') if not isinstance(config, dict): raise TypeError('args[0] of \'acquire_data\' is not a dict.') else: if get_safe(config,'constraints') is None and not self._semaphore.acquire(blocking=False): log.warn('Already acquiring new data - action not duplicated') return g = spawn(self._acquire_data, config, self._unlock_new_data_callback) log.debug('** Spawned {0}'.format(g)) self._glet_queue.append(g) def execute_acquire_sample(self, *args, **kwargs): #TODO: Add documentation #TODO: Fix raises statements """ Called from: InstrumentAgent._handler_observatory_execute_resource |--> ExternalDataAgent._handler_streaming_execute_resource """ # This returns the sample to the agent as an event sample = {'stream_name':'data_stream','p': [-6.945], 'c': [0.08707], 't': [20.002], 'time': [1333752198.450622]} self._dh_event(DriverAsyncEvent.SAMPLE, sample) # This does 'rpc' style event return sample def execute_start_autosample(self, *args, **kwargs): #TODO: Add documentation #TODO: Fix raises statements """ Put the DataHandler into streaming mode and start polling for new data Called from: InstrumentAgent._handler_observatory_go_streaming @raises TimeoutError: @raises ProtocolError: @raises NotImplementedError: @raises ParameterError: """ log.debug('Entered execute_start_autosample with args={0} & kwargs={1}'.format(args, kwargs)) if not self._polling and self._polling_glet is None: self._polling_glet = spawn(self._poll) return None def execute_stop_autosample(self, *args, **kwargs): #TODO: Add documentation #TODO: Fix raises statements """ Stop polling for new data and put the DataHandler into observatory mode Called from: InstrumentAgent._handler_streaming_go_observatory @raises TimeoutError: @raises ProtocolError: @raises NotImplementedError: @raises ParameterError: """ log.debug('Entered execute_stop_autosample with args={0} & kwargs={1}'.format(args, kwargs)) if self._polling and not self._polling_glet is None: self._polling_glet.kill() self._polling = False self._polling_glet = None return None def get(self, *args, **kwargs): #TODO: Add documentation #TODO: Fix raises statements """ Called from: InstrumentAgent._handler_get_params @raises TimeoutError: @raises ProtocolError: @raises NotImplementedError: @raises ParameterError: """ try: pnames=args[0] except IndexError: log.warn("No argument provided to get, return all parameters") pnames = [DataHandlerParameter.ALL] result = None if DataHandlerParameter.ALL in pnames: result = self._params else: if not isinstance(pnames, (list,tuple)): raise ParameterError('Get argument not a list or tuple: {0}'.format(pnames)) result={} for pn in pnames: try: result[pn] = self._params.get(pn) except KeyError: raise ParameterError('{0} is not a valid parameter for this DataHandler.'.format(pn)) return result def set(self, *args, **kwargs): #TODO: Add documentation #TODO: Fix raises statements """ Called from: InstrumentAgent._handler_observatory_set_params @raises TimeoutError: @raises ProtocolError: @raises NotImplementedError: @raises ParameterError: """ # Retrieve required parameter. # Raise if no parameter provided, or not a dict. try: params = args[0] except IndexError: raise ParameterError('Set command requires a parameter dict.') if not isinstance(params, dict): raise ParameterError('Set parameters not a dict.') else: for (key, val) in params.iteritems(): self._params[key] = val #TODO: Add rejection of unknown parameter def get_resource_params(self, *args, **kwargs): """ Return list of resource parameters. Implemented in specific DataHandlers Called from: InstrumentAgent._handler_get_resource_params """ # TODO: Should raise NotImplementedError here, return temporarily for prototyping # raise NotImplementedError('get_resource_params() not implemented in BaseDataHandler') return [DataHandlerParameter.POLLING_INTERVAL] def get_resource_commands(self, *args, **kwargs): """ Return list of DataHandler execute commands available. Called from: InstrumentAgent._handler_get_resource_commands """ cmds = [cmd.replace('execute_','') for cmd in dir(self) if cmd.startswith('execute_')] return cmds def _unlock_new_data_callback(self, caller): log.debug('** Release {0}'.format(caller)) self._semaphore.release() @classmethod def _acquire_data(cls, config, unlock_new_data_callback): """ Ensures required keys (such as stream_id) are available from config, configures the publisher and then calls: BaseDataHandler._new_data_constraints (only if config does not contain 'constraints') BaseDataHandler._publish_data passing BaseDataHandler._get_data as a parameter @param config Dict containing configuration parameters, may include constraints, formatters, etc @param unlock_new_data_callback BaseDataHandler callback function to allow conditional unlocking of the BaseDataHandler._semaphore """ stream_id = get_safe(config, 'stream_id') if not stream_id: raise ConfigurationError('Configuration does not contain required \'stream_id\' key') #TODO: Configure the publisher publisher=None constraints = get_safe(config,'constraints') if not constraints: gevent.getcurrent().link(unlock_new_data_callback) constraints = cls._new_data_constraints(config) config['constraints']=constraints cls._publish_data(publisher, config, cls._get_data(config)) # Publish a 'TestFinished' event if get_safe(config,'TESTING'): log.debug('Publish TestingFinished event') pub = EventPublisher('DeviceCommonLifecycleEvent') pub.publish_event(origin='BaseDataHandler._acquire_data', description='TestingFinished') @classmethod def _new_data_constraints(cls, config): #TODO: Document what "constraints" looks like (yml)!! """ Determines the appropriate constraints for acquiring any "new data" from the external dataset The format of the constraints are documented: @param config Dict of configuration parameters - may be used to generate the returned 'constraints' dict @retval Dict that constrains retrieval of new data from the external dataset """ raise NotImplementedError @classmethod def _get_data(cls, config): """ Generator function that acquires data from a source iteratively based on constraints provided by config Passed into BaseDataHandler._publish_data and iterated to publish samples. Data should be conformant with the requirements of the publisher (granule) @param config Dict containing configuration parameters, may include constraints, formatters, etc """ raise NotImplementedError @classmethod def _publish_data(cls, publisher, config, data_generator): """ Iterates over the data_generator and publishes granules to the stream indicated in stream_id """ stream_id=config['stream_id'] log.debug('Start publishing to stream_id = {0}'.format(stream_id)) for count, ivals in enumerate(data_generator): log.info('Publish data to stream \'{0}\' [{1}]: {2}'.format(stream_id,count,ivals))
class nscl_dm_adapter(Plugin): def _init(self, ): self._initialized() def _start(self, ): self.sem = Semaphore() self.sem_counter = 0 self.set_configurations() self.api.run_task(self.create_server) self.subscribe_nscl() self.api.run_task(self.subscribe_dm_server) if self.config["enable_test"]: pass # self.api.run_task(self.send_execute_command) # Uncomment to check these operations # self.api.run_task(self.send_specific_observation) # self.api.run_task(self.send_specific_observation1) # self.api.run_task(self.send_cancel_observation) #self.api.run_task(self.send_discover_resources) #self.api.run_task(self.send_write_attributes) #self.api.run_task(self.send_create) self._started() def _stop(self, ): self.local_server.stop() self._stopped() def set_configurations(self, ): self.lwm2m_server_ip = self.config["lwm2m_dm_server_ip"] self.lwm2m_server_port = self.config["lwm2m_dm_server_port"] self.nscl_dm_adapter_listener_ip = self.config[ "nscl_dm_adapter_listener_ip"] self.nscl_dm_adapter_listener_port = self.config[ "nscl_dm_adapter_listener_port"] self.nscl_dm_adapter_client_ip = self.config[ "nscl_dm_adapter_client_ip"] self.nscl_dm_adapter_client_port = self.config[ "nscl_dm_adapter_client_port"] def create_server(self, ): self.local_server = DatagramServer( (self.nscl_dm_adapter_listener_ip, self.nscl_dm_adapter_listener_port), self.handle_request) self.local_server.start() def handle_request(self, message, remote): rx_record = connection.ReceptionRecord(None, message, remote) msg = rx_record.message uriQuery = msg.findOption(options.UriQuery) self.process(rx_record, remote, uriQuery) def process(self, rx_record, remote, uri_query): if rx_record.message.transaction_type == connection.Message.CON: if constants.POST == rx_record.message.code: if self.general_notification_token == rx_record.message.token: self.logger.info("General Notification received") msg = connection.Message(connection.Message.ACK, code=constants.CREATED) self.local_server.sendto( msg._pack(rx_record.transaction_id), remote) self.process_resources( json.loads(rx_record.message.payload)) else: self.logger.info("Specific Notification received") msg = connection.Message(connection.Message.ACK, code=constants.CREATED) self.local_server.sendto( msg._pack(rx_record.transaction_id), remote) payload = json.loads(rx_record.message.payload) observer_ip = payload["observer_ip"] observer_port = payload["observer_port"] del payload["observer_ip"] del payload["observer_port"] self.process_resources(payload, observer_ip=observer_ip, observer_port=observer_port) elif rx_record.message.transaction_type == connection.Message.NON: if self.general_notification_token == rx_record.message.token: self.logger.info("General Notification received") self.process_resources(json.loads(rx_record.message.payload)) else: self.logger.info("Specific Notification received") payload = json.loads(rx_record.message.payload) observer_ip = payload["observer_ip"] observer_port = payload["observer_port"] del payload["observer_ip"] del payload["observer_port"] self.process_resources(payload, observer_ip=observer_ip, observer_port=observer_port) def process_resources(self, payload, observer_ip=None, observer_port=None): total_resources = payload if observer_ip != None and observer_port != None: self.logger.info("The notification should be sent to %s:%s", observer_ip, observer_port) for ep_name, object_resources in total_resources.iteritems(): endpoint_name = ep_name for object_ids, resources in object_resources.iteritems(): object_id = object_ids.split("_")[0] object_inst_id = object_ids.split("_")[1] resources_dict = {} for res_ids, res_value in resources["resources"].iteritems(): res_id = res_ids.split("_")[0] res_inst_id = res_ids.split("_")[1] res_value = res_value resource_name = lwm2m_dict_objects[str( object_id)]["resource_list"][str(res_id)]["resName"] is_multi_inst = lwm2m_dict_objects[str( object_id)]["resource_list"][str(res_id)]["multiInst"] if not is_multi_inst: resources_dict.update({resource_name: res_value}) else: resources_dict.update({ resource_name + "_" + str(res_inst_id): res_value }) self.handle_m2m_server(endpoint_name, object_id, object_inst_id, res_id, res_inst_id, resource_name, res_value, resources_dict) def handle_m2m_server(self, endpoint_name, object_id, object_inst_id, res_id, res_inst_id, res_name, res_value, resources_dict): preferred_scl = endpoint_name.split("/")[0] if endpoint_name.find("attachedDevices") == -1: bool_attachedDevices = False else: attached_device_name = endpoint_name.split("/")[-1] bool_attachedDevices = True object_name = lwm2m_dict_objects[str(object_id)]["object_name"] resource_name = lwm2m_dict_objects[str(object_id)]["resource_list"][ str(res_id)]["resName"] moID_value = lwm2m_dict_objects[str(object_id)]["urn"] res_name_res_inst_id = resource_name + "_" + str(res_inst_id) def add_parameters(response): path = response.resource.path resource = ('{"mgmtObjs" : ' + json.dumps(resources_dict) + '}') request = UpdateRequestIndication(path, resource, content_type="application/json") response = self.api.handle_request_indication(request) def handle_mgmtobjs(response): mgmtobj_exists = False for mgmtobj in response.resource.mgmtObjCollection: if mgmtobj.name == object_name + "_" + str(object_inst_id): mgmtobj_exists = True path = mgmtobj.path request = RetrieveRequestIndication(path) response = self.api.handle_request_indication(request) try: if res_name_res_inst_id in response.value.resource.flex_values: if response.value.resource.flex_values[ res_name_res_inst_id] == str(res_value): continue elif res_name in response.value.resource.flex_values: if response.value.resource.flex_values[ res_name] == str(res_value): continue except: pass resource = ('{"mgmtObjs" : ' + json.dumps(resources_dict) + '}') request = UpdateRequestIndication( path, resource, content_type="application/json") response = self.api.handle_request_indication(request) break if not mgmtobj_exists: mgmtobj_ = MgmtObj(id=str(object_name) + "_" + str(object_inst_id), moID=moID_value) path = response.resource.path request = CreateRequestIndication(path, mgmtobj_) response = self.api.handle_request_indication(request) response.then(add_parameters) def retrieve_mgmtobjs(response): path = response.resource.path + "/mgmtObjs" request = RetrieveRequestIndication(path) response = self.api.handle_request_indication(request) response.then(handle_mgmtobjs) def handle_attached_devices(response): attached_device_exists = False for attached_device in response.resource.attachedDeviceCollection: if attached_device.name == attached_device_name: attached_device_exists = True path = attached_device.path + "/mgmtObjs" request = RetrieveRequestIndication(path) response = self.api.handle_request_indication(request) response.then(handle_mgmtobjs) break if not attached_device_exists: attached_device_object = AttachedDevice( id=attached_device_name) path = response.resource.path request = CreateRequestIndication( path=path, resource=attached_device_object) response = self.api.handle_request_indication(request) response.then(retrieve_mgmtobjs) def retrieve_attached_devices(response): path = response.resource.path + "/attachedDevices" request = RetrieveRequestIndication(path) response = self.api.handle_request_indication(request) response.then(handle_attached_devices) def handle_scl(response): scl_exists = False for _scl in response.resource.sclCollection: if _scl.name == preferred_scl: scl_exists = True if bool_attachedDevices: path = _scl.path + "/attachedDevices" else: path = _scl.path + "/mgmtObjs" request = RetrieveRequestIndication(path) response = self.api.handle_request_indication(request) if bool_attachedDevices: response.then(handle_attached_devices) else: response.then(handle_mgmtobjs) break if not scl_exists: scl_object = Scl(sclId=preferred_scl, link="127.0.0.1", sclType="GSCL", mgmtProtocolType="LWM2M") request = CreateRequestIndication(path="/m2m/scls", resource=scl_object) response = self.api.handle_request_indication(request) if bool_attachedDevices: response.then(retrieve_attached_devices) else: response.then(retrieve_mgmtobjs) path = "/m2m/scls" request = RetrieveRequestIndication(path) response = self.api.handle_request_indication(request) response.then(handle_scl) def _handle_mgmtcmd_created(self, instance, request_indication): pass def _handle_mgmtcmd_updated(self, instance, request_indication): pass def _handle_mgmtobj_created(self, instance, request_indication): pass def _handle_mgmtobj_updated(self, instance, request_indication): filter_keyword = "TransportMgmtPolicy" filter_keyword1 = "DeviceCapability" mgmtobj_name = instance.path.split("/")[-1] if mgmtobj_name.startswith(filter_keyword): self.handle_transport_mgmt_policy(instance, mgmtobj_name) elif mgmtobj_name.startswith(filter_keyword1): self.handle_device_capability(instance, mgmtobj_name, request_indication) def handle_device_capability(self, instance, mgmtobj_name, request_indication): generate_endpoint = instance.path.split("/")[3:-2] endpoint_name = "/".join(generate_endpoint) object_name = mgmtobj_name.split("_")[0] object_id = lwm2m_reverse_dict_objects[object_name]["object_id"] object_inst_id = mgmtobj_name.split("_")[1] if "opEnable" in request_indication.resource and "opDisable" in request_indication.resource: return elif "opEnable" in request_indication.resource: res_id = 5 res_inst_id = 0 elif "opDisable" in request_indication.resource: res_id = 6 res_inst_id = 0 else: return self.send_execute_resource(endpoint_name, object_id, object_inst_id, res_id, res_inst_id) def handle_transport_mgmt_policy(self, instance, mgmtobj_name): res_value_exists = False resources_dict = {} total_dict = {} endpoint_dict = {} generate_endpoint = instance.path.split("/")[3:-2] endpoint_name = "/".join(generate_endpoint) object_name = mgmtobj_name.split("_")[0] object_id = lwm2m_reverse_dict_objects[object_name]["object_id"] object_inst_id = mgmtobj_name.split("_")[1] for key, value in instance.flex_values.iteritems(): res_name = key.split("_")[0] try: res_inst_id = key.split("_")[1] except: res_inst_id = 0 res_value = value res_id = lwm2m_reverse_dict_objects[object_name]["resource_list"][ res_name]["resId"] resources_dict.update( {res_id: { "res_inst_id": res_inst_id, "res_value": res_value }}) if res_value != "" and not res_value_exists: res_value_exists = True if res_value_exists: self.logger.info("Sending the Resource Updates to LWM2M Server") payload = json.dumps(resources_dict) content_type = "application/json" request = lwm2m_api() self.sem.acquire() client_port = self.generate_client_port() response = request.write_resource(self.lwm2m_server_ip, self.lwm2m_server_port, endpoint_name, object_id, payload, content_type, object_inst_id=object_inst_id, client_port=client_port) self.sem.release() def generate_client_port(self, ): if self.sem_counter >= 1000: self.sem_counter = 0 self.sem_counter += 1 sem_counter = self.sem_counter client_port = self.nscl_dm_adapter_client_port + sem_counter return client_port def subscribe_dm_server(self, ): self.logger.info( "Trying to subscribe to LWM2M DM Server for General Subscription") payload = json.dumps({"listener_ip": self.nscl_dm_adapter_listener_ip, "listener_port": \ self.nscl_dm_adapter_listener_port}) content_type = "application/json" request = lwm2m_api() response = request.observe_resource( self.lwm2m_server_ip, self.lwm2m_server_port, payload=payload, content_type=content_type, client_port=self.generate_client_port()) def _handle_response(response): self.logger.info( "Successfully subscribed to LWM2M DM Server for General Subscription" ) self.general_notification_token = response.token def _handle_error(*args): self.subscribe_dm_server() response.then(_handle_response, _handle_error) def subscribe_nscl(self, ): self.events.resource_created.register_handler( self._handle_mgmtobj_created, MgmtObj) self.events.resource_updated.register_handler( self._handle_mgmtobj_updated, MgmtObj) self.events.resource_created.register_handler( self._handle_mgmtcmd_created, MgmtCmd) self.events.resource_updated.register_handler( self._handle_mgmtcmd_updated, MgmtCmd) def send_discover_resources(self, ): sleep(20) self.logger.info("Sending discover request to Dm server") server_ip = self.lwm2m_server_ip server_port = self.lwm2m_server_port payload = "/.well-known/core" request = lwm2m_api() response = request.discover_resources( server_ip, server_port, payload=payload, client_port=self.generate_client_port()) discover = Discovery() payload = json.loads(response.payload) discover.display_all_resources(payload) def send_write_attributes(self, ): sleep(10) self.logger.info("Sending attributes info to DM server") server_ip = self.lwm2m_server_ip server_port = self.lwm2m_server_port endpoint_name = "emulated_device_nb_0" object_id = 3 object_inst_id = 0 res_id = 1 res_inst_id = 0 pmax = 50 pmin = 10 gt = None lt = None st = None cancel = None content_type = "application/json" payload = json.dumps({ "pmax": pmax, "pmin": pmin, "gt": gt, "lt": lt, "st": st, "cancel": cancel }) request = lwm2m_api() response = request.write_attributes( server_ip, server_port, endpoint_name, object_id, payload, content_type, object_inst_id=object_inst_id, res_id=res_id, res_inst_id=res_inst_id, client_port=self.generate_client_port()) def send_create(self, ): sleep(10) self.logger.info("Sending create info to DM server") server_ip = self.lwm2m_server_ip server_port = self.lwm2m_server_port endpoint_name = "emulated_device_nb_0" object_id = 3 object_inst_id = 4 res_id = 0 res_inst_id = 0 res_value = "fokus" res_id_res_inst_id = str(res_id) + "_" + str(res_inst_id) payload = {} res_id_res_inst_id = str(res_id) + "_" + str(res_inst_id) payload[res_id_res_inst_id] = { "res_id": res_id, "res_inst_id": res_inst_id, "res_value": res_value } content_type = "application/json" request = lwm2m_api() response = request.create_object_instance( server_ip, server_port, endpoint_name, object_id, json.dumps(payload), content_type, object_inst_id=object_inst_id, client_port=self.generate_client_port()) def send_specific_observation(self, ): sleep(15) self.logger.info("Sending specific observation to DM server") app_ip = "localhost" app_port = "1111" server_ip = self.lwm2m_server_ip server_port = self.lwm2m_server_port endpoint_name = "gscl/attachedDevices/PulseOximeter" object_id = 4200 object_inst_id = 0 res_id = 1 res_inst_id = 0 request = lwm2m_api() response = request.observe_resource( server_ip, server_port, app_ip=app_ip, app_port=app_port, endpoint_name=endpoint_name, object_id=object_id, object_inst_id=object_inst_id, res_id=res_id, res_inst_id=res_inst_id, client_port=self.generate_client_port()) def _handle_response(response): self.logger.info("response token: %s", response.token) response.then(_handle_response) def send_specific_observation1(self, ): sleep(20) self.logger.info("Sending specific observation to DM server") app_ip = "localhost" app_port = "1115" server_ip = self.lwm2m_server_ip server_port = self.lwm2m_server_port endpoint_name = "gscl_PulseOximeter" object_id = 4200 object_inst_id = 0 res_id = 0 res_inst_id = 0 request = lwm2m_api() response = request.observe_resource( server_ip, server_port, app_ip=app_ip, app_port=app_port, endpoint_name=endpoint_name, object_id=object_id, object_inst_id=object_inst_id, res_id=res_id, res_inst_id=res_inst_id, client_port=self.generate_client_port()) def _handle_response(response): self.logger.info("response token: %s", response.token) response.then(_handle_response) def send_cancel_observation(self, ): sleep(22) self.logger.info("Sending Cancel Observation to DM server") app_ip = "localhost" app_port = "1111" server_ip = self.lwm2m_server_ip server_port = self.lwm2m_server_port endpoint_name = "gscl/attachedDevices/PulseOximeter" object_id = 4200 object_inst_id = 0 res_id = 1 res_inst_id = 0 request = lwm2m_api() response = request.cancel_observe_resource( server_ip, server_port, app_ip, app_port, endpoint_name, object_id, object_inst_id=object_inst_id, res_id=res_id, res_inst_id=res_inst_id, client_port=self.generate_client_port()) def _handle_response(response): self.logger.info("response token: %s", response.token) self.logger.info("response %s", response.payload) response.then(_handle_response) def send_execute_resource(self, endpoint_name, object_id, object_inst_id, res_id, res_inst_id, payload=None): self.logger.info("Sending execution to DM server") server_ip = self.lwm2m_server_ip server_port = self.lwm2m_server_port payload = None request = lwm2m_api() response = request.execute_resource( server_ip, server_port, endpoint_name, object_id, object_inst_id, res_id, res_inst_id=res_inst_id, payload=payload, client_port=self.generate_client_port()) self.logger.info("Updating M2M Resource Tree") resources_dict = {} object_id_res_id = str(object_id) + "/" + str(res_id) if object_id_res_id in action_mapping: res_id = action_mapping[object_id_res_id]["target_res_id"] res_value = action_mapping[object_id_res_id]["target_action"] res_name = lwm2m_dict_objects[str(object_id)]["resource_list"][str( res_id)]["resName"] is_multi_inst = lwm2m_dict_objects[str( object_id)]["resource_list"][str(res_id)]["multiInst"] if not is_multi_inst: resources_dict.update({res_name: res_value}) else: resources_dict.update( {res_name + "_" + str(res_inst_id): res_value}) self.handle_m2m_server(endpoint_name, object_id, object_inst_id, res_id, res_inst_id, res_name, res_value, resources_dict)
class SchedSite(SchedCommon): """Mirrors a site""" rain_timer = None rain_counter = 0 _run_delay = None _sched = None _sched_running = None _delay_on = None running = False def __new__(cls, s): if s.id in sites: return sites[s.id] self = object.__new__(cls) sites[s.id] = self self.s = s self.connect() self._delay_on = Semaphore() self.controllers = set() self.envgroups = set() self.meters = {} for M in METERS: ml = set() self.meters[M.meter_type] = ml for d in getattr(self.s, M.meter_type + "_meters").all(): ml.add(M(d)) self.log("Startup") self.connect_monitors(do_controllers=False) signal.signal(signal.SIGINT, self.do_shutdown) signal.signal(signal.SIGTERM, self.do_shutdown) signal.signal(signal.SIGHUP, self.do_syncsched) self.running = True return self def __init__(self, s): pass def do_shutdown(self, x, y, **k): gevent.spawn_later(0.1, connwrap, self.shutdown) def do_syncsched(self, x, y): gevent.spawn_later(0.1, connwrap, self.syncsched) def syncsched(self): print("Sync+Sched", file=sys.stderr) self.sync() self.refresh() self.run_sched_task(reason="Sync+Sched") def delay_on(self): self._delay_on.acquire() gevent.spawn_later(1, self._delay_on.release) @async_gevent def check_flow(self, **k): for c in self.controllers: c.check_flow(**k) def connect(self): d = dict() if self.s.port: d['port'] = self.s.port if self.s.username: d['login'] = self.s.username if self.s.password: d['password'] = self.s.password if self.s.virtualhost: d['virtualhost'] = self.s.virtualhost try: self.qb = qbroker.make_unit_gevent( "moat.rain.runschedule", amqp=dict(server=dict(host=self.s.host, **d))) except Exception: print("Could not connect:", self.s.host, file=sys.stderr) raise def maybe_restart(self): self.log("reconnecting") try: self.connect() except Exception: print_exc() gevent.spawn_later(100, connwrap, self.maybe_restart) else: self.connect_monitors() def connect_monitors(self, do_controllers=True): if self.qb is None: return if do_controllers: for c in self.controllers: c.connect_monitors() for mm in self.meters.values(): for m in mm: m.connect_monitors() n = self.s.var.replace(' ', '.') self.ckf = self.qb.register_rpc_gevent("rain.check.flow." + n, self.check_flow) self.cks = self.qb.register_rpc_gevent( "rain.read.schedule." + n, partial(self.run_sched_ext, reason="read schedule")) self.ckt = self.qb.register_rpc_gevent("rain.sync." + n, self.sync_ext) self.cku = self.qb.register_rpc_gevent("rain.shutdown." + n, self.do_shutdown) def sync(self, **k): print("Sync", file=sys.stderr) for c in self.controllers: c.sync() for eg in self.envgroups: eg.sync() for mm in self.meters.values(): for m in mm: m.sync() self.run_main_task() #Save(None) print("Sync end", file=sys.stderr) sync_ext = async_gevent(sync) def shutdown(self, **k): print("Shutdown", file=sys.stderr) signal.signal(signal.SIGINT, signal.SIG_DFL) signal.signal(signal.SIGTERM, signal.SIG_DFL) if self.running: self.running = False self.sync() for eg in self.envgroups: eg.sync() for c in self.controllers: c.shutdown() for mm in self.meters.values(): for m in mm: m.shutdown() #Save(None) sys.exit(0) def run_schedule(self): for c in self.controllers: c.run_schedule() def refresh(self): self.s.refresh() for eg in self.envgroups: eg.refresh() for c in self.controllers: c.refresh() for mm in self.meters.values(): for m in mm: m.refresh() def log(self, txt): log(self.s, txt) def add_controller(self, controller): self.controllers.add(controller) def no_rain(self): """Rain has stopped.""" # called by timer self.rain_timer = None self.log("Stopped raining") self.run_main_task() def has_rain(self): """Some monitor told us that it started raining""" r, self.rain_timer = self.rain_timer, gevent.spawn_later( self.s.db_rain_delay, connwrap, self.no_rain) if r: r.kill() return self.log("Started raining") self.rain = True #for v in self.s.valves.all(): vo = Valve.objects.filter(controller__site=self.s, runoff__gt=0) for v in vo.all(): valve = SchedValve(v) if valve.locked: continue try: valve._off(1) except NotConnected: pass except Exception: self.log_error(v) Schedule.objects.filter(valve__in=vo, start__gte=now() - timedelta(1), seen=False).delete() self.run_main_task() def send_command(self, *a, **k): # TODO: return a sensible error and handle that correctly if self.qb is None: raise NotConnected try: self.qb.rpc_gevent("moat.cmd", args=a, _dest="moat.main", **k) except asyncio.TimeoutError as e: print("Timeout sending %s %s" % (repr(a), repr(k))) def run_every(self, delay): """Initiate running the calculation and scheduling loop every @delay seconds.""" if self._run_delay is not None: self._run_delay = delay # just update return self._run_delay = delay self._run_last = now() self._running = Semaphore() self._run_result = None sd = self._run_delay.total_seconds() / 10 if sd < 66: sd = 66 self._run = gevent.spawn_later(sd, connwrap, self.run_main_task, kill=False) if self._sched is not None: self._sched.kill() self._sched = gevent.spawn_later(2, connwrap, self.run_sched_task, kill=False, reason="run_every") def run_main_task(self, kill=True): """Run the calculation loop.""" res = None if not self._running.acquire(blocking=False): return self._run_result.get() try: self._run_result = AsyncResult() if kill: self._run.kill() n = now() ts = (n - self._run_last).total_seconds() if ts < 5: try: res = self.s.history.order_by("-time")[0] except IndexError: return None else: return res self._run_last = n res = self.main_task() return res finally: self._run = gevent.spawn_later( (self._run_last + self._run_delay - n).total_seconds(), connwrap, self.run_main_task, kill=False) r, self._run_result = self._run_result, None self._running.release() r.set(res) def current_history_entry(self, delta=15): # assure that the last history entry is reasonably current try: he = self.s.history.order_by("-time")[0] except IndexError: pass else: if (now() - he.time).total_seconds() < delta: return he return self.new_history_entry() def new_history_entry(self, rain=0): """Create a new history entry""" values = {} n = now() for t, ml in self.meters.items(): sum_it = False sum_val = 0 sum_f = 0 for m in ml: f = m.weight v = m.get_value() if m.last_time is None: f *= 0.01 else: s = (n - m.last_time).total_seconds() if s > METER_MAXTIME: f *= 0.01 elif s > METER_TIME: f *= METER_TIME / s if v is not None: sum_val += f * v sum_f += f if m.sum_it: sum_it = True if sum_f: if not sum_it: sum_val /= sum_f values[t] = sum_val print("Values:", values, file=sys.stderr) h = History(site=self.s, time=now(), **values) h.save() return h def sync_history(self): for c in self.controllers: c.sync_history() def main_task(self): print("MainTask", file=sys.stderr) self.refresh() h = self.current_history_entry(3) self.sync_history() gevent.spawn_later(2, connwrap, self.sched_task) print("MainTask end", h, file=sys.stderr) return h def run_sched_task(self, delayed=False, reason=None, kill=True, **k): if self._sched_running is not None: print("RunSched.running", reason, file=sys.stderr) return self._sched_running.get() if self._sched is not None: if kill: self._sched.kill() if delayed: print("RunSched.delay", reason, file=sys.stderr) self._sched = gevent.spawn_later(10, connwrap, self.run_sched_task, kill=False, reason="Timer 10") return print("RunSched", reason, file=sys.stderr) self._sched = None self._sched_running = AsyncResult() try: self.sched_task() except Exception: self.log(format_exc()) finally: r, self._sched_running = self._sched_running, None if self._sched is None: self._sched = gevent.spawn_later(600, connwrap, self.run_sched_task, kill=False, reason="Timer 600") if r is not None: r.set(None) print("RunSched end", file=sys.stderr) run_sched_ext = async_gevent(run_sched_task) def sched_task(self, kill=True): self.refresh() self.run_schedule()
class Queue(Greenlet): """Manages the queue of |Envelope| objects waiting for delivery. This is not a standard FIFO queue, a message's place in the queue depends entirely on the timestamp of its next delivery attempt. :param store: Object implementing :class:`QueueStorage`. :param relay: |Relay| object used to attempt message deliveries. :param backoff: Function that, given an |Envelope| and number of delivery attempts, will return the number of seconds before the next attempt. If it returns ``None``, the message will be permanently failed. The default backoff function simply returns ``None`` and messages are never retried. :param bounce_factory: Function that produces a |Bounce| object given the same parameters as the |Bounce| constructor. If the function returns ``None``, no bounce is delivered. By default, a new |Bounce| is created in every case. :param store_pool: Number of simultaneous operations performable against the ``store`` object. Default is unlimited. :param relay_pool: Number of simultaneous operations performable against the ``relay`` object. Default is unlimited. """ def __init__(self, store, relay, backoff=None, bounce_factory=None, store_pool=None, relay_pool=None): super(Queue, self).__init__() self.store = store self.relay = relay self.backoff = backoff or self._default_backoff self.bounce_factory = bounce_factory or Bounce self.wake = Event() self.queued = [] self.queued_lock = Semaphore(1) self.queue_policies = [] self._use_pool('store_pool', store_pool) self._use_pool('relay_pool', relay_pool) def add_policy(self, policy): """Adds a |QueuePolicy| to be executed before messages are persisted to storage. :param policy: |QueuePolicy| object to execute. """ if isinstance(policy, QueuePolicy): self.queue_policies.append(policy) else: raise TypeError('Argument not a QueuePolicy.') @staticmethod def _default_backoff(envelope, attempts): pass def _run_policies(self, envelope): results = [envelope] def recurse(current, i): try: policy = self.queue_policies[i] except IndexError: return ret = policy.apply(current) if ret: results.remove(current) results.extend(ret) for env in ret: recurse(env, i+1) else: recurse(current, i+1) recurse(envelope, 0) return results def _use_pool(self, attr, pool): if pool is None: pass elif isinstance(pool, Pool): setattr(self, attr, pool) else: setattr(self, attr, Pool(pool)) def _pool_run(self, which, func, *args, **kwargs): pool = getattr(self, which+'_pool', None) if pool: ret = pool.spawn(func, *args, **kwargs) return ret.get() else: return func(*args, **kwargs) def _pool_imap(self, which, func, *iterables): pool = getattr(self, which+'_pool', gevent) threads = imap(pool.spawn, repeat(func), *iterables) ret = [] for thread in threads: thread.join() ret.append(thread.exception or thread.value) return ret def _pool_spawn(self, which, func, *args, **kwargs): pool = getattr(self, which+'_pool', gevent) return pool.spawn(func, *args, **kwargs) def _add_queued(self, entry): for i, info in enumerate(self.queued): if info[0] > entry[0]: # [0] is the timestamp. self.queued.insert(i, entry) break else: self.queued.append(entry) self.wake.set() def enqueue(self, envelope): now = time.time() envelopes = self._run_policies(envelope) ids = self._pool_imap('store', self.store.write, envelopes, repeat(now)) results = zip(envelopes, ids) for env, id in results: if not isinstance(id, BaseException): self._pool_spawn('relay', self._attempt, id, env, 0) elif not isinstance(id, QueueError): raise id # Re-raise exceptions that are not QueueError. return results def _load_all(self): for entry in self.store.load(): self._add_queued(entry) def _bounce(self, envelope, reply): bounce = self.bounce_factory(envelope, reply) if bounce: return self.enqueue(bounce) def _perm_fail(self, id, envelope, reply): self._pool_spawn('store', self.store.remove, id) if envelope.sender: # Can't bounce to null-sender. self._pool_spawn('bounce', self._bounce, envelope, reply) def _retry_later(self, id, envelope, reply): attempts = self.store.increment_attempts(id) wait = self.backoff(envelope, attempts) if wait is None: reply.message += ' (Too many retries)' self._perm_fail(id, envelope, reply) else: when = time.time() + wait self.store.set_timestamp(id, when) self._add_queued((when, id)) def _attempt(self, id, envelope, attempts): try: self.relay._attempt(envelope, attempts) except TransientRelayError as e: self._pool_spawn('store', self._retry_later, id, envelope, e.reply) except PermanentRelayError as e: self._perm_fail(id, envelope, e.reply) else: self._pool_spawn('store', self.store.remove, id) def _dequeue(self, id): envelope, attempts = self.store.get(id) self._pool_spawn('relay', self._attempt, id, envelope, attempts) def _check_ready(self, now): last_i = 0 for i, entry in enumerate(self.queued): timestamp, id = entry if now >= timestamp: self._pool_spawn('store', self._dequeue, id) last_i = i+1 else: break if last_i > 0: self.queued = self.queued[last_i:] def _wait_ready(self, now): try: first = self.queued[0] except IndexError: self.wake.wait() self.wake.clear() return first_timestamp = first[0] if first_timestamp > now: self.wake.wait(first_timestamp-now) self.wake.clear() def flush(self): """Attempts to immediately flush all messages waiting in the queue, regardless of their retry timers. ***Note:*** This can be a very expensive operation, use with care. """ self.wake.set() self.wake.clear() self.queued_lock.acquire() try: for entry in self.queued: self._pool_spawn('store', self._dequeue, entry[1]) self.queued = [] finally: self.queued_lock.release() def _run(self): self._pool_spawn('store', self._load_all) while True: self.queued_lock.acquire() try: now = time.time() self._check_ready(now) self._wait_ready(now) finally: self.queued_lock.release()
class SchedValve(SchedCommon): """Mirrors (and monitors) a valve.""" locked = False # external command, don't change sched = None sched_ts = None sched_job = None sched_lock = None on = False on_ts = None flow = 0 _flow_check = None def __new__(cls, v): if v.id in valves: return valves[v.id] self = object.__new__(cls) valves[v.id] = self self.v = v self.site = SchedSite(self.v.controller.site) self.env = EnvGroup(self.v.envgroup) self.controller = SchedController(self.v.controller) self.sched_lock = Semaphore() if self.site.qb: try: self.site.send_command("set", "output", "off", *(self.v.var.split())) except NotConnected: pass except Exception as e: raise RuntimeError(self.v.var) from e return self def __init__(self, v): pass def _on(self, caller, sched=None, duration=None): print("Open", caller, self.v.var, file=sys.stderr) self.site.delay_on() if duration is None and sched is not None: duration = sched.duration if self.controller.has_max_on(): print("… but too many:", ", ".join( str(v) for v in self.controller.c.valves.all() if SchedValve(v).on), file=sys.stderr) if sched: sched.update(seen=False) self.log("NOT running %s for %s: too many" % ( self.v, duration, )) raise TooManyOn(self) if duration is None: self.log("Run (indefinitely)") self.site.send_command("set", "output", "on", *(self.v.var.split())) else: self.log("Run for %s" % (duration, )) if not isinstance(duration, six.integer_types): duration = duration.total_seconds() try: self.site.send_command("set", "output", "on", *(self.v.var.split()), sub=(("for", duration), ("async", ))) except Exception: # Something broke. Try to turn this thing off. self.log(format_exc()) self.site.send_command("set", "output", "off", *(self.v.var.split())) raise RuntimeError("Could not start (logged)") if sched is not None: if self.v.verbose: self.log("Opened for %s" % (sched, )) self.sched = sched if not sched.seen: sched.update(start=now(), seen=True) sched.refresh() #Save(sched) else: if self.v.verbose: self.log("Opened for %s" % (duration, )) def _off(self, num): if self.on: if self.v.verbose: self.log("Closing " + str(num)) print("Close", self.v.var, file=sys.stderr) try: self.site.send_command("set", "output", "off", *(self.v.var.split())) except NotConnected: pass def shutdown(self): if self._flow_check is not None: self._flow_check.dead() def run_schedule(self): if not self.sched_lock.acquire(blocking=False): if self.v.verbose: print("SCHED LOCKED1 %s" % (self.v.name, ), file=sys.stderr) return try: self._run_schedule() except Exception: self.log(format_exc()) finally: self.sched_lock.release() def _run_schedule(self): if self.sched_job is not None: self.sched_job.kill() self.sched_job = None if self.locked: if self.v.verbose: print("SCHED LOCKED2 %s" % (self.v.name, ), file=sys.stderr) return n = now() try: if self.sched is not None: self.sched.refresh() if self.sched.end <= n: if self.v.verbose: print("Turn off: %s+%s <= %s" % (self.sched.start, self.sched.duration, n), file=sys.stderr) self._off(2) self.sched = None else: self.sched_job = gevent.spawn_later( (self.sched.end - n).total_seconds(), connwrap, self.run_sched_task, reason="_run_schedule 1") if self.v.verbose: print("SCHED LATER %s: %s" % (self.v.name, humandelta(self.sched.end - n)), file=sys.stderr) return except ObjectDoesNotExist: pass # somebody deleted it *shrug* sched = None if self.sched_ts is None: try: sched = self.v.schedules.filter( start__lt=n).order_by("-start")[0] except IndexError: self.sched_ts = n - timedelta(1, 0) else: self.sched_ts = sched.end if sched.end > n: # still running if self.v.verbose: print("SCHED RUNNING %s: %s" % (self.v.name, humandelta(sched.end - n)), file=sys.stderr) try: self._on(1, sched, sched.end - n) except TooManyOn: self.log("Could not schedule: too many open valves") except NotConnected: self.log( "Could not schedule: connection to MoaT failed") return try: sched = self.v.schedules.filter( start__gte=self.sched_ts).order_by("start")[0] except IndexError: if self.v.verbose: print("SCHED EMPTY %s: %s" % (self.v.name, str_tz(self.sched_ts)), file=sys.stderr) self._off(3) return if sched.end <= n: if self.v.verbose: print("SCHED %s: sched %d done for %s" % (self.v.name, sched.id, humandelta(n - sched.end)), file=sys.stderr) self.sched_ts = None return if sched.start > n: if self.v.verbose: print("SCHED %s: sched %d in %s" % (self.v.name, sched.id, humandelta(sched.start - n)), file=sys.stderr) self._off(4) self.sched_job = gevent.spawn_later( (sched.start - n).total_seconds(), connwrap, self.run_sched_task, reason="_run_schedule 2") return try: self._on(2, sched) except TooManyOn: self.log("Could not schedule: too many open valves") except NotConnected: self.log("Could not schedule: connection to MoaT failed") def run_sched_task(self, reason="valve"): self.sched_job = None self.site.run_sched_task(reason=reason) run_sched_ext = async_gevent(run_sched_task) def add_flow(self, val): if self._flow_check is not None: if self._flow_check.add_flow(val): return if self.v.verbose: print("FLOW %s: %s %s" % (self.v.name, self.flow, val), file=sys.stderr) self.flow += val @async_gevent def check_flow(self, **k): cf = None try: cf = FlowCheck(self) cf.run() except Exception as ex: log_error(self.v) if cf is not None: cf._unlock() def refresh(self): self.v.refresh() # if self.sched is not None: # self.sched.refresh() def connect_monitors(self): if self.site.qb is None: return n = self.v.var.replace(' ', '.') self.mon = self.site.qb.register_alert_gevent( "moat.event.output.change." + n, self.watch_state, call_conv=CC_DICT) self.ckf = self.site.qb.register_rpc_gevent("rain.check.flow." + n, self.check_flow, call_conv=CC_DICT) @async_gevent def watch_state(self, value=None, **kv): """output change NAME ::value ON""" on = (str(value).lower() in ("1", "true", "on")) if self._flow_check is not None: # TODO self.on = on self._flow_check.state(on) return if self.locked: self.on = on return try: if on != self.on: n = now() print("Report %s" % ("ON" if on else "OFF"), self.v.var, self.sched, file=sys.stderr) if self.sched is not None and not on: self.sched.update( db_duration=(n - self.sched.start).total_seconds()) self.sched.refresh() self.sched_ts = self.sched.end self.sched = None flow, self.flow = self.flow, 0 # If nothing happened, calculate. if not on: duration = n - self.on_ts maxflow = self.v.flow * duration.total_seconds() if (not flow or not self.v.feed.var) or flow > 2 * maxflow: flow = maxflow self.new_level_entry(flow) if not on: if self.v.level > self.v.stop_level + ( self.v.start_level - self.v.stop_level) / 5: self.v.update(priority=True) self.log("Done for %s, level is now %s" % (duration, self.v.level)) self.on = on self.on_ts = n except Exception: print_exc() def sync(self): flow, self.flow = self.flow, 0 self.new_level_entry(flow) def sync_history(self): n = now() try: lv = self.v.levels.order_by("-time")[0] except IndexError: pass else: if self.v.time > lv.time: self.log("Timestamp downdate: %s %s" % (self.v.time, lv.time)) self.v.update(time=lv.time) self.v.refresh() #Save(self.v) if (n - self.v.time).total_seconds() >= 295: flow, self.flow = self.flow, 0 self.new_level_entry(flow) def new_level_entry(self, flow=0): self.site.current_history_entry() n = now() self.v.refresh() hts = None try: lv = self.v.levels.order_by("-time")[0] except IndexError: ts = n - timedelta(1, 0) else: ts = lv.time sum_f = 0 sum_r = 0 for h in self.site.s.history.filter(time__gt=ts).order_by("time"): if self.v.verbose > 2: self.log("Env factor for %s: T=%s W=%s S=%s" % (h, h.temp, h.wind, h.sun)) f = self.env.env_factor( h, logger=self.log if self.v.verbose > 2 else None) * self.v.adj if self.v.verbose > 1: self.log("Env factor for %s is %s" % (h, f)) sum_f += self.site.s.db_rate * self.v.do_shade( self.env.eg.factor * f) * (h.time - ts).total_seconds() sum_r += self.v.runoff * h.rain ts = h.time if self.v.verbose: self.log("Apply env %f, rain %r,, flow %f = %f" % (sum_f, sum_r, flow, flow / self.v.area)) if self.v.time == ts: return if self.v.level < 0: level = 0 else: level = F('level') level += sum_f if (flow > 0 or sum_r > 0) and self.v.level > self.v.max_level: level = self.v.max_level level -= flow / self.v.area + sum_r #if level < 0: # self.log("Level %s ?!?"%(self.v.level,)) self.v.update(time=ts, level=level) self.v.refresh() lv = Level(valve=self.v, time=ts, level=self.v.level, flow=flow) lv.save() if self.on and not (self.sched and self.sched.forced ) and self.v.level <= self.v.stop_level: self._off(5) def log(self, txt): log(self.v, txt)
class M2M_Device_Gateway_DM_Adapter(LoggerMixin): def __init__(self, events, config, api, client): self.events = events self.config = config self.api = api self.client = client self.clients_collection = {} self.gscl_collection = {} self.sem = Semaphore() self.sem_counter = 0 self.events.resource_created.register_handler(self._handle_scl_created, Scl) self.events.resource_created.register_handler( self._handle_mgmtobj_created, MgmtObj) self.events.resource_created.register_handler( self._handle_mgmtcmd_created, MgmtCmd) self.events.resource_updated.register_handler( self._handle_mgmtobj_updated, MgmtObj) self.events.resource_created.register_handler( self._handle_attached_device_created, AttachedDevice) def _handle_scl_created(self, instance, request_indication): gscl_name = instance.path.split("/")[-1] if gscl_name != "nscl": self.create_scl_client(gscl_name) def create_scl_client(self, gscl_name): self.sem.acquire() self.sem_counter += 1 sem_counter = self.sem_counter self.sem.release() local_listener_port = self.config["local_listener_port"] + sem_counter local_client_port = self.config["local_client_port"] + sem_counter dm_client = self.client.create_client(local_listener_port, local_client_port) if dm_client is not None: self.gscl_collection[gscl_name] = dm_client dm_client.load_dm_adapter(self) self.client.do_registration(gscl_name) else: self.client.do_registration(gscl_name, status=True, cb=self.load_elements) def load_elements(self, endpoint_name, dm_client_object): self.gscl_collection[endpoint_name] = dm_client_object dm_client_object.load_dm_adapter(self) def load_elements1(self, endpoint_name, dm_client_object): self.clients_collection[ endpoint_name] = dm_client_object #self.gscl_collection[instance.path.split("/")[3]] def _handle_attached_device_created(self, instance, request_indication): #self.sem.acquire() self.sem_counter += 1 sem_counter = self.sem_counter #self.sem.release() generate_endpoint = instance.path.split("/")[3:] endpoint_name = "/".join(generate_endpoint) local_client_port = self.config["local_client_port"] + sem_counter try: self.clients_collection[endpoint_name] = self.gscl_collection[ instance.path.split("/")[3]] self.client.do_registration(endpoint_name, local_client_port=local_client_port) except KeyError: self.client.do_registration(endpoint_name, local_client_port=local_client_port, status=True, cb=self.load_elements1) def _handle_mgmtobj(self, dm_client, endpoint_name, instance, request_indication): send_update_flag = False object_and_resources = {} resources = {} path = instance.path resource = request_indication.resource #get object name and resource name mgmt_obj_name = path.split("/")[-1] self.logger.debug("endpoint_name: %s", endpoint_name) self.logger.debug("management object %s was added" % (mgmt_obj_name)) lwm2m_mgmt_obj = lwm2m_reverse_dict_objects[mgmt_obj_name.split("_") [0]] lwm2m_mgmt_obj_inst_id = mgmt_obj_name.split("_")[1] if lwm2m_mgmt_obj is not None: lwm2m_mgmt_obj_id = lwm2m_mgmt_obj["object_id"] self.logger.debug("flex values are " + str(instance.flex_values)) for param_name, param_value in instance.flex_values.items(): self.logger.debug("param name [%s] and param value [%s]" % (param_name, param_value)) lwm2m_resource_id = lwm2m_mgmt_obj["resource_list"][ param_name.split("_")[0]]["resId"] if lwm2m_mgmt_obj["resource_list"][param_name.split("_") [0]]["multiInst"]: lwm2m_resource_id_inst = param_name.split("_")[1] else: lwm2m_resource_id_inst = "" resource_change_flag = dm_client.add_resource(endpoint_name, lwm2m_mgmt_obj_id, \ lwm2m_resource_id, param_value, lwm2m_mgmt_obj_inst_id=lwm2m_mgmt_obj_inst_id) if resource_change_flag: send_update_flag = True resources.update({ lwm2m_resource_id: { "res_inst_id": lwm2m_resource_id_inst, "res_value": param_value } }) object_and_resources = { str(lwm2m_mgmt_obj_id) + "_" + str(lwm2m_mgmt_obj_inst_id): { "resources": resources } } mgmt_obj_id_inst_id = str(lwm2m_mgmt_obj_id) + "_" + str( lwm2m_mgmt_obj_inst_id) return mgmt_obj_id_inst_id, object_and_resources, send_update_flag else: self.logger.error( "could not find management object for management object name %s" % (mgmt_obj_name)) return None def create_mgmtobj_dict(self, endpoint_name, instance, request_indication): send_update_flag = False object_and_resources = {} resources = {} path = instance.path resource = request_indication.resource mgmt_obj_name = path.split("/")[-1] lwm2m_mgmt_obj = lwm2m_reverse_dict_objects[mgmt_obj_name.split("_") [0]] lwm2m_mgmt_obj_inst_id = mgmt_obj_name.split("_")[1] if lwm2m_mgmt_obj is not None: lwm2m_mgmt_obj_id = lwm2m_mgmt_obj["object_id"] for param_name, param_value in instance.flex_values.items(): lwm2m_resource_id = lwm2m_mgmt_obj["resource_list"][ param_name.split("_")[0]]["resId"] if lwm2m_mgmt_obj["resource_list"][param_name.split("_") [0]]["multiInst"]: lwm2m_resource_id_inst = param_name.split("_")[1] else: lwm2m_resource_id_inst = "" resources.update({ lwm2m_resource_id: { "res_inst_id": lwm2m_resource_id_inst, "res_value": param_value } }) object_and_resources = { str(lwm2m_mgmt_obj_id) + "_" + str(lwm2m_mgmt_obj_inst_id): { "resources": resources } } mgmt_obj_id_inst_id = str(lwm2m_mgmt_obj_id) + "_" + str( lwm2m_mgmt_obj_inst_id) return mgmt_obj_id_inst_id, object_and_resources else: self.logger.error("Could not find management object name %s" % (mgmt_obj_name)) return None def _handle_mgmtobj_created(self, instance, request_indication): generate_endpoint = instance.path.split("/")[3:-2] endpoint_name = "/".join(generate_endpoint) if endpoint_name == "": endpoint_name = self.config["global"]["scl_id"] if not self.gscl_collection.has_key(endpoint_name): if not any(self.gscl_collection): self.create_scl_client(endpoint_name) else: for _, obj in self.gscl_collection.iteritems(): self.gscl_collection[endpoint_name] = obj self.sem_counter += 1 sem_counter = self.sem_counter local_client_port = self.config[ "local_client_port"] + sem_counter self.client.do_registration( endpoint_name, local_client_port=local_client_port) break try: dm_client = self.clients_collection[endpoint_name] except KeyError: try: dm_client = self.gscl_collection[endpoint_name] except KeyError: self.logger.error( "Error while creating mgmtObj!! Endpoint mayn't be registered. Storing MgmtObj contents" ) self.logger.debug("%s, %s", instance.path, endpoint_name) mgmt_obj_id_inst_id, object_and_resources = self.create_mgmtobj_dict( endpoint_name, instance, request_indication) self.client.store_mgmtobj(endpoint_name, mgmt_obj_id_inst_id, object_and_resources) return mgmt_obj_id_inst_id, object_and_resources, send_update_flag = self._handle_mgmtobj( dm_client, endpoint_name, instance, request_indication) if object_and_resources is not None: self.logger.info( "Sending add resource updates from gateway(create) to client") response = dm_client.send_add_resources(object_and_resources, endpoint_name, mgmt_obj_id_inst_id) else: self.logger.info( "Not sending add resource updates from gateway(create) to client" ) def _handle_mgmtcmd_created(self, instance, request_indication): path = instance.path return def _handle_mgmtobj_updated(self, instance, request_indication): generate_endpoint = instance.path.split("/")[3:-2] endpoint_name = "/".join(generate_endpoint) if endpoint_name == "": endpoint_name = self.config["global"]["scl_id"] try: dm_client = self.clients_collection[endpoint_name] except KeyError: try: dm_client = self.gscl_collection[endpoint_name] except KeyError: self.logger.error( "Error while updating mgmtObj!! Endpoint mayn't be registered. Storing MgmtObj contents" ) self.logger.debug("%s, %s", instance.path, endpoint_name) mgmt_obj_id_inst_id, object_and_resources = self.create_mgmtobj_dict( endpoint_name, instance, request_indication) self.client.store_mgmtobj(endpoint_name, mgmt_obj_id_inst_id, object_and_resources) return mgmt_obj_id_inst_id, object_and_resources, send_update_flag = self._handle_mgmtobj( dm_client, endpoint_name, instance, request_indication) if object_and_resources is not None and send_update_flag: self.logger.info( "Sending add resource updates from gateway(update) to client") response = dm_client.send_add_resources(object_and_resources, endpoint_name, mgmt_obj_id_inst_id) else: self.logger.info( "Not sending add resource updates from gateway(udpate) to client!! Already has updated information" ) def update_resources(self, endpoint_name, object_id, object_inst_id, payload, content_type=None): """ Push the resouces change/update on the gateway resource tree """ self.logger.info("Updating the Gateway Resource Tree") resource_dict = {} resources = loads(payload) object_id_object_inst_id = str(object_id) + "_" + str(object_inst_id) object_name = lwm2m_dict_objects[str(object_id)]["object_name"] object_name_obj_inst_id = object_name + "_" + str(object_inst_id) for res_id, res in resources.iteritems(): res_name = lwm2m_dict_objects[str(object_id)]["resource_list"][str( res_id)]["resName"] if lwm2m_dict_objects[str(object_id)]["resource_list"][str( res_id)]["multiInst"]: res_name_res_inst_id = res_name + "_" + res["res_inst_id"] else: res_name_res_inst_id = res_name res_value = res["res_value"] resource_dict.update({res_name_res_inst_id: res_value}) def success(result): self.logger.info("Resource Tree is updated") def failure(result): self.logger.error("Error occurred: %s", result) def error_handling(result): path = "/m2m/scls/mgmtObjs/" + object_name_obj_inst_id self.logger.warning("Path not found. %s", result) self.logger.info("Looking for other path at %s", path) resource = ('{"mgmtObjs" : ' + dumps(resource_dict) + '}') request = UpdateRequestIndication(path, resource, content_type="application/json") response = self.api.handle_request_indication(request) response.then(success, failure) path = "/m2m/scls/" + endpoint_name + "/mgmtObjs/" + object_name_obj_inst_id resource = ('{"mgmtObjs" : ' + dumps(resource_dict) + '}') request = UpdateRequestIndication(path, resource, content_type="application/json") response = self.api.handle_request_indication(request) response.then(success, error_handling)
class KasayaLocalClient(Sender): """ KasayaLocalClient is communication class to talk with kasaya daemon. It's used by workers and clients. """ __metaclass__ = SingletonCreator def __init__(self, *args, **kwargs): # connect to kasaya super(KasayaLocalClient, self).__init__( 'tcp://127.0.0.1:' + str(settings.KASAYAD_CONTROL_PORT), *args, **kwargs) self.SEMA = Semaphore() # worker methods def setup(self, servicename, address, ID, pid): self.srvname = servicename self.ID = ID self.__pingmsg = { "message": messages.WORKER_LIVE, "addr": address, "id": self.ID, "service": servicename, "pid": pid, "status": 0, } def notify_worker_live(self, status): self.SEMA.acquire() self.__pingmsg['status'] = status try: self.send(self.__pingmsg) return True except ConnectionClosed: return False finally: self.SEMA.release() def notify_worker_stop(self): self.SEMA.acquire() msg = { "message": messages.WORKER_LEAVE, "id": self.ID, } try: self.send(msg) return True except ConnectionClosed: return False finally: self.SEMA.release() # client methods def query(self, service): """ odpytuje lokalny serwer kasaya o to gdzie realizowany jest serwis o żądanej nazwie """ msg = {'message': messages.QUERY, 'service': service} self.SEMA.acquire() try: return self.send_and_receive(msg) finally: self.SEMA.release() def control_task(self, msg): """ zadanie tego typu jest wysyłane do serwera kasayad nie do workera! """ self.SEMA.acquire() try: return self.send_and_receive(msg) finally: self.SEMA.release()