class Observable(object): terminator = None def __init__(self): self.done = Channel() def wait_eq(self, terminator, message=None, timeout=None): if self.value == terminator: return self.value self.terminator = terminator try: with_timeout(timeout, self.done.get) return self.value except Timeout: eq_(self.value, terminator, message) def _check(self): if self.value == self.terminator: self.done.put(None) def __eq__(self, x): return self.value == x def __repr__(self): return "%s(%r)" % (type(self).__name__, self.value)
def _test_atomic(): # NOTE: Nested context by comma is not available in Python 2.6. # o -- No gevent. with lets.atomic(): 1 + 2 + 3 # x -- gevent.sleep() with pytest.raises(AssertionError): with lets.atomic(): gevent.sleep(0.1) # x -- gevent.sleep() with 0 seconds. with pytest.raises(AssertionError): with lets.atomic(): gevent.sleep(0) # o -- Greenlet.spawn() with lets.atomic(): gevent.spawn(gevent.sleep, 0.1) # x -- Greenlet.join() with pytest.raises(AssertionError): with lets.atomic(): g = gevent.spawn(gevent.sleep, 0.1) g.join() # x -- Greenlet.get() with pytest.raises(AssertionError): with lets.atomic(): g = gevent.spawn(gevent.sleep, 0.1) g.get() # x -- gevent.joinall() with pytest.raises(AssertionError): with lets.atomic(): g = gevent.spawn(gevent.sleep, 0.1) gevent.joinall([g]) # o -- Event.set(), AsyncResult.set() with lets.atomic(): Event().set() AsyncResult().set() # x -- Event.wait() with pytest.raises(AssertionError): with lets.atomic(): Event().wait() # x -- Event.wait() with pytest.raises(AssertionError): with lets.atomic(): AsyncResult().wait() # x -- Channel.put() with pytest.raises(AssertionError): with lets.atomic(): ch = Channel() ch.put(123) # o -- First Semaphore.acquire() with lets.atomic(): lock = Semaphore() lock.acquire() # x -- Second Semaphore.acquire() with pytest.raises(AssertionError): with lets.atomic(): lock = Semaphore() lock.acquire() lock.acquire() # Back to normal. gevent.sleep(1)
def try_write_channel(channel: Channel, item: Any, timeout: int) -> bool: try: if (timeout == 0): item = channel.put(item, block=False) elif (timeout > 0): item = channel.put(item, block=True, timeout=timeout) else: item = channel.put(item, block=True) return True except QueueFullException: return False
def test_gevent1(): global CH import gevent print gevent, gevent.__version__ from gevent.queue import Channel from gevent import spawn, wait CH = Channel() start = time() spawn(channel_get_put) CH.put(True) spawn(channel_get_put) wait() return time() - start
class Server(Endpoint, Greenlet): ''' Used at client side, to represent server ''' def __init__(self, sock, addr): Endpoint.__init__(self, sock, addr) Greenlet.__init__(self) self.ctlcmds = Channel() self.userid = 0 self.gamedata = Gamedata(recording=True) def _run(self): while True: cmd, data = self.read() if cmd == 'gamedata': self.gamedata.feed(data) else: self.ctlcmds.put([cmd, data]) def gexpect(self, tag, blocking=True): return self.gamedata.gexpect(tag, blocking) def gbreak(self): return self.gamedata.gbreak() def gclear(self): self.gamedata = Gamedata(recording=True) def gwrite(self, tag, data): log.debug('GAME_WRITE: %s', repr([tag, data])) encoded = self.encode(['gamedata', [tag, data]]) self.raw_write(encoded) def wait_till_live(self): self.gamedata.wait_empty() def gamedata_piled(self): return len(self.gamedata.gdqueue) > 60 def shutdown(self): self.kill() self.join() self.ctlcmds.put(['shutdown', None]) self.close()
class tokizer(Jobber): def __init__(self, output, parent=None, endput=None): self._output = output self._endput = endput self.parent = parent self.q = Channel() self.init() self.start_job("job",self._job) self.job.link(self._end) def init(self): self.lnum = self.parenlev = self.continued = 0 self.contstr, self.needcont = '', 0 self.contline = None self.indents = [0] def exit(self): self.stop_job("job") def output(self,*a): log("token",TRACE,repr(a)) self._output(*a) def feed_end(self): for indent in self.indents[1:]: # pop remaining indent levels self.output(DEDENT, '', (self.lnum, 0), (self.lnum, 0), '') def feed(self,line): if self.q: if line is None: q,self.q = self.q,None if gevent.getcurrent() is not self.job: q.put(None) else: self.q.put(line) else: raise StopParsing def _end(self, res): self.feed_end() if self.q: q,self.q = self.q,None try: q.get_nowait() except Empty: pass if self._endput: self._endput() def _job(self): line = "x" while line: if self.q: line = self.q.get() else: line = None self._do_line(line) def _do_line(self,line): try: try: if line is None: raise StopIteration log("token",TRACE,"IN",line) except StopIteration: line = '' log("token",TRACE,"IN_END") self.lnum = self.lnum + 1 pos, max = 0, len(line) if self.contstr: # continued string if not line: raise TokenError, ("EOF in multi-line string", strstart) endmatch = endprog.match(line) if endmatch: pos = end = endmatch.end(0) self.output(STRING, self.contstr + line[:end], strstart, (self.lnum, end), self.contline + line) self.contstr, self.needcont = '', 0 self.contline = None elif self.needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n': self.output(ERRORTOKEN, self.contstr + line, strstart, (self.lnum, len(line)), self.contline) self.contstr = '' self.contline = None return else: self.contstr += line self.contline += line return elif self.parenlev == 0 and not self.continued: # new statement if not line: self.feed_end() self.output(ENDMARKER, '', (self.lnum, 0), (self.lnum, 0), '') return column = 0 while pos < max: # measure leading whitespace if line[pos] == ' ': column = column + 1 elif line[pos] == '\t': column = (column/tabsize + 1)*tabsize elif line[pos] == '\f': column = 0 else: break pos = pos + 1 if pos == max: self.feed_end() return if line[pos] in '#\r\n': # skip comments or blank lines self.output((NL, COMMENT)[line[pos] == '#'], line[pos:], (self.lnum, pos), (self.lnum, len(line)), line) return if column > self.indents[-1]: # count indents or dedents self.indents.append(column) self.output(INDENT, line[:pos], (self.lnum, 0), (self.lnum, pos), line) if column < self.indents[-1]: while column < self.indents[-1]: self.indents.pop() self.output(DEDENT, '', (self.lnum, pos), (self.lnum, pos), line) if column != self.indents[-1]: raise IndentationError( "unindent does not match any outer indentation level", (u"‹tokenize›", self.lnum, pos, line)) else: # continued statement if not line: raise TokenError, ("EOF in multi-line statement", (self.lnum, 0)) self.continued = 0 while pos < max: pseudomatch = pseudoprog.match(line, pos) if pseudomatch: # scan for tokens start, end = pseudomatch.span(1) spos, epos, pos = (self.lnum, start), (self.lnum, end), end token, initial = line[start:end], line[start] if num.match(token) or \ (initial == '.' and token != '.'): # ordinary number self.output(NUMBER, token, spos, epos, line) elif initial in '\r\n': self.output(NL if self.parenlev > 0 else NEWLINE, token, spos, epos, line) elif initial == '#': self.output(COMMENT, token, spos, epos, line) elif token in triple_quoted: endprog = endprogs[token] endmatch = endprog.match(line, pos) if endmatch: # all on one line pos = endmatch.end(0) token = line[start:pos] self.output(STRING, token, spos, (self.lnum, pos), line) else: strstart = (self.lnum, start) # multiple lines self.contstr = line[start:] self.contline = line break elif initial in single_quoted or \ token[:2] in single_quoted or \ token[:3] in single_quoted: if token[-1] == '\n': # continued string strstart = (self.lnum, start) endprog = (endprogs[initial] or endprogs[token[1]] or endprogs[token[2]]) self.contstr, self.needcont = line[start:], 1 self.contline = line break else: # ordinary string self.output(STRING, token, spos, epos, line) elif namestart.match(initial): # ordinary name self.output(NAME, token, spos, epos, line) elif initial == '\\': # continued stmt self.continued = 1 else: if initial in '([{': self.parenlev = self.parenlev + 1 elif initial in ')]}': self.parenlev = self.parenlev - 1 self.output(OP, token, spos, epos, line) else: self.output(ERRORTOKEN, line[pos], (self.lnum, pos), (self.lnum, pos+1), line) pos = pos + 1 except StopParsing as e: fix_exception(e) self.q = None if self.parent: self.parent.kill(e) return
class VirtualMachine(object): def __init__(self, context): self.context = context self.id = None self.name = None self.nmdm = None self.state = VirtualMachineState.STOPPED self.config = None self.bhyve_process = None self.scrollback = io.BytesIO() self.console_fd = None self.console_channel = Channel() self.console_thread = None self.logger = logging.getLogger('VM:{0}'.format(self.name)) def get_nmdm(self): #for i in range(0, 255): # if os.path.exists('/dev/nmdm{0}A'.format(i)): # continue # # a = '/dev/nmdm{0}A'.format(i) # b = '/dev/nmdm{0}B'.format(i) # self.logger.info('Assigned nmdm device pair: {0}, {1}'.format(a, b)) # return a, b return '/dev/nmdm1A', '/dev/nmdm1B' def start(self): self.nmdm = self.get_nmdm() gevent.spawn(self.run) self.console_thread = gevent.spawn(self.console_worker) def stop(self): if self.state == VirtualMachineState.STOPPED: raise RuntimeError() gevent.kill(self.console_thread) self.bhyve_process.terminate() subprocess.call('/usr/sbin/bhyvectl --destroy --vm={0}'.format(self.name)) self.set_state(VirtualMachineState.STOPPED) def set_state(self, state): self.state = state self.context.client.emit_event('container.changed', { 'operation': 'update', 'ids': [self.id] }) def run(self): self.set_state(VirtualMachineState.BOOTLOADER) if self.config['bootloader'] == 'GRUB': self.bhyve_process = subprocess.Popen( ['/usr/local/lib/grub-bhyve'] ) if self.config['bootloader'] == 'BHYVELOAD': self.bhyve_process = subprocess.Popen( [ '/usr/sbin/bhyveload', '-c', self.nmdm[0], '-m', self.config['memsize'], '-d', self.config['bootdisk'], self.name, ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True ) out, err = self.bhyve_process.communicate() self.logger.debug('bhyveload: {0}'.format(out)) args = [ '/usr/sbin/bhyve', '-A', '-H', '-P', '-c', str(self.config['cpus']), '-m', self.config['memsize'], '-s', '0:0,hostbridge', '-s', '2:0,ahci-cd,{0}'.format(self.config['bootdisk']), '-s', '31,lpc', '-l', 'com1,{0}'.format(self.nmdm[0]), self.name ] self.set_state(VirtualMachineState.RUNNING) self.bhyve_process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) out, err = self.bhyve_process.communicate() self.logger.debug('bhyve: {0}'.format(out)) def console_worker(self): BUFSIZE = 1024 self.console_fd = open(self.nmdm[1], 'r+b') tty.setraw(self.console_fd.fileno()) while True: data = gevent.os.tp_read(self.console_fd.fileno(), BUFSIZE) self.logger.debug('Read: {0}'.format(data)) if not data: self.logger.info('Reopening {0} device'.format(self.nmdm[1])) self.console_fd.close() gevent.sleep(1) self.console_fd = open(self.nmdm[1], 'r+b') tty.setraw(self.console_fd.fileno()) continue self.scrollback.write(data) try: self.console_channel.put(data, block=False) except: pass def console_write(self, data): self.logger.debug('Write: {0}'.format(data)) self.console_fd.write(data) self.console_fd.flush()
class Waiter(Collected,Jobber): """This is the thing that waits.""" force = False storage = Waiters.storage _plinger = None _running = False q = None def __init__(self,parent,name,force): self.ctx = parent.ctx self.start = now() self.force = force try: self.parent = parent.parent except AttributeError: pass super(Waiter,self).__init__(name) def list(self): end=now()+dt.timedelta(0,self.value) yield super(Waiter,self) yield("start",self.start) yield("end",end) yield("total", humandelta(end-self.start)) w = self while True: w = getattr(w,"parent",None) if w is None: break n = getattr(w,"displayname",None) if n is not None: if not isinstance(n,basestring): n = " ".join(unicode(x) for x in n) else: try: if w.args: n = unicode(w.args) except AttributeError: pass if n is None: try: if isinstance(w.name,basestring): n = w.name else: n = " ".join(unicode(x) for x in w.name) except AttributeError: n = w.__class__.__name__ if n is not None: yield("in",n) def info(self): return str(self.value) def __repr__(self): return u"‹%s %s %s›" % (self.__class__.__name__, self.name,self.value) def _pling(self): self._plinger = None self._cmd("timeout") def _set_pling(self): timeout = unixtime(self.end) - unixtime(now(self.force)) if timeout <= 0.1: timeout = 0.1 if self._plinger: self._plinger.cancel() self._plinger = callLater(self.force, timeout, self._pling) def _job(self): try: self._set_pling() self._running = True while True: cmd = self.q.get() q = cmd[0] a = cmd[2] if len(cmd)>2 else None cmd = cmd[1] if cmd == "timeout": assert self._plinger is None q.put(None) return True elif cmd == "cancel": if self._plinger: self._plinger.cancel() self._plinger = None q.put(None) return False elif cmd == "update": q.put(None) self.end = a self._set_pling() elif cmd == "remain": q.put(unixtime(self.end)-unixtime(now(self.force))) else: q.put(RuntimeError('Unknown command: '+cmd)) finally: q,self.q = self.q,None super(Waiter,self).delete() if q is not None: while not q.empty(): q.get()[0].put(StopIteration()) def init(self,dest): self.q = Channel() self.end = dest self.start_job("job",self._job) def _cmd(self,cmd,*a): if self.q is None: raise DelayDone(self) q = Channel() self.q.put((q,cmd)+tuple(a)) res = q.get() if isinstance(res,BaseException): raise res return res @property def value(self): if self.q is None: return 0 if not self._running: return "??" res = self._cmd("remain") if TESTING: res = "%.1f" % (res,) return res def delete(self,ctx=None): self._cmd("cancel") def cancel(self, err=DelayCancelled): """Cancel a waiter.""" process_event(Event(self.ctx(loglevel=TRACE),"wait","cancel",ixtime(self.end,self.force),*self.name)) self._cmd("cancel") def retime(self, dest): process_event(Event(self.ctx(loglevel=TRACE),"wait","update",dest,*self.name)) self._cmd("update",dest)
class tokizer(Jobber): def __init__(self, output, parent=None, endput=None): super(tokizer, self).__init__() self._output = output self._endput = endput self.parent = parent self.q = Channel() self.init() self.start_job("job", self._job) self.job.link(self._end) def init(self): self.lnum = self.parenlev = self.continued = 0 self.contstr, self.needcont = '', 0 self.contline = None self.indents = [0] def exit(self): self.stop_job("job") def output(self, *a): log("token", TRACE, repr(a)) self._output(*a) def feed_end(self): for indent in self.indents[1:]: # pop remaining indent levels self.output(DEDENT, '', (self.lnum, 0), (self.lnum, 0), '') def feed(self, line): if self.q: if line is None: q, self.q = self.q, None if gevent.getcurrent() is not self.job: q.put(None) else: self.q.put(line) else: raise StopParsing def _end(self, res): self.feed_end() if self.q: q, self.q = self.q, None try: q.get_nowait() except Empty: pass if self._endput: self._endput() def _job(self): line = "x" while line: if self.q: line = self.q.get() else: line = None self._do_line(line) def _do_line(self, line): try: try: if line is None: raise StopIteration log("token", TRACE, "IN", line) except StopIteration: line = '' log("token", TRACE, "IN_END") self.lnum = self.lnum + 1 pos, max = 0, len(line) if self.contstr: # continued string if not line: raise TokenError("EOF in multi-line string", strstart) endmatch = endprog.match(line) if endmatch: pos = end = endmatch.end(0) self.output(STRING, self.contstr + line[:end], strstart, (self.lnum, end), self.contline + line) self.contstr, self.needcont = '', 0 self.contline = None elif self.needcont and line[-2:] != '\\\n' and line[ -3:] != '\\\r\n': self.output(ERRORTOKEN, self.contstr + line, strstart, (self.lnum, len(line)), self.contline) self.contstr = '' self.contline = None return else: self.contstr += line self.contline += line return elif self.parenlev == 0 and not self.continued: # new statement if not line: self.feed_end() self.output(ENDMARKER, '', (self.lnum, 0), (self.lnum, 0), '') return column = 0 while pos < max: # measure leading whitespace if line[pos] == ' ': column = column + 1 elif line[pos] == '\t': column = (column / tabsize + 1) * tabsize elif line[pos] == '\f': column = 0 else: break pos = pos + 1 if pos == max: self.feed_end() return if line[pos] in '#\r\n': # skip comments or blank lines self.output((NL, COMMENT)[line[pos] == '#'], line[pos:], (self.lnum, pos), (self.lnum, len(line)), line) return if column > self.indents[-1]: # count indents or dedents self.indents.append(column) self.output(INDENT, line[:pos], (self.lnum, 0), (self.lnum, pos), line) if column < self.indents[-1]: while column < self.indents[-1]: self.indents.pop() self.output(DEDENT, '', (self.lnum, pos), (self.lnum, pos), line) if column != self.indents[-1]: raise IndentationError( "unindent does not match any outer indentation level", (u"‹tokenize›", self.lnum, pos, line)) else: # continued statement if not line: raise TokenError("EOF in multi-line statement", (self.lnum, 0)) self.continued = 0 while pos < max: pseudomatch = pseudoprog.match(line, pos) if pseudomatch: # scan for tokens start, end = pseudomatch.span(1) spos, epos, pos = (self.lnum, start), (self.lnum, end), end token, initial = line[start:end], line[start] if num.match(token) or \ (initial == '.' and token != '.'): # ordinary number self.output(NUMBER, token, spos, epos, line) elif initial in '\r\n': self.output(NL if self.parenlev > 0 else NEWLINE, token, spos, epos, line) elif initial == '#': self.output(COMMENT, token, spos, epos, line) elif token in triple_quoted: endprog = endprogs[token] endmatch = endprog.match(line, pos) if endmatch: # all on one line pos = endmatch.end(0) token = line[start:pos] self.output(STRING, token, spos, (self.lnum, pos), line) else: strstart = (self.lnum, start) # multiple lines self.contstr = line[start:] self.contline = line break elif initial in single_quoted or \ token[:2] in single_quoted or \ token[:3] in single_quoted: if token[-1] == '\n': # continued string strstart = (self.lnum, start) endprog = (endprogs[initial] or endprogs[token[1]] or endprogs[token[2]]) self.contstr, self.needcont = line[start:], 1 self.contline = line break else: # ordinary string self.output(STRING, token, spos, epos, line) elif namestart.match(initial): # ordinary name self.output(NAME, token, spos, epos, line) elif initial == '\\': # continued stmt self.continued = 1 else: if initial in '([{': self.parenlev = self.parenlev + 1 elif initial in ')]}': self.parenlev = self.parenlev - 1 self.output(OP, token, spos, epos, line) else: self.output(ERRORTOKEN, line[pos], (self.lnum, pos), (self.lnum, pos + 1), line) pos = pos + 1 except StopParsing as e: fix_exception(e) self.q = None if self.parent: self.parent.kill(e) return