class Observable(object): terminator = None def __init__(self): self.done = Channel() def wait_eq(self, terminator, message=None, timeout=None): if self.value == terminator: return self.value self.terminator = terminator try: with_timeout(timeout, self.done.get) return self.value except Timeout: eq_(self.value, terminator, message) def _check(self): if self.value == self.terminator: self.done.put(None) def __eq__(self, x): return self.value == x def __repr__(self): return "%s(%r)" % (type(self).__name__, self.value)
def _test_atomic(): # NOTE: Nested context by comma is not available in Python 2.6. # o -- No gevent. with lets.atomic(): 1 + 2 + 3 # x -- gevent.sleep() with pytest.raises(AssertionError): with lets.atomic(): gevent.sleep(0.1) # x -- gevent.sleep() with 0 seconds. with pytest.raises(AssertionError): with lets.atomic(): gevent.sleep(0) # o -- Greenlet.spawn() with lets.atomic(): gevent.spawn(gevent.sleep, 0.1) # x -- Greenlet.join() with pytest.raises(AssertionError): with lets.atomic(): g = gevent.spawn(gevent.sleep, 0.1) g.join() # x -- Greenlet.get() with pytest.raises(AssertionError): with lets.atomic(): g = gevent.spawn(gevent.sleep, 0.1) g.get() # x -- gevent.joinall() with pytest.raises(AssertionError): with lets.atomic(): g = gevent.spawn(gevent.sleep, 0.1) gevent.joinall([g]) # o -- Event.set(), AsyncResult.set() with lets.atomic(): Event().set() AsyncResult().set() # x -- Event.wait() with pytest.raises(AssertionError): with lets.atomic(): Event().wait() # x -- Event.wait() with pytest.raises(AssertionError): with lets.atomic(): AsyncResult().wait() # x -- Channel.put() with pytest.raises(AssertionError): with lets.atomic(): ch = Channel() ch.put(123) # o -- First Semaphore.acquire() with lets.atomic(): lock = Semaphore() lock.acquire() # x -- Second Semaphore.acquire() with pytest.raises(AssertionError): with lets.atomic(): lock = Semaphore() lock.acquire() lock.acquire() # Back to normal. gevent.sleep(1)
def cpu(self, amount, call_if_blocking): """Block until `amount` of cpu, in millicores, is available. Requesting 0 cpu will never block or wait. Requesting < 0 cpu will raise ValueError. Requesting > _millicores_max will acquire the full CPU. Args: * amount (int) - The amount of millicores to acquire before yielding. Must be positive or will raise ValueError. If this exceeds the maximum amount of millicores available on the system, this will instead acquire the system maximum. * call_if_blocking (None|func(amount_blocked_on)) - `cpu` will invoke this callback if we would end up blocking before yielding. This callback should only be used for reporting/diagnostics (i.e. it shouldn't raise an exception.) Yields control once the requisite amount of cpu is available. """ if amount < 0: raise ValueError('negative cpu amount') if amount > self._millicores_max: amount = self._millicores_max if amount > 0 and (self._waiters or self._millicores_available < amount): # we need some amount of cores AND # someone else is already waiting, or there aren't enough cores left. if call_if_blocking: call_if_blocking(amount - self._millicores_available) wake_me = Channel() self._waiters.append((amount, wake_me)) wake_me.get() # At this point the greenlet that woke us already reserved our cores for # us, and we're free to go. else: # Just directly take our cores. assert self._millicores_available >= amount self._millicores_available -= amount try: yield finally: self._millicores_available += amount # We just added some resource back to the pot. Try to wake as many others # as we can before proceeding. to_wake, to_keep = [], [] for waiting_amount, chan in self._waiters: if waiting_amount <= self._millicores_available: to_wake.append(chan) self._millicores_available -= waiting_amount else: to_keep.append((waiting_amount, chan)) self._waiters = to_keep for chan in to_wake: chan.put(None)
def __init__(self, output, parent=None, endput=None): super(tokizer, self).__init__() self._output = output self._endput = endput self.parent = parent self.q = Channel() self.init() self.start_job("job", self._job) self.job.link(self._end)
def _cmd(self,cmd,*a): if self.q is None: raise DelayDone(self) q = Channel() self.q.put((q,cmd)+tuple(a)) res = q.get() if isinstance(res,BaseException): raise res return res
def test_link_to_channel(self): p1 = gevent.spawn(lambda: 101) p2 = gevent.spawn(lambda: 102) p3 = gevent.spawn(lambda: 103) q = Channel() p1.link(q.put) p2.link(q.put) p3.link(q.put) results = [q.get().get(), q.get().get(), q.get().get()] assert sorted(results) == [101, 102, 103], results
def try_read_channel(channel: Channel, timeout: int) -> Tuple[bool, Any]: try: if (timeout == 0): item = channel.get(block=False) elif (timeout > 0): item = channel.get(block=True, timeout=timeout) else: item = channel.get(block=True) return (True, item) except QueueEmptyException: return (False, None)
def try_write_channel(channel: Channel, item: Any, timeout: int) -> bool: try: if (timeout == 0): item = channel.put(item, block=False) elif (timeout > 0): item = channel.put(item, block=True, timeout=timeout) else: item = channel.put(item, block=True) return True except QueueFullException: return False
def connection_handler(client_socket: socket, address: Tuple[str, int]): '''This greenlet is spawned for each client that connects to the server''' logger.info('New connection from %s:%s' % address) try: output = Channel() cancel = Channel() r = gevent.spawn(reader, client_socket, output, cancel) w = gevent.spawn(writer, client_socket, output, cancel) gevent.joinall([r, w]) finally: logger.info('Disconnected %s:%s' % address)
def test_gevent1(): global CH import gevent print gevent, gevent.__version__ from gevent.queue import Channel from gevent import spawn, wait CH = Channel() start = time() spawn(channel_get_put) CH.put(True) spawn(channel_get_put) wait() return time() - start
def handle_wsgi_request(self, env, start_response): ch = Channel() req = Request(ch, env, start_response, content_type=self.default_content_type) self << ('handle', req) return response_stream(ch)
def set_links_timeout(self, link): # stuff that won't be touched event = AsyncResult() link(event) queue = Channel() link(queue.put) return event, queue
def __init__(self, output, parent=None, endput=None): self._output = output self._endput = endput self.parent = parent self.q = Channel() self.init() self.start_job("job",self._job) self.job.link(self._end)
class Server(Endpoint, Greenlet): ''' Used at client side, to represent server ''' def __init__(self, sock, addr): Endpoint.__init__(self, sock, addr) Greenlet.__init__(self) self.ctlcmds = Channel() self.userid = 0 self.gamedata = Gamedata(recording=True) def _run(self): while True: cmd, data = self.read() if cmd == 'gamedata': self.gamedata.feed(data) else: self.ctlcmds.put([cmd, data]) def gexpect(self, tag, blocking=True): return self.gamedata.gexpect(tag, blocking) def gbreak(self): return self.gamedata.gbreak() def gclear(self): self.gamedata = Gamedata(recording=True) def gwrite(self, tag, data): log.debug('GAME_WRITE: %s', repr([tag, data])) encoded = self.encode(['gamedata', [tag, data]]) self.raw_write(encoded) def wait_till_live(self): self.gamedata.wait_empty() def gamedata_piled(self): return len(self.gamedata.gdqueue) > 60 def shutdown(self): self.kill() self.join() self.ctlcmds.put(['shutdown', None]) self.close()
def subscribe(ws): try: if not channels: return subscriptions = random.choice(channels) ch = Channel() subscriptions.append(ch) _log("feed ready! - added subscription " + str(len(subscriptions))) while True: #_log("getting") msg = ch.get() if not msg: _log("nothing to send!") continue ws.send(msg) #_log("sent!") gevent.sleep(0) except WebSocketError: _log("WebSocketError on feed! - retrying") return
def __init__(self, context): self.context = context self.id = None self.name = None self.nmdm = None self.state = VirtualMachineState.STOPPED self.config = None self.bhyve_process = None self.scrollback = io.BytesIO() self.console_fd = None self.console_channel = Channel() self.console_thread = None self.logger = logging.getLogger('VM:{0}'.format(self.name))
class tokizer(Jobber): def __init__(self, output, parent=None, endput=None): self._output = output self._endput = endput self.parent = parent self.q = Channel() self.init() self.start_job("job",self._job) self.job.link(self._end) def init(self): self.lnum = self.parenlev = self.continued = 0 self.contstr, self.needcont = '', 0 self.contline = None self.indents = [0] def exit(self): self.stop_job("job") def output(self,*a): log("token",TRACE,repr(a)) self._output(*a) def feed_end(self): for indent in self.indents[1:]: # pop remaining indent levels self.output(DEDENT, '', (self.lnum, 0), (self.lnum, 0), '') def feed(self,line): if self.q: if line is None: q,self.q = self.q,None if gevent.getcurrent() is not self.job: q.put(None) else: self.q.put(line) else: raise StopParsing def _end(self, res): self.feed_end() if self.q: q,self.q = self.q,None try: q.get_nowait() except Empty: pass if self._endput: self._endput() def _job(self): line = "x" while line: if self.q: line = self.q.get() else: line = None self._do_line(line) def _do_line(self,line): try: try: if line is None: raise StopIteration log("token",TRACE,"IN",line) except StopIteration: line = '' log("token",TRACE,"IN_END") self.lnum = self.lnum + 1 pos, max = 0, len(line) if self.contstr: # continued string if not line: raise TokenError, ("EOF in multi-line string", strstart) endmatch = endprog.match(line) if endmatch: pos = end = endmatch.end(0) self.output(STRING, self.contstr + line[:end], strstart, (self.lnum, end), self.contline + line) self.contstr, self.needcont = '', 0 self.contline = None elif self.needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n': self.output(ERRORTOKEN, self.contstr + line, strstart, (self.lnum, len(line)), self.contline) self.contstr = '' self.contline = None return else: self.contstr += line self.contline += line return elif self.parenlev == 0 and not self.continued: # new statement if not line: self.feed_end() self.output(ENDMARKER, '', (self.lnum, 0), (self.lnum, 0), '') return column = 0 while pos < max: # measure leading whitespace if line[pos] == ' ': column = column + 1 elif line[pos] == '\t': column = (column/tabsize + 1)*tabsize elif line[pos] == '\f': column = 0 else: break pos = pos + 1 if pos == max: self.feed_end() return if line[pos] in '#\r\n': # skip comments or blank lines self.output((NL, COMMENT)[line[pos] == '#'], line[pos:], (self.lnum, pos), (self.lnum, len(line)), line) return if column > self.indents[-1]: # count indents or dedents self.indents.append(column) self.output(INDENT, line[:pos], (self.lnum, 0), (self.lnum, pos), line) if column < self.indents[-1]: while column < self.indents[-1]: self.indents.pop() self.output(DEDENT, '', (self.lnum, pos), (self.lnum, pos), line) if column != self.indents[-1]: raise IndentationError( "unindent does not match any outer indentation level", (u"‹tokenize›", self.lnum, pos, line)) else: # continued statement if not line: raise TokenError, ("EOF in multi-line statement", (self.lnum, 0)) self.continued = 0 while pos < max: pseudomatch = pseudoprog.match(line, pos) if pseudomatch: # scan for tokens start, end = pseudomatch.span(1) spos, epos, pos = (self.lnum, start), (self.lnum, end), end token, initial = line[start:end], line[start] if num.match(token) or \ (initial == '.' and token != '.'): # ordinary number self.output(NUMBER, token, spos, epos, line) elif initial in '\r\n': self.output(NL if self.parenlev > 0 else NEWLINE, token, spos, epos, line) elif initial == '#': self.output(COMMENT, token, spos, epos, line) elif token in triple_quoted: endprog = endprogs[token] endmatch = endprog.match(line, pos) if endmatch: # all on one line pos = endmatch.end(0) token = line[start:pos] self.output(STRING, token, spos, (self.lnum, pos), line) else: strstart = (self.lnum, start) # multiple lines self.contstr = line[start:] self.contline = line break elif initial in single_quoted or \ token[:2] in single_quoted or \ token[:3] in single_quoted: if token[-1] == '\n': # continued string strstart = (self.lnum, start) endprog = (endprogs[initial] or endprogs[token[1]] or endprogs[token[2]]) self.contstr, self.needcont = line[start:], 1 self.contline = line break else: # ordinary string self.output(STRING, token, spos, epos, line) elif namestart.match(initial): # ordinary name self.output(NAME, token, spos, epos, line) elif initial == '\\': # continued stmt self.continued = 1 else: if initial in '([{': self.parenlev = self.parenlev + 1 elif initial in ')]}': self.parenlev = self.parenlev - 1 self.output(OP, token, spos, epos, line) else: self.output(ERRORTOKEN, line[pos], (self.lnum, pos), (self.lnum, pos+1), line) pos = pos + 1 except StopParsing as e: fix_exception(e) self.q = None if self.parent: self.parent.kill(e) return
class VirtualMachine(object): def __init__(self, context): self.context = context self.id = None self.name = None self.nmdm = None self.state = VirtualMachineState.STOPPED self.config = None self.bhyve_process = None self.scrollback = io.BytesIO() self.console_fd = None self.console_channel = Channel() self.console_thread = None self.logger = logging.getLogger('VM:{0}'.format(self.name)) def get_nmdm(self): #for i in range(0, 255): # if os.path.exists('/dev/nmdm{0}A'.format(i)): # continue # # a = '/dev/nmdm{0}A'.format(i) # b = '/dev/nmdm{0}B'.format(i) # self.logger.info('Assigned nmdm device pair: {0}, {1}'.format(a, b)) # return a, b return '/dev/nmdm1A', '/dev/nmdm1B' def start(self): self.nmdm = self.get_nmdm() gevent.spawn(self.run) self.console_thread = gevent.spawn(self.console_worker) def stop(self): if self.state == VirtualMachineState.STOPPED: raise RuntimeError() gevent.kill(self.console_thread) self.bhyve_process.terminate() subprocess.call('/usr/sbin/bhyvectl --destroy --vm={0}'.format(self.name)) self.set_state(VirtualMachineState.STOPPED) def set_state(self, state): self.state = state self.context.client.emit_event('container.changed', { 'operation': 'update', 'ids': [self.id] }) def run(self): self.set_state(VirtualMachineState.BOOTLOADER) if self.config['bootloader'] == 'GRUB': self.bhyve_process = subprocess.Popen( ['/usr/local/lib/grub-bhyve'] ) if self.config['bootloader'] == 'BHYVELOAD': self.bhyve_process = subprocess.Popen( [ '/usr/sbin/bhyveload', '-c', self.nmdm[0], '-m', self.config['memsize'], '-d', self.config['bootdisk'], self.name, ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True ) out, err = self.bhyve_process.communicate() self.logger.debug('bhyveload: {0}'.format(out)) args = [ '/usr/sbin/bhyve', '-A', '-H', '-P', '-c', str(self.config['cpus']), '-m', self.config['memsize'], '-s', '0:0,hostbridge', '-s', '2:0,ahci-cd,{0}'.format(self.config['bootdisk']), '-s', '31,lpc', '-l', 'com1,{0}'.format(self.nmdm[0]), self.name ] self.set_state(VirtualMachineState.RUNNING) self.bhyve_process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) out, err = self.bhyve_process.communicate() self.logger.debug('bhyve: {0}'.format(out)) def console_worker(self): BUFSIZE = 1024 self.console_fd = open(self.nmdm[1], 'r+b') tty.setraw(self.console_fd.fileno()) while True: data = gevent.os.tp_read(self.console_fd.fileno(), BUFSIZE) self.logger.debug('Read: {0}'.format(data)) if not data: self.logger.info('Reopening {0} device'.format(self.nmdm[1])) self.console_fd.close() gevent.sleep(1) self.console_fd = open(self.nmdm[1], 'r+b') tty.setraw(self.console_fd.fileno()) continue self.scrollback.write(data) try: self.console_channel.put(data, block=False) except: pass def console_write(self, data): self.logger.debug('Write: {0}'.format(data)) self.console_fd.write(data) self.console_fd.flush()
def init(self,dest): self.q = Channel() self.end = dest self.start_job("job",self._job)
def __init__(self): self.done = Channel()
def __init__(self, sock, addr): Endpoint.__init__(self, sock, addr) Greenlet.__init__(self) self.ctlcmds = Channel() self.gamedata = Gamedata(recording=True)
class tokizer(Jobber): def __init__(self, output, parent=None, endput=None): super(tokizer, self).__init__() self._output = output self._endput = endput self.parent = parent self.q = Channel() self.init() self.start_job("job", self._job) self.job.link(self._end) def init(self): self.lnum = self.parenlev = self.continued = 0 self.contstr, self.needcont = '', 0 self.contline = None self.indents = [0] def exit(self): self.stop_job("job") def output(self, *a): log("token", TRACE, repr(a)) self._output(*a) def feed_end(self): for indent in self.indents[1:]: # pop remaining indent levels self.output(DEDENT, '', (self.lnum, 0), (self.lnum, 0), '') def feed(self, line): if self.q: if line is None: q, self.q = self.q, None if gevent.getcurrent() is not self.job: q.put(None) else: self.q.put(line) else: raise StopParsing def _end(self, res): self.feed_end() if self.q: q, self.q = self.q, None try: q.get_nowait() except Empty: pass if self._endput: self._endput() def _job(self): line = "x" while line: if self.q: line = self.q.get() else: line = None self._do_line(line) def _do_line(self, line): try: try: if line is None: raise StopIteration log("token", TRACE, "IN", line) except StopIteration: line = '' log("token", TRACE, "IN_END") self.lnum = self.lnum + 1 pos, max = 0, len(line) if self.contstr: # continued string if not line: raise TokenError("EOF in multi-line string", strstart) endmatch = endprog.match(line) if endmatch: pos = end = endmatch.end(0) self.output(STRING, self.contstr + line[:end], strstart, (self.lnum, end), self.contline + line) self.contstr, self.needcont = '', 0 self.contline = None elif self.needcont and line[-2:] != '\\\n' and line[ -3:] != '\\\r\n': self.output(ERRORTOKEN, self.contstr + line, strstart, (self.lnum, len(line)), self.contline) self.contstr = '' self.contline = None return else: self.contstr += line self.contline += line return elif self.parenlev == 0 and not self.continued: # new statement if not line: self.feed_end() self.output(ENDMARKER, '', (self.lnum, 0), (self.lnum, 0), '') return column = 0 while pos < max: # measure leading whitespace if line[pos] == ' ': column = column + 1 elif line[pos] == '\t': column = (column / tabsize + 1) * tabsize elif line[pos] == '\f': column = 0 else: break pos = pos + 1 if pos == max: self.feed_end() return if line[pos] in '#\r\n': # skip comments or blank lines self.output((NL, COMMENT)[line[pos] == '#'], line[pos:], (self.lnum, pos), (self.lnum, len(line)), line) return if column > self.indents[-1]: # count indents or dedents self.indents.append(column) self.output(INDENT, line[:pos], (self.lnum, 0), (self.lnum, pos), line) if column < self.indents[-1]: while column < self.indents[-1]: self.indents.pop() self.output(DEDENT, '', (self.lnum, pos), (self.lnum, pos), line) if column != self.indents[-1]: raise IndentationError( "unindent does not match any outer indentation level", (u"‹tokenize›", self.lnum, pos, line)) else: # continued statement if not line: raise TokenError("EOF in multi-line statement", (self.lnum, 0)) self.continued = 0 while pos < max: pseudomatch = pseudoprog.match(line, pos) if pseudomatch: # scan for tokens start, end = pseudomatch.span(1) spos, epos, pos = (self.lnum, start), (self.lnum, end), end token, initial = line[start:end], line[start] if num.match(token) or \ (initial == '.' and token != '.'): # ordinary number self.output(NUMBER, token, spos, epos, line) elif initial in '\r\n': self.output(NL if self.parenlev > 0 else NEWLINE, token, spos, epos, line) elif initial == '#': self.output(COMMENT, token, spos, epos, line) elif token in triple_quoted: endprog = endprogs[token] endmatch = endprog.match(line, pos) if endmatch: # all on one line pos = endmatch.end(0) token = line[start:pos] self.output(STRING, token, spos, (self.lnum, pos), line) else: strstart = (self.lnum, start) # multiple lines self.contstr = line[start:] self.contline = line break elif initial in single_quoted or \ token[:2] in single_quoted or \ token[:3] in single_quoted: if token[-1] == '\n': # continued string strstart = (self.lnum, start) endprog = (endprogs[initial] or endprogs[token[1]] or endprogs[token[2]]) self.contstr, self.needcont = line[start:], 1 self.contline = line break else: # ordinary string self.output(STRING, token, spos, epos, line) elif namestart.match(initial): # ordinary name self.output(NAME, token, spos, epos, line) elif initial == '\\': # continued stmt self.continued = 1 else: if initial in '([{': self.parenlev = self.parenlev + 1 elif initial in ')]}': self.parenlev = self.parenlev - 1 self.output(OP, token, spos, epos, line) else: self.output(ERRORTOKEN, line[pos], (self.lnum, pos), (self.lnum, pos + 1), line) pos = pos + 1 except StopParsing as e: fix_exception(e) self.q = None if self.parent: self.parent.kill(e) return
class Waiter(Collected,Jobber): """This is the thing that waits.""" force = False storage = Waiters.storage _plinger = None _running = False q = None def __init__(self,parent,name,force): self.ctx = parent.ctx self.start = now() self.force = force try: self.parent = parent.parent except AttributeError: pass super(Waiter,self).__init__(name) def list(self): end=now()+dt.timedelta(0,self.value) yield super(Waiter,self) yield("start",self.start) yield("end",end) yield("total", humandelta(end-self.start)) w = self while True: w = getattr(w,"parent",None) if w is None: break n = getattr(w,"displayname",None) if n is not None: if not isinstance(n,basestring): n = " ".join(unicode(x) for x in n) else: try: if w.args: n = unicode(w.args) except AttributeError: pass if n is None: try: if isinstance(w.name,basestring): n = w.name else: n = " ".join(unicode(x) for x in w.name) except AttributeError: n = w.__class__.__name__ if n is not None: yield("in",n) def info(self): return str(self.value) def __repr__(self): return u"‹%s %s %s›" % (self.__class__.__name__, self.name,self.value) def _pling(self): self._plinger = None self._cmd("timeout") def _set_pling(self): timeout = unixtime(self.end) - unixtime(now(self.force)) if timeout <= 0.1: timeout = 0.1 if self._plinger: self._plinger.cancel() self._plinger = callLater(self.force, timeout, self._pling) def _job(self): try: self._set_pling() self._running = True while True: cmd = self.q.get() q = cmd[0] a = cmd[2] if len(cmd)>2 else None cmd = cmd[1] if cmd == "timeout": assert self._plinger is None q.put(None) return True elif cmd == "cancel": if self._plinger: self._plinger.cancel() self._plinger = None q.put(None) return False elif cmd == "update": q.put(None) self.end = a self._set_pling() elif cmd == "remain": q.put(unixtime(self.end)-unixtime(now(self.force))) else: q.put(RuntimeError('Unknown command: '+cmd)) finally: q,self.q = self.q,None super(Waiter,self).delete() if q is not None: while not q.empty(): q.get()[0].put(StopIteration()) def init(self,dest): self.q = Channel() self.end = dest self.start_job("job",self._job) def _cmd(self,cmd,*a): if self.q is None: raise DelayDone(self) q = Channel() self.q.put((q,cmd)+tuple(a)) res = q.get() if isinstance(res,BaseException): raise res return res @property def value(self): if self.q is None: return 0 if not self._running: return "??" res = self._cmd("remain") if TESTING: res = "%.1f" % (res,) return res def delete(self,ctx=None): self._cmd("cancel") def cancel(self, err=DelayCancelled): """Cancel a waiter.""" process_event(Event(self.ctx(loglevel=TRACE),"wait","cancel",ixtime(self.end,self.force),*self.name)) self._cmd("cancel") def retime(self, dest): process_event(Event(self.ctx(loglevel=TRACE),"wait","update",dest,*self.name)) self._cmd("update",dest)