def cpu(self, amount, call_if_blocking): """Block until `amount` of cpu, in millicores, is available. Requesting 0 cpu will never block or wait. Requesting < 0 cpu will raise ValueError. Requesting > _millicores_max will acquire the full CPU. Args: * amount (int) - The amount of millicores to acquire before yielding. Must be positive or will raise ValueError. If this exceeds the maximum amount of millicores available on the system, this will instead acquire the system maximum. * call_if_blocking (None|func(amount_blocked_on)) - `cpu` will invoke this callback if we would end up blocking before yielding. This callback should only be used for reporting/diagnostics (i.e. it shouldn't raise an exception.) Yields control once the requisite amount of cpu is available. """ if amount < 0: raise ValueError('negative cpu amount') if amount > self._millicores_max: amount = self._millicores_max if amount > 0 and (self._waiters or self._millicores_available < amount): # we need some amount of cores AND # someone else is already waiting, or there aren't enough cores left. if call_if_blocking: call_if_blocking(amount - self._millicores_available) wake_me = Channel() self._waiters.append((amount, wake_me)) wake_me.get() # At this point the greenlet that woke us already reserved our cores for # us, and we're free to go. else: # Just directly take our cores. assert self._millicores_available >= amount self._millicores_available -= amount try: yield finally: self._millicores_available += amount # We just added some resource back to the pot. Try to wake as many others # as we can before proceeding. to_wake, to_keep = [], [] for waiting_amount, chan in self._waiters: if waiting_amount <= self._millicores_available: to_wake.append(chan) self._millicores_available -= waiting_amount else: to_keep.append((waiting_amount, chan)) self._waiters = to_keep for chan in to_wake: chan.put(None)
def test_link_to_channel(self): p1 = gevent.spawn(lambda: 101) p2 = gevent.spawn(lambda: 102) p3 = gevent.spawn(lambda: 103) q = Channel() p1.link(q.put) p2.link(q.put) p3.link(q.put) results = [q.get().get(), q.get().get(), q.get().get()] assert sorted(results) == [101, 102, 103], results
def try_read_channel(channel: Channel, timeout: int) -> Tuple[bool, Any]: try: if (timeout == 0): item = channel.get(block=False) elif (timeout > 0): item = channel.get(block=True, timeout=timeout) else: item = channel.get(block=True) return (True, item) except QueueEmptyException: return (False, None)
def _cmd(self,cmd,*a): if self.q is None: raise DelayDone(self) q = Channel() self.q.put((q,cmd)+tuple(a)) res = q.get() if isinstance(res,BaseException): raise res return res
def subscribe(ws): try: if not channels: return subscriptions = random.choice(channels) ch = Channel() subscriptions.append(ch) _log("feed ready! - added subscription " + str(len(subscriptions))) while True: #_log("getting") msg = ch.get() if not msg: _log("nothing to send!") continue ws.send(msg) #_log("sent!") gevent.sleep(0) except WebSocketError: _log("WebSocketError on feed! - retrying") return
class tokizer(Jobber): def __init__(self, output, parent=None, endput=None): self._output = output self._endput = endput self.parent = parent self.q = Channel() self.init() self.start_job("job",self._job) self.job.link(self._end) def init(self): self.lnum = self.parenlev = self.continued = 0 self.contstr, self.needcont = '', 0 self.contline = None self.indents = [0] def exit(self): self.stop_job("job") def output(self,*a): log("token",TRACE,repr(a)) self._output(*a) def feed_end(self): for indent in self.indents[1:]: # pop remaining indent levels self.output(DEDENT, '', (self.lnum, 0), (self.lnum, 0), '') def feed(self,line): if self.q: if line is None: q,self.q = self.q,None if gevent.getcurrent() is not self.job: q.put(None) else: self.q.put(line) else: raise StopParsing def _end(self, res): self.feed_end() if self.q: q,self.q = self.q,None try: q.get_nowait() except Empty: pass if self._endput: self._endput() def _job(self): line = "x" while line: if self.q: line = self.q.get() else: line = None self._do_line(line) def _do_line(self,line): try: try: if line is None: raise StopIteration log("token",TRACE,"IN",line) except StopIteration: line = '' log("token",TRACE,"IN_END") self.lnum = self.lnum + 1 pos, max = 0, len(line) if self.contstr: # continued string if not line: raise TokenError, ("EOF in multi-line string", strstart) endmatch = endprog.match(line) if endmatch: pos = end = endmatch.end(0) self.output(STRING, self.contstr + line[:end], strstart, (self.lnum, end), self.contline + line) self.contstr, self.needcont = '', 0 self.contline = None elif self.needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n': self.output(ERRORTOKEN, self.contstr + line, strstart, (self.lnum, len(line)), self.contline) self.contstr = '' self.contline = None return else: self.contstr += line self.contline += line return elif self.parenlev == 0 and not self.continued: # new statement if not line: self.feed_end() self.output(ENDMARKER, '', (self.lnum, 0), (self.lnum, 0), '') return column = 0 while pos < max: # measure leading whitespace if line[pos] == ' ': column = column + 1 elif line[pos] == '\t': column = (column/tabsize + 1)*tabsize elif line[pos] == '\f': column = 0 else: break pos = pos + 1 if pos == max: self.feed_end() return if line[pos] in '#\r\n': # skip comments or blank lines self.output((NL, COMMENT)[line[pos] == '#'], line[pos:], (self.lnum, pos), (self.lnum, len(line)), line) return if column > self.indents[-1]: # count indents or dedents self.indents.append(column) self.output(INDENT, line[:pos], (self.lnum, 0), (self.lnum, pos), line) if column < self.indents[-1]: while column < self.indents[-1]: self.indents.pop() self.output(DEDENT, '', (self.lnum, pos), (self.lnum, pos), line) if column != self.indents[-1]: raise IndentationError( "unindent does not match any outer indentation level", (u"‹tokenize›", self.lnum, pos, line)) else: # continued statement if not line: raise TokenError, ("EOF in multi-line statement", (self.lnum, 0)) self.continued = 0 while pos < max: pseudomatch = pseudoprog.match(line, pos) if pseudomatch: # scan for tokens start, end = pseudomatch.span(1) spos, epos, pos = (self.lnum, start), (self.lnum, end), end token, initial = line[start:end], line[start] if num.match(token) or \ (initial == '.' and token != '.'): # ordinary number self.output(NUMBER, token, spos, epos, line) elif initial in '\r\n': self.output(NL if self.parenlev > 0 else NEWLINE, token, spos, epos, line) elif initial == '#': self.output(COMMENT, token, spos, epos, line) elif token in triple_quoted: endprog = endprogs[token] endmatch = endprog.match(line, pos) if endmatch: # all on one line pos = endmatch.end(0) token = line[start:pos] self.output(STRING, token, spos, (self.lnum, pos), line) else: strstart = (self.lnum, start) # multiple lines self.contstr = line[start:] self.contline = line break elif initial in single_quoted or \ token[:2] in single_quoted or \ token[:3] in single_quoted: if token[-1] == '\n': # continued string strstart = (self.lnum, start) endprog = (endprogs[initial] or endprogs[token[1]] or endprogs[token[2]]) self.contstr, self.needcont = line[start:], 1 self.contline = line break else: # ordinary string self.output(STRING, token, spos, epos, line) elif namestart.match(initial): # ordinary name self.output(NAME, token, spos, epos, line) elif initial == '\\': # continued stmt self.continued = 1 else: if initial in '([{': self.parenlev = self.parenlev + 1 elif initial in ')]}': self.parenlev = self.parenlev - 1 self.output(OP, token, spos, epos, line) else: self.output(ERRORTOKEN, line[pos], (self.lnum, pos), (self.lnum, pos+1), line) pos = pos + 1 except StopParsing as e: fix_exception(e) self.q = None if self.parent: self.parent.kill(e) return
class tokizer(Jobber): def __init__(self, output, parent=None, endput=None): super(tokizer, self).__init__() self._output = output self._endput = endput self.parent = parent self.q = Channel() self.init() self.start_job("job", self._job) self.job.link(self._end) def init(self): self.lnum = self.parenlev = self.continued = 0 self.contstr, self.needcont = '', 0 self.contline = None self.indents = [0] def exit(self): self.stop_job("job") def output(self, *a): log("token", TRACE, repr(a)) self._output(*a) def feed_end(self): for indent in self.indents[1:]: # pop remaining indent levels self.output(DEDENT, '', (self.lnum, 0), (self.lnum, 0), '') def feed(self, line): if self.q: if line is None: q, self.q = self.q, None if gevent.getcurrent() is not self.job: q.put(None) else: self.q.put(line) else: raise StopParsing def _end(self, res): self.feed_end() if self.q: q, self.q = self.q, None try: q.get_nowait() except Empty: pass if self._endput: self._endput() def _job(self): line = "x" while line: if self.q: line = self.q.get() else: line = None self._do_line(line) def _do_line(self, line): try: try: if line is None: raise StopIteration log("token", TRACE, "IN", line) except StopIteration: line = '' log("token", TRACE, "IN_END") self.lnum = self.lnum + 1 pos, max = 0, len(line) if self.contstr: # continued string if not line: raise TokenError("EOF in multi-line string", strstart) endmatch = endprog.match(line) if endmatch: pos = end = endmatch.end(0) self.output(STRING, self.contstr + line[:end], strstart, (self.lnum, end), self.contline + line) self.contstr, self.needcont = '', 0 self.contline = None elif self.needcont and line[-2:] != '\\\n' and line[ -3:] != '\\\r\n': self.output(ERRORTOKEN, self.contstr + line, strstart, (self.lnum, len(line)), self.contline) self.contstr = '' self.contline = None return else: self.contstr += line self.contline += line return elif self.parenlev == 0 and not self.continued: # new statement if not line: self.feed_end() self.output(ENDMARKER, '', (self.lnum, 0), (self.lnum, 0), '') return column = 0 while pos < max: # measure leading whitespace if line[pos] == ' ': column = column + 1 elif line[pos] == '\t': column = (column / tabsize + 1) * tabsize elif line[pos] == '\f': column = 0 else: break pos = pos + 1 if pos == max: self.feed_end() return if line[pos] in '#\r\n': # skip comments or blank lines self.output((NL, COMMENT)[line[pos] == '#'], line[pos:], (self.lnum, pos), (self.lnum, len(line)), line) return if column > self.indents[-1]: # count indents or dedents self.indents.append(column) self.output(INDENT, line[:pos], (self.lnum, 0), (self.lnum, pos), line) if column < self.indents[-1]: while column < self.indents[-1]: self.indents.pop() self.output(DEDENT, '', (self.lnum, pos), (self.lnum, pos), line) if column != self.indents[-1]: raise IndentationError( "unindent does not match any outer indentation level", (u"‹tokenize›", self.lnum, pos, line)) else: # continued statement if not line: raise TokenError("EOF in multi-line statement", (self.lnum, 0)) self.continued = 0 while pos < max: pseudomatch = pseudoprog.match(line, pos) if pseudomatch: # scan for tokens start, end = pseudomatch.span(1) spos, epos, pos = (self.lnum, start), (self.lnum, end), end token, initial = line[start:end], line[start] if num.match(token) or \ (initial == '.' and token != '.'): # ordinary number self.output(NUMBER, token, spos, epos, line) elif initial in '\r\n': self.output(NL if self.parenlev > 0 else NEWLINE, token, spos, epos, line) elif initial == '#': self.output(COMMENT, token, spos, epos, line) elif token in triple_quoted: endprog = endprogs[token] endmatch = endprog.match(line, pos) if endmatch: # all on one line pos = endmatch.end(0) token = line[start:pos] self.output(STRING, token, spos, (self.lnum, pos), line) else: strstart = (self.lnum, start) # multiple lines self.contstr = line[start:] self.contline = line break elif initial in single_quoted or \ token[:2] in single_quoted or \ token[:3] in single_quoted: if token[-1] == '\n': # continued string strstart = (self.lnum, start) endprog = (endprogs[initial] or endprogs[token[1]] or endprogs[token[2]]) self.contstr, self.needcont = line[start:], 1 self.contline = line break else: # ordinary string self.output(STRING, token, spos, epos, line) elif namestart.match(initial): # ordinary name self.output(NAME, token, spos, epos, line) elif initial == '\\': # continued stmt self.continued = 1 else: if initial in '([{': self.parenlev = self.parenlev + 1 elif initial in ')]}': self.parenlev = self.parenlev - 1 self.output(OP, token, spos, epos, line) else: self.output(ERRORTOKEN, line[pos], (self.lnum, pos), (self.lnum, pos + 1), line) pos = pos + 1 except StopParsing as e: fix_exception(e) self.q = None if self.parent: self.parent.kill(e) return