def cpu(self, amount, call_if_blocking):
"""Block until `amount` of cpu, in millicores, is available.
Requesting 0 cpu will never block or wait.
Requesting < 0 cpu will raise ValueError.
Requesting > _millicores_max will acquire the full CPU.
Args:
* amount (int) - The amount of millicores to acquire before yielding. Must
be positive or will raise ValueError. If this exceeds the maximum amount
of millicores available on the system, this will instead acquire the
system maximum.
* call_if_blocking (None|func(amount_blocked_on)) - `cpu` will invoke this
callback if we would end up blocking before yielding. This callback
should only be used for reporting/diagnostics (i.e. it shouldn't raise
an exception.)
Yields control once the requisite amount of cpu is available.
"""
if amount < 0:
raise ValueError('negative cpu amount')
if amount > self._millicores_max:
amount = self._millicores_max
if amount > 0 and (self._waiters
or self._millicores_available < amount):
# we need some amount of cores AND
# someone else is already waiting, or there aren't enough cores left.
if call_if_blocking:
call_if_blocking(amount - self._millicores_available)
wake_me = Channel()
self._waiters.append((amount, wake_me))
wake_me.get()
# At this point the greenlet that woke us already reserved our cores for
# us, and we're free to go.
else:
# Just directly take our cores.
assert self._millicores_available >= amount
self._millicores_available -= amount
try:
yield
finally:
self._millicores_available += amount
# We just added some resource back to the pot. Try to wake as many others
# as we can before proceeding.
to_wake, to_keep = [], []
for waiting_amount, chan in self._waiters:
if waiting_amount <= self._millicores_available:
to_wake.append(chan)
self._millicores_available -= waiting_amount
else:
to_keep.append((waiting_amount, chan))
self._waiters = to_keep
for chan in to_wake:
chan.put(None)
def test_link_to_channel(self):
p1 = gevent.spawn(lambda: 101)
p2 = gevent.spawn(lambda: 102)
p3 = gevent.spawn(lambda: 103)
q = Channel()
p1.link(q.put)
p2.link(q.put)
p3.link(q.put)
results = [q.get().get(), q.get().get(), q.get().get()]
assert sorted(results) == [101, 102, 103], results
def test_link_to_channel(self):
p1 = gevent.spawn(lambda: 101)
p2 = gevent.spawn(lambda: 102)
p3 = gevent.spawn(lambda: 103)
q = Channel()
p1.link(q.put)
p2.link(q.put)
p3.link(q.put)
results = [q.get().get(), q.get().get(), q.get().get()]
assert sorted(results) == [101, 102, 103], results
def try_read_channel(channel: Channel, timeout: int) -> Tuple[bool, Any]:
try:
if (timeout == 0):
item = channel.get(block=False)
elif (timeout > 0):
item = channel.get(block=True, timeout=timeout)
else:
item = channel.get(block=True)
return (True, item)
except QueueEmptyException:
return (False, None)
def _cmd(self,cmd,*a): if self.q is None: raise DelayDone(self) q = Channel() self.q.put((q,cmd)+tuple(a)) res = q.get() if isinstance(res,BaseException): raise res return res
def subscribe(ws):
try:
if not channels:
return
subscriptions = random.choice(channels)
ch = Channel()
subscriptions.append(ch)
_log("feed ready! - added subscription " + str(len(subscriptions)))
while True:
#_log("getting")
msg = ch.get()
if not msg:
_log("nothing to send!")
continue
ws.send(msg)
#_log("sent!")
gevent.sleep(0)
except WebSocketError:
_log("WebSocketError on feed! - retrying")
return
def subscribe(ws):
try:
if not channels:
return
subscriptions = random.choice(channels)
ch = Channel()
subscriptions.append(ch)
_log("feed ready! - added subscription " + str(len(subscriptions)))
while True:
#_log("getting")
msg = ch.get()
if not msg:
_log("nothing to send!")
continue
ws.send(msg)
#_log("sent!")
gevent.sleep(0)
except WebSocketError:
_log("WebSocketError on feed! - retrying")
return
class tokizer(Jobber):
def __init__(self, output, parent=None, endput=None):
self._output = output
self._endput = endput
self.parent = parent
self.q = Channel()
self.init()
self.start_job("job",self._job)
self.job.link(self._end)
def init(self):
self.lnum = self.parenlev = self.continued = 0
self.contstr, self.needcont = '', 0
self.contline = None
self.indents = [0]
def exit(self):
self.stop_job("job")
def output(self,*a):
log("token",TRACE,repr(a))
self._output(*a)
def feed_end(self):
for indent in self.indents[1:]: # pop remaining indent levels
self.output(DEDENT, '', (self.lnum, 0), (self.lnum, 0), '')
def feed(self,line):
if self.q:
if line is None:
q,self.q = self.q,None
if gevent.getcurrent() is not self.job:
q.put(None)
else:
self.q.put(line)
else:
raise StopParsing
def _end(self, res):
self.feed_end()
if self.q:
q,self.q = self.q,None
try:
q.get_nowait()
except Empty:
pass
if self._endput:
self._endput()
def _job(self):
line = "x"
while line:
if self.q:
line = self.q.get()
else:
line = None
self._do_line(line)
def _do_line(self,line):
try:
try:
if line is None:
raise StopIteration
log("token",TRACE,"IN",line)
except StopIteration:
line = ''
log("token",TRACE,"IN_END")
self.lnum = self.lnum + 1
pos, max = 0, len(line)
if self.contstr: # continued string
if not line:
raise TokenError, ("EOF in multi-line string", strstart)
endmatch = endprog.match(line)
if endmatch:
pos = end = endmatch.end(0)
self.output(STRING, self.contstr + line[:end],
strstart, (self.lnum, end), self.contline + line)
self.contstr, self.needcont = '', 0
self.contline = None
elif self.needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
self.output(ERRORTOKEN, self.contstr + line,
strstart, (self.lnum, len(line)), self.contline)
self.contstr = ''
self.contline = None
return
else:
self.contstr += line
self.contline += line
return
elif self.parenlev == 0 and not self.continued: # new statement
if not line:
self.feed_end()
self.output(ENDMARKER, '', (self.lnum, 0), (self.lnum, 0), '')
return
column = 0
while pos < max: # measure leading whitespace
if line[pos] == ' ': column = column + 1
elif line[pos] == '\t': column = (column/tabsize + 1)*tabsize
elif line[pos] == '\f': column = 0
else: break
pos = pos + 1
if pos == max:
self.feed_end()
return
if line[pos] in '#\r\n': # skip comments or blank lines
self.output((NL, COMMENT)[line[pos] == '#'], line[pos:],
(self.lnum, pos), (self.lnum, len(line)), line)
return
if column > self.indents[-1]: # count indents or dedents
self.indents.append(column)
self.output(INDENT, line[:pos], (self.lnum, 0), (self.lnum, pos), line)
if column < self.indents[-1]:
while column < self.indents[-1]:
self.indents.pop()
self.output(DEDENT, '', (self.lnum, pos), (self.lnum, pos), line)
if column != self.indents[-1]:
raise IndentationError(
"unindent does not match any outer indentation level",
(u"‹tokenize›", self.lnum, pos, line))
else: # continued statement
if not line:
raise TokenError, ("EOF in multi-line statement", (self.lnum, 0))
self.continued = 0
while pos < max:
pseudomatch = pseudoprog.match(line, pos)
if pseudomatch: # scan for tokens
start, end = pseudomatch.span(1)
spos, epos, pos = (self.lnum, start), (self.lnum, end), end
token, initial = line[start:end], line[start]
if num.match(token) or \
(initial == '.' and token != '.'): # ordinary number
self.output(NUMBER, token, spos, epos, line)
elif initial in '\r\n':
self.output(NL if self.parenlev > 0 else NEWLINE, token, spos, epos, line)
elif initial == '#':
self.output(COMMENT, token, spos, epos, line)
elif token in triple_quoted:
endprog = endprogs[token]
endmatch = endprog.match(line, pos)
if endmatch: # all on one line
pos = endmatch.end(0)
token = line[start:pos]
self.output(STRING, token, spos, (self.lnum, pos), line)
else:
strstart = (self.lnum, start) # multiple lines
self.contstr = line[start:]
self.contline = line
break
elif initial in single_quoted or \
token[:2] in single_quoted or \
token[:3] in single_quoted:
if token[-1] == '\n': # continued string
strstart = (self.lnum, start)
endprog = (endprogs[initial] or endprogs[token[1]] or
endprogs[token[2]])
self.contstr, self.needcont = line[start:], 1
self.contline = line
break
else: # ordinary string
self.output(STRING, token, spos, epos, line)
elif namestart.match(initial): # ordinary name
self.output(NAME, token, spos, epos, line)
elif initial == '\\': # continued stmt
self.continued = 1
else:
if initial in '([{': self.parenlev = self.parenlev + 1
elif initial in ')]}': self.parenlev = self.parenlev - 1
self.output(OP, token, spos, epos, line)
else:
self.output(ERRORTOKEN, line[pos],
(self.lnum, pos), (self.lnum, pos+1), line)
pos = pos + 1
except StopParsing as e:
fix_exception(e)
self.q = None
if self.parent:
self.parent.kill(e)
return
class tokizer(Jobber):
def __init__(self, output, parent=None, endput=None):
super(tokizer, self).__init__()
self._output = output
self._endput = endput
self.parent = parent
self.q = Channel()
self.init()
self.start_job("job", self._job)
self.job.link(self._end)
def init(self):
self.lnum = self.parenlev = self.continued = 0
self.contstr, self.needcont = '', 0
self.contline = None
self.indents = [0]
def exit(self):
self.stop_job("job")
def output(self, *a):
log("token", TRACE, repr(a))
self._output(*a)
def feed_end(self):
for indent in self.indents[1:]: # pop remaining indent levels
self.output(DEDENT, '', (self.lnum, 0), (self.lnum, 0), '')
def feed(self, line):
if self.q:
if line is None:
q, self.q = self.q, None
if gevent.getcurrent() is not self.job:
q.put(None)
else:
self.q.put(line)
else:
raise StopParsing
def _end(self, res):
self.feed_end()
if self.q:
q, self.q = self.q, None
try:
q.get_nowait()
except Empty:
pass
if self._endput:
self._endput()
def _job(self):
line = "x"
while line:
if self.q:
line = self.q.get()
else:
line = None
self._do_line(line)
def _do_line(self, line):
try:
try:
if line is None:
raise StopIteration
log("token", TRACE, "IN", line)
except StopIteration:
line = ''
log("token", TRACE, "IN_END")
self.lnum = self.lnum + 1
pos, max = 0, len(line)
if self.contstr: # continued string
if not line:
raise TokenError("EOF in multi-line string", strstart)
endmatch = endprog.match(line)
if endmatch:
pos = end = endmatch.end(0)
self.output(STRING, self.contstr + line[:end], strstart,
(self.lnum, end), self.contline + line)
self.contstr, self.needcont = '', 0
self.contline = None
elif self.needcont and line[-2:] != '\\\n' and line[
-3:] != '\\\r\n':
self.output(ERRORTOKEN, self.contstr + line, strstart,
(self.lnum, len(line)), self.contline)
self.contstr = ''
self.contline = None
return
else:
self.contstr += line
self.contline += line
return
elif self.parenlev == 0 and not self.continued: # new statement
if not line:
self.feed_end()
self.output(ENDMARKER, '', (self.lnum, 0), (self.lnum, 0),
'')
return
column = 0
while pos < max: # measure leading whitespace
if line[pos] == ' ': column = column + 1
elif line[pos] == '\t':
column = (column / tabsize + 1) * tabsize
elif line[pos] == '\f':
column = 0
else:
break
pos = pos + 1
if pos == max:
self.feed_end()
return
if line[pos] in '#\r\n': # skip comments or blank lines
self.output((NL, COMMENT)[line[pos] == '#'], line[pos:],
(self.lnum, pos), (self.lnum, len(line)), line)
return
if column > self.indents[-1]: # count indents or dedents
self.indents.append(column)
self.output(INDENT, line[:pos], (self.lnum, 0),
(self.lnum, pos), line)
if column < self.indents[-1]:
while column < self.indents[-1]:
self.indents.pop()
self.output(DEDENT, '', (self.lnum, pos),
(self.lnum, pos), line)
if column != self.indents[-1]:
raise IndentationError(
"unindent does not match any outer indentation level",
(u"‹tokenize›", self.lnum, pos, line))
else: # continued statement
if not line:
raise TokenError("EOF in multi-line statement",
(self.lnum, 0))
self.continued = 0
while pos < max:
pseudomatch = pseudoprog.match(line, pos)
if pseudomatch: # scan for tokens
start, end = pseudomatch.span(1)
spos, epos, pos = (self.lnum, start), (self.lnum, end), end
token, initial = line[start:end], line[start]
if num.match(token) or \
(initial == '.' and token != '.'): # ordinary number
self.output(NUMBER, token, spos, epos, line)
elif initial in '\r\n':
self.output(NL if self.parenlev > 0 else NEWLINE,
token, spos, epos, line)
elif initial == '#':
self.output(COMMENT, token, spos, epos, line)
elif token in triple_quoted:
endprog = endprogs[token]
endmatch = endprog.match(line, pos)
if endmatch: # all on one line
pos = endmatch.end(0)
token = line[start:pos]
self.output(STRING, token, spos, (self.lnum, pos),
line)
else:
strstart = (self.lnum, start) # multiple lines
self.contstr = line[start:]
self.contline = line
break
elif initial in single_quoted or \
token[:2] in single_quoted or \
token[:3] in single_quoted:
if token[-1] == '\n': # continued string
strstart = (self.lnum, start)
endprog = (endprogs[initial] or endprogs[token[1]]
or endprogs[token[2]])
self.contstr, self.needcont = line[start:], 1
self.contline = line
break
else: # ordinary string
self.output(STRING, token, spos, epos, line)
elif namestart.match(initial): # ordinary name
self.output(NAME, token, spos, epos, line)
elif initial == '\\': # continued stmt
self.continued = 1
else:
if initial in '([{': self.parenlev = self.parenlev + 1
elif initial in ')]}':
self.parenlev = self.parenlev - 1
self.output(OP, token, spos, epos, line)
else:
self.output(ERRORTOKEN, line[pos], (self.lnum, pos),
(self.lnum, pos + 1), line)
pos = pos + 1
except StopParsing as e:
fix_exception(e)
self.q = None
if self.parent:
self.parent.kill(e)
return