class Observable(object):
terminator = None
def __init__(self):
self.done = Channel()
def wait_eq(self, terminator, message=None, timeout=None):
if self.value == terminator:
return self.value
self.terminator = terminator
try:
with_timeout(timeout, self.done.get)
return self.value
except Timeout:
eq_(self.value, terminator, message)
def _check(self):
if self.value == self.terminator:
self.done.put(None)
def __eq__(self, x):
return self.value == x
def __repr__(self):
return "%s(%r)" % (type(self).__name__, self.value)
class Observable(object):
terminator = None
def __init__(self):
self.done = Channel()
def wait_eq(self, terminator, message=None, timeout=None):
if self.value == terminator:
return self.value
self.terminator = terminator
try:
with_timeout(timeout, self.done.get)
return self.value
except Timeout:
eq_(self.value, terminator, message)
def _check(self):
if self.value == self.terminator:
self.done.put(None)
def __eq__(self, x):
return self.value == x
def __repr__(self):
return "%s(%r)" % (type(self).__name__, self.value)
def _test_atomic():
# NOTE: Nested context by comma is not available in Python 2.6.
# o -- No gevent.
with lets.atomic():
1 + 2 + 3
# x -- gevent.sleep()
with pytest.raises(AssertionError):
with lets.atomic():
gevent.sleep(0.1)
# x -- gevent.sleep() with 0 seconds.
with pytest.raises(AssertionError):
with lets.atomic():
gevent.sleep(0)
# o -- Greenlet.spawn()
with lets.atomic():
gevent.spawn(gevent.sleep, 0.1)
# x -- Greenlet.join()
with pytest.raises(AssertionError):
with lets.atomic():
g = gevent.spawn(gevent.sleep, 0.1)
g.join()
# x -- Greenlet.get()
with pytest.raises(AssertionError):
with lets.atomic():
g = gevent.spawn(gevent.sleep, 0.1)
g.get()
# x -- gevent.joinall()
with pytest.raises(AssertionError):
with lets.atomic():
g = gevent.spawn(gevent.sleep, 0.1)
gevent.joinall([g])
# o -- Event.set(), AsyncResult.set()
with lets.atomic():
Event().set()
AsyncResult().set()
# x -- Event.wait()
with pytest.raises(AssertionError):
with lets.atomic():
Event().wait()
# x -- Event.wait()
with pytest.raises(AssertionError):
with lets.atomic():
AsyncResult().wait()
# x -- Channel.put()
with pytest.raises(AssertionError):
with lets.atomic():
ch = Channel()
ch.put(123)
# o -- First Semaphore.acquire()
with lets.atomic():
lock = Semaphore()
lock.acquire()
# x -- Second Semaphore.acquire()
with pytest.raises(AssertionError):
with lets.atomic():
lock = Semaphore()
lock.acquire()
lock.acquire()
# Back to normal.
gevent.sleep(1)
def try_write_channel(channel: Channel, item: Any, timeout: int) -> bool:
try:
if (timeout == 0):
item = channel.put(item, block=False)
elif (timeout > 0):
item = channel.put(item, block=True, timeout=timeout)
else:
item = channel.put(item, block=True)
return True
except QueueFullException:
return False
def test_gevent1():
global CH
import gevent
print gevent, gevent.__version__
from gevent.queue import Channel
from gevent import spawn, wait
CH = Channel()
start = time()
spawn(channel_get_put)
CH.put(True)
spawn(channel_get_put)
wait()
return time() - start
def test_gevent1():
global CH
import gevent
print gevent, gevent.__version__
from gevent.queue import Channel
from gevent import spawn, wait
CH = Channel()
start = time()
spawn(channel_get_put)
CH.put(True)
spawn(channel_get_put)
wait()
return time() - start
class Server(Endpoint, Greenlet):
'''
Used at client side, to represent server
'''
def __init__(self, sock, addr):
Endpoint.__init__(self, sock, addr)
Greenlet.__init__(self)
self.ctlcmds = Channel()
self.userid = 0
self.gamedata = Gamedata(recording=True)
def _run(self):
while True:
cmd, data = self.read()
if cmd == 'gamedata':
self.gamedata.feed(data)
else:
self.ctlcmds.put([cmd, data])
def gexpect(self, tag, blocking=True):
return self.gamedata.gexpect(tag, blocking)
def gbreak(self):
return self.gamedata.gbreak()
def gclear(self):
self.gamedata = Gamedata(recording=True)
def gwrite(self, tag, data):
log.debug('GAME_WRITE: %s', repr([tag, data]))
encoded = self.encode(['gamedata', [tag, data]])
self.raw_write(encoded)
def wait_till_live(self):
self.gamedata.wait_empty()
def gamedata_piled(self):
return len(self.gamedata.gdqueue) > 60
def shutdown(self):
self.kill()
self.join()
self.ctlcmds.put(['shutdown', None])
self.close()
class tokizer(Jobber):
def __init__(self, output, parent=None, endput=None):
self._output = output
self._endput = endput
self.parent = parent
self.q = Channel()
self.init()
self.start_job("job",self._job)
self.job.link(self._end)
def init(self):
self.lnum = self.parenlev = self.continued = 0
self.contstr, self.needcont = '', 0
self.contline = None
self.indents = [0]
def exit(self):
self.stop_job("job")
def output(self,*a):
log("token",TRACE,repr(a))
self._output(*a)
def feed_end(self):
for indent in self.indents[1:]: # pop remaining indent levels
self.output(DEDENT, '', (self.lnum, 0), (self.lnum, 0), '')
def feed(self,line):
if self.q:
if line is None:
q,self.q = self.q,None
if gevent.getcurrent() is not self.job:
q.put(None)
else:
self.q.put(line)
else:
raise StopParsing
def _end(self, res):
self.feed_end()
if self.q:
q,self.q = self.q,None
try:
q.get_nowait()
except Empty:
pass
if self._endput:
self._endput()
def _job(self):
line = "x"
while line:
if self.q:
line = self.q.get()
else:
line = None
self._do_line(line)
def _do_line(self,line):
try:
try:
if line is None:
raise StopIteration
log("token",TRACE,"IN",line)
except StopIteration:
line = ''
log("token",TRACE,"IN_END")
self.lnum = self.lnum + 1
pos, max = 0, len(line)
if self.contstr: # continued string
if not line:
raise TokenError, ("EOF in multi-line string", strstart)
endmatch = endprog.match(line)
if endmatch:
pos = end = endmatch.end(0)
self.output(STRING, self.contstr + line[:end],
strstart, (self.lnum, end), self.contline + line)
self.contstr, self.needcont = '', 0
self.contline = None
elif self.needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
self.output(ERRORTOKEN, self.contstr + line,
strstart, (self.lnum, len(line)), self.contline)
self.contstr = ''
self.contline = None
return
else:
self.contstr += line
self.contline += line
return
elif self.parenlev == 0 and not self.continued: # new statement
if not line:
self.feed_end()
self.output(ENDMARKER, '', (self.lnum, 0), (self.lnum, 0), '')
return
column = 0
while pos < max: # measure leading whitespace
if line[pos] == ' ': column = column + 1
elif line[pos] == '\t': column = (column/tabsize + 1)*tabsize
elif line[pos] == '\f': column = 0
else: break
pos = pos + 1
if pos == max:
self.feed_end()
return
if line[pos] in '#\r\n': # skip comments or blank lines
self.output((NL, COMMENT)[line[pos] == '#'], line[pos:],
(self.lnum, pos), (self.lnum, len(line)), line)
return
if column > self.indents[-1]: # count indents or dedents
self.indents.append(column)
self.output(INDENT, line[:pos], (self.lnum, 0), (self.lnum, pos), line)
if column < self.indents[-1]:
while column < self.indents[-1]:
self.indents.pop()
self.output(DEDENT, '', (self.lnum, pos), (self.lnum, pos), line)
if column != self.indents[-1]:
raise IndentationError(
"unindent does not match any outer indentation level",
(u"‹tokenize›", self.lnum, pos, line))
else: # continued statement
if not line:
raise TokenError, ("EOF in multi-line statement", (self.lnum, 0))
self.continued = 0
while pos < max:
pseudomatch = pseudoprog.match(line, pos)
if pseudomatch: # scan for tokens
start, end = pseudomatch.span(1)
spos, epos, pos = (self.lnum, start), (self.lnum, end), end
token, initial = line[start:end], line[start]
if num.match(token) or \
(initial == '.' and token != '.'): # ordinary number
self.output(NUMBER, token, spos, epos, line)
elif initial in '\r\n':
self.output(NL if self.parenlev > 0 else NEWLINE, token, spos, epos, line)
elif initial == '#':
self.output(COMMENT, token, spos, epos, line)
elif token in triple_quoted:
endprog = endprogs[token]
endmatch = endprog.match(line, pos)
if endmatch: # all on one line
pos = endmatch.end(0)
token = line[start:pos]
self.output(STRING, token, spos, (self.lnum, pos), line)
else:
strstart = (self.lnum, start) # multiple lines
self.contstr = line[start:]
self.contline = line
break
elif initial in single_quoted or \
token[:2] in single_quoted or \
token[:3] in single_quoted:
if token[-1] == '\n': # continued string
strstart = (self.lnum, start)
endprog = (endprogs[initial] or endprogs[token[1]] or
endprogs[token[2]])
self.contstr, self.needcont = line[start:], 1
self.contline = line
break
else: # ordinary string
self.output(STRING, token, spos, epos, line)
elif namestart.match(initial): # ordinary name
self.output(NAME, token, spos, epos, line)
elif initial == '\\': # continued stmt
self.continued = 1
else:
if initial in '([{': self.parenlev = self.parenlev + 1
elif initial in ')]}': self.parenlev = self.parenlev - 1
self.output(OP, token, spos, epos, line)
else:
self.output(ERRORTOKEN, line[pos],
(self.lnum, pos), (self.lnum, pos+1), line)
pos = pos + 1
except StopParsing as e:
fix_exception(e)
self.q = None
if self.parent:
self.parent.kill(e)
return
class VirtualMachine(object):
def __init__(self, context):
self.context = context
self.id = None
self.name = None
self.nmdm = None
self.state = VirtualMachineState.STOPPED
self.config = None
self.bhyve_process = None
self.scrollback = io.BytesIO()
self.console_fd = None
self.console_channel = Channel()
self.console_thread = None
self.logger = logging.getLogger('VM:{0}'.format(self.name))
def get_nmdm(self):
#for i in range(0, 255):
# if os.path.exists('/dev/nmdm{0}A'.format(i)):
# continue
#
# a = '/dev/nmdm{0}A'.format(i)
# b = '/dev/nmdm{0}B'.format(i)
# self.logger.info('Assigned nmdm device pair: {0}, {1}'.format(a, b))
# return a, b
return '/dev/nmdm1A', '/dev/nmdm1B'
def start(self):
self.nmdm = self.get_nmdm()
gevent.spawn(self.run)
self.console_thread = gevent.spawn(self.console_worker)
def stop(self):
if self.state == VirtualMachineState.STOPPED:
raise RuntimeError()
gevent.kill(self.console_thread)
self.bhyve_process.terminate()
subprocess.call('/usr/sbin/bhyvectl --destroy --vm={0}'.format(self.name))
self.set_state(VirtualMachineState.STOPPED)
def set_state(self, state):
self.state = state
self.context.client.emit_event('container.changed', {
'operation': 'update',
'ids': [self.id]
})
def run(self):
self.set_state(VirtualMachineState.BOOTLOADER)
if self.config['bootloader'] == 'GRUB':
self.bhyve_process = subprocess.Popen(
['/usr/local/lib/grub-bhyve']
)
if self.config['bootloader'] == 'BHYVELOAD':
self.bhyve_process = subprocess.Popen(
[
'/usr/sbin/bhyveload', '-c', self.nmdm[0], '-m', self.config['memsize'],
'-d', self.config['bootdisk'], self.name,
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
close_fds=True
)
out, err = self.bhyve_process.communicate()
self.logger.debug('bhyveload: {0}'.format(out))
args = [
'/usr/sbin/bhyve', '-A', '-H', '-P', '-c', str(self.config['cpus']), '-m', self.config['memsize'],
'-s', '0:0,hostbridge',
'-s', '2:0,ahci-cd,{0}'.format(self.config['bootdisk']),
'-s', '31,lpc', '-l', 'com1,{0}'.format(self.nmdm[0]),
self.name
]
self.set_state(VirtualMachineState.RUNNING)
self.bhyve_process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
out, err = self.bhyve_process.communicate()
self.logger.debug('bhyve: {0}'.format(out))
def console_worker(self):
BUFSIZE = 1024
self.console_fd = open(self.nmdm[1], 'r+b')
tty.setraw(self.console_fd.fileno())
while True:
data = gevent.os.tp_read(self.console_fd.fileno(), BUFSIZE)
self.logger.debug('Read: {0}'.format(data))
if not data:
self.logger.info('Reopening {0} device'.format(self.nmdm[1]))
self.console_fd.close()
gevent.sleep(1)
self.console_fd = open(self.nmdm[1], 'r+b')
tty.setraw(self.console_fd.fileno())
continue
self.scrollback.write(data)
try:
self.console_channel.put(data, block=False)
except:
pass
def console_write(self, data):
self.logger.debug('Write: {0}'.format(data))
self.console_fd.write(data)
self.console_fd.flush()
class Waiter(Collected,Jobber):
"""This is the thing that waits."""
force = False
storage = Waiters.storage
_plinger = None
_running = False
q = None
def __init__(self,parent,name,force):
self.ctx = parent.ctx
self.start = now()
self.force = force
try:
self.parent = parent.parent
except AttributeError:
pass
super(Waiter,self).__init__(name)
def list(self):
end=now()+dt.timedelta(0,self.value)
yield super(Waiter,self)
yield("start",self.start)
yield("end",end)
yield("total", humandelta(end-self.start))
w = self
while True:
w = getattr(w,"parent",None)
if w is None: break
n = getattr(w,"displayname",None)
if n is not None:
if not isinstance(n,basestring):
n = " ".join(unicode(x) for x in n)
else:
try:
if w.args:
n = unicode(w.args)
except AttributeError:
pass
if n is None:
try:
if isinstance(w.name,basestring):
n = w.name
else:
n = " ".join(unicode(x) for x in w.name)
except AttributeError:
n = w.__class__.__name__
if n is not None:
yield("in",n)
def info(self):
return str(self.value)
def __repr__(self):
return u"‹%s %s %s›" % (self.__class__.__name__, self.name,self.value)
def _pling(self):
self._plinger = None
self._cmd("timeout")
def _set_pling(self):
timeout = unixtime(self.end) - unixtime(now(self.force))
if timeout <= 0.1:
timeout = 0.1
if self._plinger:
self._plinger.cancel()
self._plinger = callLater(self.force, timeout, self._pling)
def _job(self):
try:
self._set_pling()
self._running = True
while True:
cmd = self.q.get()
q = cmd[0]
a = cmd[2] if len(cmd)>2 else None
cmd = cmd[1]
if cmd == "timeout":
assert self._plinger is None
q.put(None)
return True
elif cmd == "cancel":
if self._plinger:
self._plinger.cancel()
self._plinger = None
q.put(None)
return False
elif cmd == "update":
q.put(None)
self.end = a
self._set_pling()
elif cmd == "remain":
q.put(unixtime(self.end)-unixtime(now(self.force)))
else:
q.put(RuntimeError('Unknown command: '+cmd))
finally:
q,self.q = self.q,None
super(Waiter,self).delete()
if q is not None:
while not q.empty():
q.get()[0].put(StopIteration())
def init(self,dest):
self.q = Channel()
self.end = dest
self.start_job("job",self._job)
def _cmd(self,cmd,*a):
if self.q is None:
raise DelayDone(self)
q = Channel()
self.q.put((q,cmd)+tuple(a))
res = q.get()
if isinstance(res,BaseException):
raise res
return res
@property
def value(self):
if self.q is None:
return 0
if not self._running:
return "??"
res = self._cmd("remain")
if TESTING:
res = "%.1f" % (res,)
return res
def delete(self,ctx=None):
self._cmd("cancel")
def cancel(self, err=DelayCancelled):
"""Cancel a waiter."""
process_event(Event(self.ctx(loglevel=TRACE),"wait","cancel",ixtime(self.end,self.force),*self.name))
self._cmd("cancel")
def retime(self, dest):
process_event(Event(self.ctx(loglevel=TRACE),"wait","update",dest,*self.name))
self._cmd("update",dest)
class tokizer(Jobber):
def __init__(self, output, parent=None, endput=None):
super(tokizer, self).__init__()
self._output = output
self._endput = endput
self.parent = parent
self.q = Channel()
self.init()
self.start_job("job", self._job)
self.job.link(self._end)
def init(self):
self.lnum = self.parenlev = self.continued = 0
self.contstr, self.needcont = '', 0
self.contline = None
self.indents = [0]
def exit(self):
self.stop_job("job")
def output(self, *a):
log("token", TRACE, repr(a))
self._output(*a)
def feed_end(self):
for indent in self.indents[1:]: # pop remaining indent levels
self.output(DEDENT, '', (self.lnum, 0), (self.lnum, 0), '')
def feed(self, line):
if self.q:
if line is None:
q, self.q = self.q, None
if gevent.getcurrent() is not self.job:
q.put(None)
else:
self.q.put(line)
else:
raise StopParsing
def _end(self, res):
self.feed_end()
if self.q:
q, self.q = self.q, None
try:
q.get_nowait()
except Empty:
pass
if self._endput:
self._endput()
def _job(self):
line = "x"
while line:
if self.q:
line = self.q.get()
else:
line = None
self._do_line(line)
def _do_line(self, line):
try:
try:
if line is None:
raise StopIteration
log("token", TRACE, "IN", line)
except StopIteration:
line = ''
log("token", TRACE, "IN_END")
self.lnum = self.lnum + 1
pos, max = 0, len(line)
if self.contstr: # continued string
if not line:
raise TokenError("EOF in multi-line string", strstart)
endmatch = endprog.match(line)
if endmatch:
pos = end = endmatch.end(0)
self.output(STRING, self.contstr + line[:end], strstart,
(self.lnum, end), self.contline + line)
self.contstr, self.needcont = '', 0
self.contline = None
elif self.needcont and line[-2:] != '\\\n' and line[
-3:] != '\\\r\n':
self.output(ERRORTOKEN, self.contstr + line, strstart,
(self.lnum, len(line)), self.contline)
self.contstr = ''
self.contline = None
return
else:
self.contstr += line
self.contline += line
return
elif self.parenlev == 0 and not self.continued: # new statement
if not line:
self.feed_end()
self.output(ENDMARKER, '', (self.lnum, 0), (self.lnum, 0),
'')
return
column = 0
while pos < max: # measure leading whitespace
if line[pos] == ' ': column = column + 1
elif line[pos] == '\t':
column = (column / tabsize + 1) * tabsize
elif line[pos] == '\f':
column = 0
else:
break
pos = pos + 1
if pos == max:
self.feed_end()
return
if line[pos] in '#\r\n': # skip comments or blank lines
self.output((NL, COMMENT)[line[pos] == '#'], line[pos:],
(self.lnum, pos), (self.lnum, len(line)), line)
return
if column > self.indents[-1]: # count indents or dedents
self.indents.append(column)
self.output(INDENT, line[:pos], (self.lnum, 0),
(self.lnum, pos), line)
if column < self.indents[-1]:
while column < self.indents[-1]:
self.indents.pop()
self.output(DEDENT, '', (self.lnum, pos),
(self.lnum, pos), line)
if column != self.indents[-1]:
raise IndentationError(
"unindent does not match any outer indentation level",
(u"‹tokenize›", self.lnum, pos, line))
else: # continued statement
if not line:
raise TokenError("EOF in multi-line statement",
(self.lnum, 0))
self.continued = 0
while pos < max:
pseudomatch = pseudoprog.match(line, pos)
if pseudomatch: # scan for tokens
start, end = pseudomatch.span(1)
spos, epos, pos = (self.lnum, start), (self.lnum, end), end
token, initial = line[start:end], line[start]
if num.match(token) or \
(initial == '.' and token != '.'): # ordinary number
self.output(NUMBER, token, spos, epos, line)
elif initial in '\r\n':
self.output(NL if self.parenlev > 0 else NEWLINE,
token, spos, epos, line)
elif initial == '#':
self.output(COMMENT, token, spos, epos, line)
elif token in triple_quoted:
endprog = endprogs[token]
endmatch = endprog.match(line, pos)
if endmatch: # all on one line
pos = endmatch.end(0)
token = line[start:pos]
self.output(STRING, token, spos, (self.lnum, pos),
line)
else:
strstart = (self.lnum, start) # multiple lines
self.contstr = line[start:]
self.contline = line
break
elif initial in single_quoted or \
token[:2] in single_quoted or \
token[:3] in single_quoted:
if token[-1] == '\n': # continued string
strstart = (self.lnum, start)
endprog = (endprogs[initial] or endprogs[token[1]]
or endprogs[token[2]])
self.contstr, self.needcont = line[start:], 1
self.contline = line
break
else: # ordinary string
self.output(STRING, token, spos, epos, line)
elif namestart.match(initial): # ordinary name
self.output(NAME, token, spos, epos, line)
elif initial == '\\': # continued stmt
self.continued = 1
else:
if initial in '([{': self.parenlev = self.parenlev + 1
elif initial in ')]}':
self.parenlev = self.parenlev - 1
self.output(OP, token, spos, epos, line)
else:
self.output(ERRORTOKEN, line[pos], (self.lnum, pos),
(self.lnum, pos + 1), line)
pos = pos + 1
except StopParsing as e:
fix_exception(e)
self.q = None
if self.parent:
self.parent.kill(e)
return
