Example #1
0
class Observable(object):
    terminator = None

    def __init__(self):
        self.done = Channel()

    def wait_eq(self, terminator, message=None, timeout=None):
        if self.value == terminator:
            return self.value
        self.terminator = terminator
        try:
            with_timeout(timeout, self.done.get)
            return self.value
        except Timeout:
            eq_(self.value, terminator, message)

    def _check(self):
        if self.value == self.terminator:
            self.done.put(None)

    def __eq__(self, x):
        return self.value == x

    def __repr__(self):
        return "%s(%r)" % (type(self).__name__, self.value)
Example #2
0
class Observable(object):
    terminator = None

    def __init__(self):
        self.done = Channel()

    def wait_eq(self, terminator, message=None, timeout=None):
        if self.value == terminator:
            return self.value
        self.terminator = terminator
        try:
            with_timeout(timeout, self.done.get)
            return self.value
        except Timeout:
            eq_(self.value, terminator, message)

    def _check(self):
        if self.value == self.terminator:
            self.done.put(None)

    def __eq__(self, x):
        return self.value == x

    def __repr__(self):
        return "%s(%r)" % (type(self).__name__, self.value)
Example #3
0
def _test_atomic():
    # NOTE: Nested context by comma is not available in Python 2.6.
    # o -- No gevent.
    with lets.atomic():
        1 + 2 + 3
    # x -- gevent.sleep()
    with pytest.raises(AssertionError):
        with lets.atomic():
            gevent.sleep(0.1)
    # x -- gevent.sleep() with 0 seconds.
    with pytest.raises(AssertionError):
        with lets.atomic():
            gevent.sleep(0)
    # o -- Greenlet.spawn()
    with lets.atomic():
        gevent.spawn(gevent.sleep, 0.1)
    # x -- Greenlet.join()
    with pytest.raises(AssertionError):
        with lets.atomic():
            g = gevent.spawn(gevent.sleep, 0.1)
            g.join()
    # x -- Greenlet.get()
    with pytest.raises(AssertionError):
        with lets.atomic():
            g = gevent.spawn(gevent.sleep, 0.1)
            g.get()
    # x -- gevent.joinall()
    with pytest.raises(AssertionError):
        with lets.atomic():
            g = gevent.spawn(gevent.sleep, 0.1)
            gevent.joinall([g])
    # o -- Event.set(), AsyncResult.set()
    with lets.atomic():
        Event().set()
        AsyncResult().set()
    # x -- Event.wait()
    with pytest.raises(AssertionError):
        with lets.atomic():
            Event().wait()
    # x -- Event.wait()
    with pytest.raises(AssertionError):
        with lets.atomic():
            AsyncResult().wait()
    # x -- Channel.put()
    with pytest.raises(AssertionError):
        with lets.atomic():
            ch = Channel()
            ch.put(123)
    # o -- First Semaphore.acquire()
    with lets.atomic():
        lock = Semaphore()
        lock.acquire()
    # x -- Second Semaphore.acquire()
    with pytest.raises(AssertionError):
        with lets.atomic():
            lock = Semaphore()
            lock.acquire()
            lock.acquire()
    # Back to normal.
    gevent.sleep(1)
Example #4
0
    def cpu(self, amount, call_if_blocking):
        """Block until `amount` of cpu, in millicores, is available.

    Requesting 0 cpu will never block or wait.
    Requesting < 0 cpu will raise ValueError.
    Requesting > _millicores_max will acquire the full CPU.

    Args:

      * amount (int) - The amount of millicores to acquire before yielding. Must
        be positive or will raise ValueError. If this exceeds the maximum amount
        of millicores available on the system, this will instead acquire the
        system maximum.
      * call_if_blocking (None|func(amount_blocked_on)) - `cpu` will invoke this
        callback if we would end up blocking before yielding. This callback
        should only be used for reporting/diagnostics (i.e. it shouldn't raise
        an exception.)

    Yields control once the requisite amount of cpu is available.
    """
        if amount < 0:
            raise ValueError('negative cpu amount')

        if amount > self._millicores_max:
            amount = self._millicores_max

        if amount > 0 and (self._waiters
                           or self._millicores_available < amount):
            # we need some amount of cores AND
            # someone else is already waiting, or there aren't enough cores left.
            if call_if_blocking:
                call_if_blocking(amount - self._millicores_available)
            wake_me = Channel()
            self._waiters.append((amount, wake_me))
            wake_me.get()
            # At this point the greenlet that woke us already reserved our cores for
            # us, and we're free to go.
        else:
            # Just directly take our cores.
            assert self._millicores_available >= amount
            self._millicores_available -= amount

        try:
            yield
        finally:
            self._millicores_available += amount
            # We just added some resource back to the pot. Try to wake as many others
            # as we can before proceeding.

            to_wake, to_keep = [], []
            for waiting_amount, chan in self._waiters:
                if waiting_amount <= self._millicores_available:
                    to_wake.append(chan)
                    self._millicores_available -= waiting_amount
                else:
                    to_keep.append((waiting_amount, chan))
            self._waiters = to_keep
            for chan in to_wake:
                chan.put(None)
Example #5
0
 def __init__(self, output, parent=None, endput=None):
     super(tokizer, self).__init__()
     self._output = output
     self._endput = endput
     self.parent = parent
     self.q = Channel()
     self.init()
     self.start_job("job", self._job)
     self.job.link(self._end)
Example #6
0
	def _cmd(self,cmd,*a):
		if self.q is None:
			raise DelayDone(self)

		q = Channel()
		self.q.put((q,cmd)+tuple(a))
		res = q.get()
		if isinstance(res,BaseException):
			raise res
		return res
Example #7
0
 def test_link_to_channel(self):
     p1 = gevent.spawn(lambda: 101)
     p2 = gevent.spawn(lambda: 102)
     p3 = gevent.spawn(lambda: 103)
     q = Channel()
     p1.link(q.put)
     p2.link(q.put)
     p3.link(q.put)
     results = [q.get().get(), q.get().get(), q.get().get()]
     assert sorted(results) == [101, 102, 103], results
Example #8
0
 def test_link_to_channel(self):
     p1 = gevent.spawn(lambda: 101)
     p2 = gevent.spawn(lambda: 102)
     p3 = gevent.spawn(lambda: 103)
     q = Channel()
     p1.link(q.put)
     p2.link(q.put)
     p3.link(q.put)
     results = [q.get().get(), q.get().get(), q.get().get()]
     assert sorted(results) == [101, 102, 103], results
Example #9
0
def try_read_channel(channel: Channel, timeout: int) -> Tuple[bool, Any]:
    try:
        if (timeout == 0):
            item = channel.get(block=False)
        elif (timeout > 0):
            item = channel.get(block=True, timeout=timeout)
        else:
            item = channel.get(block=True)

        return (True, item)
    except QueueEmptyException:
        return (False, None)
Example #10
0
def try_write_channel(channel: Channel, item: Any, timeout: int) -> bool:
    try:
        if (timeout == 0):
            item = channel.put(item, block=False)
        elif (timeout > 0):
            item = channel.put(item, block=True, timeout=timeout)
        else:
            item = channel.put(item, block=True)

        return True
    except QueueFullException:
        return False
Example #11
0
def connection_handler(client_socket: socket, address: Tuple[str, int]):
    '''This greenlet is spawned for each client that connects to the server'''

    logger.info('New connection from %s:%s' % address)

    try:
        output = Channel()
        cancel = Channel()
        r = gevent.spawn(reader, client_socket, output, cancel)
        w = gevent.spawn(writer, client_socket, output, cancel)
        gevent.joinall([r, w])
    finally:
        logger.info('Disconnected %s:%s' % address)
Example #12
0
def test_gevent1():
    global CH
    import gevent
    print gevent, gevent.__version__
    from gevent.queue import Channel
    from gevent import spawn, wait

    CH = Channel()

    start = time()
    spawn(channel_get_put)
    CH.put(True)
    spawn(channel_get_put)
    wait()
    return time() - start
Example #13
0
def test_gevent1():
    global CH
    import gevent
    print gevent, gevent.__version__
    from gevent.queue import Channel
    from gevent import spawn, wait

    CH = Channel()

    start = time()
    spawn(channel_get_put)
    CH.put(True)
    spawn(channel_get_put)
    wait()
    return time() - start
Example #14
0
 def handle_wsgi_request(self, env, start_response):
     ch = Channel()
     req = Request(ch,
                   env,
                   start_response,
                   content_type=self.default_content_type)
     self << ('handle', req)
     return response_stream(ch)
Example #15
0
    def set_links_timeout(self, link):
        # stuff that won't be touched
        event = AsyncResult()
        link(event)

        queue = Channel()
        link(queue.put)
        return event, queue
Example #16
0
	def __init__(self, output, parent=None, endput=None):
		self._output = output
		self._endput = endput
		self.parent = parent
		self.q = Channel()
		self.init()
		self.start_job("job",self._job)
		self.job.link(self._end)
Example #17
0
class Server(Endpoint, Greenlet):
    '''
    Used at client side, to represent server
    '''

    def __init__(self, sock, addr):
        Endpoint.__init__(self, sock, addr)
        Greenlet.__init__(self)
        self.ctlcmds = Channel()
        self.userid = 0
        self.gamedata = Gamedata(recording=True)

    def _run(self):
        while True:
            cmd, data = self.read()
            if cmd == 'gamedata':
                self.gamedata.feed(data)
            else:
                self.ctlcmds.put([cmd, data])

    def gexpect(self, tag, blocking=True):
        return self.gamedata.gexpect(tag, blocking)

    def gbreak(self):
        return self.gamedata.gbreak()

    def gclear(self):
        self.gamedata = Gamedata(recording=True)

    def gwrite(self, tag, data):
        log.debug('GAME_WRITE: %s', repr([tag, data]))
        encoded = self.encode(['gamedata', [tag, data]])
        self.raw_write(encoded)

    def wait_till_live(self):
        self.gamedata.wait_empty()

    def gamedata_piled(self):
        return len(self.gamedata.gdqueue) > 60

    def shutdown(self):
        self.kill()
        self.join()
        self.ctlcmds.put(['shutdown', None])
        self.close()
Example #18
0
 def subscribe(ws):
     try:
         if not channels:
             return
         subscriptions = random.choice(channels)
         ch = Channel()
         subscriptions.append(ch)
         _log("feed ready! - added subscription " + str(len(subscriptions)))
         while True:
             #_log("getting")
             msg = ch.get()
             if not msg:
                 _log("nothing to send!")
                 continue
             ws.send(msg)
             #_log("sent!")
             gevent.sleep(0)
     except WebSocketError:
         _log("WebSocketError on feed! - retrying")
     return
Example #19
0
 def subscribe(ws):
     try:
         if not channels:
             return
         subscriptions = random.choice(channels)
         ch = Channel()
         subscriptions.append(ch)
         _log("feed ready! - added subscription " + str(len(subscriptions)))
         while True:
             #_log("getting")
             msg = ch.get()
             if not msg:
                 _log("nothing to send!")
                 continue
             ws.send(msg)
             #_log("sent!")
             gevent.sleep(0)
     except WebSocketError:
         _log("WebSocketError on feed! - retrying")
     return
Example #20
0
 def __init__(self, context):
     self.context = context
     self.id = None
     self.name = None
     self.nmdm = None
     self.state = VirtualMachineState.STOPPED
     self.config = None
     self.bhyve_process = None
     self.scrollback = io.BytesIO()
     self.console_fd = None
     self.console_channel = Channel()
     self.console_thread = None
     self.logger = logging.getLogger('VM:{0}'.format(self.name))
Example #21
0
class tokizer(Jobber):
	def __init__(self, output, parent=None, endput=None):
		self._output = output
		self._endput = endput
		self.parent = parent
		self.q = Channel()
		self.init()
		self.start_job("job",self._job)
		self.job.link(self._end)
	
	def init(self):
		self.lnum = self.parenlev = self.continued = 0
		self.contstr, self.needcont = '', 0
		self.contline = None
		self.indents = [0]
	
	def exit(self):
		self.stop_job("job")

	def output(self,*a):
		log("token",TRACE,repr(a))
		self._output(*a)
		
	def feed_end(self):
		for indent in self.indents[1:]:                 # pop remaining indent levels
			self.output(DEDENT, '', (self.lnum, 0), (self.lnum, 0), '')

	def feed(self,line):
		if self.q:
			if line is None:
				q,self.q = self.q,None
				if gevent.getcurrent() is not self.job:
					q.put(None)
			else:
				self.q.put(line)
		else:
			raise StopParsing

	def _end(self, res):
		self.feed_end()
		if self.q:
			q,self.q = self.q,None
			try:
				q.get_nowait()
			except Empty:
				pass
		if self._endput:
			self._endput()

	def _job(self):
		line = "x"
		while line:
			if self.q:
				line = self.q.get()
			else:
				line = None
			self._do_line(line)

	def _do_line(self,line):
		try:
			try:
				if line is None:
					raise StopIteration
				log("token",TRACE,"IN",line)
			except StopIteration:
				line = ''
				log("token",TRACE,"IN_END")
			self.lnum = self.lnum + 1
			pos, max = 0, len(line)

			if self.contstr:                            # continued string
				if not line:
					raise TokenError, ("EOF in multi-line string", strstart)
				endmatch = endprog.match(line)
				if endmatch:
					pos = end = endmatch.end(0)
					self.output(STRING, self.contstr + line[:end],
							strstart, (self.lnum, end), self.contline + line)
					self.contstr, self.needcont = '', 0
					self.contline = None
				elif self.needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
					self.output(ERRORTOKEN, self.contstr + line,
							strstart, (self.lnum, len(line)), self.contline)
					self.contstr = ''
					self.contline = None
					return
				else:
					self.contstr += line
					self.contline += line
					return

			elif self.parenlev == 0 and not self.continued:  # new statement
				if not line:
					self.feed_end()
					self.output(ENDMARKER, '', (self.lnum, 0), (self.lnum, 0), '')
					return
				column = 0
				while pos < max:                   # measure leading whitespace
					if line[pos] == ' ': column = column + 1
					elif line[pos] == '\t': column = (column/tabsize + 1)*tabsize
					elif line[pos] == '\f': column = 0
					else: break
					pos = pos + 1
				if pos == max:
					self.feed_end()
					return

				if line[pos] in '#\r\n':           # skip comments or blank lines
					self.output((NL, COMMENT)[line[pos] == '#'], line[pos:],
							(self.lnum, pos), (self.lnum, len(line)), line)
					return

				if column > self.indents[-1]:           # count indents or dedents
					self.indents.append(column)
					self.output(INDENT, line[:pos], (self.lnum, 0), (self.lnum, pos), line)
				if column < self.indents[-1]:
					while column < self.indents[-1]:
						self.indents.pop()
						self.output(DEDENT, '', (self.lnum, pos), (self.lnum, pos), line)

					if column != self.indents[-1]:
						raise IndentationError(
							"unindent does not match any outer indentation level",
							(u"‹tokenize›", self.lnum, pos, line))

			else:                                  # continued statement
				if not line:
					raise TokenError, ("EOF in multi-line statement", (self.lnum, 0))
				self.continued = 0

			while pos < max:
				pseudomatch = pseudoprog.match(line, pos)
				if pseudomatch:                                # scan for tokens
					start, end = pseudomatch.span(1)
					spos, epos, pos = (self.lnum, start), (self.lnum, end), end
					token, initial = line[start:end], line[start]

					if num.match(token) or \
					(initial == '.' and token != '.'):      # ordinary number
						self.output(NUMBER, token, spos, epos, line)
					elif initial in '\r\n':
						self.output(NL if self.parenlev > 0 else NEWLINE, token, spos, epos, line)
					elif initial == '#':
						self.output(COMMENT, token, spos, epos, line)
					elif token in triple_quoted:
						endprog = endprogs[token]
						endmatch = endprog.match(line, pos)
						if endmatch:                           # all on one line
							pos = endmatch.end(0)
							token = line[start:pos]
							self.output(STRING, token, spos, (self.lnum, pos), line)
						else:
							strstart = (self.lnum, start)           # multiple lines
							self.contstr = line[start:]
							self.contline = line
							break
					elif initial in single_quoted or \
						token[:2] in single_quoted or \
						token[:3] in single_quoted:
						if token[-1] == '\n':                  # continued string
							strstart = (self.lnum, start)
							endprog = (endprogs[initial] or endprogs[token[1]] or
									endprogs[token[2]])
							self.contstr, self.needcont = line[start:], 1
							self.contline = line
							break
						else:                                  # ordinary string
							self.output(STRING, token, spos, epos, line)
					elif namestart.match(initial):                 # ordinary name
						self.output(NAME, token, spos, epos, line)
					elif initial == '\\':                      # continued stmt
						self.continued = 1
					else:
						if initial in '([{': self.parenlev = self.parenlev + 1
						elif initial in ')]}': self.parenlev = self.parenlev - 1
						self.output(OP, token, spos, epos, line)
				else:
					self.output(ERRORTOKEN, line[pos],
							(self.lnum, pos), (self.lnum, pos+1), line)
					pos = pos + 1

		except StopParsing as e:
			fix_exception(e)
			self.q = None
			if self.parent:
				self.parent.kill(e)
			return
Example #22
0
class VirtualMachine(object):
    def __init__(self, context):
        self.context = context
        self.id = None
        self.name = None
        self.nmdm = None
        self.state = VirtualMachineState.STOPPED
        self.config = None
        self.bhyve_process = None
        self.scrollback = io.BytesIO()
        self.console_fd = None
        self.console_channel = Channel()
        self.console_thread = None
        self.logger = logging.getLogger('VM:{0}'.format(self.name))

    def get_nmdm(self):
        #for i in range(0, 255):
        #    if os.path.exists('/dev/nmdm{0}A'.format(i)):
        #        continue
        #
        #    a = '/dev/nmdm{0}A'.format(i)
        #    b = '/dev/nmdm{0}B'.format(i)
        #    self.logger.info('Assigned nmdm device pair: {0}, {1}'.format(a, b))
        #    return a, b

        return '/dev/nmdm1A', '/dev/nmdm1B'

    def start(self):
        self.nmdm = self.get_nmdm()

        gevent.spawn(self.run)
        self.console_thread = gevent.spawn(self.console_worker)

    def stop(self):
        if self.state == VirtualMachineState.STOPPED:
            raise RuntimeError()

        gevent.kill(self.console_thread)
        self.bhyve_process.terminate()
        subprocess.call('/usr/sbin/bhyvectl --destroy --vm={0}'.format(self.name))
        self.set_state(VirtualMachineState.STOPPED)

    def set_state(self, state):
        self.state = state
        self.context.client.emit_event('container.changed', {
            'operation': 'update',
            'ids': [self.id]
        })

    def run(self):
        self.set_state(VirtualMachineState.BOOTLOADER)

        if self.config['bootloader'] == 'GRUB':
            self.bhyve_process = subprocess.Popen(
                ['/usr/local/lib/grub-bhyve']
            )

        if self.config['bootloader'] == 'BHYVELOAD':
            self.bhyve_process = subprocess.Popen(
                [
                    '/usr/sbin/bhyveload', '-c', self.nmdm[0], '-m', self.config['memsize'],
                    '-d', self.config['bootdisk'], self.name,
                ],
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                close_fds=True
            )

        out, err = self.bhyve_process.communicate()
        self.logger.debug('bhyveload: {0}'.format(out))

        args = [
            '/usr/sbin/bhyve', '-A', '-H', '-P', '-c', str(self.config['cpus']), '-m', self.config['memsize'],
            '-s', '0:0,hostbridge',
            '-s', '2:0,ahci-cd,{0}'.format(self.config['bootdisk']),
            '-s', '31,lpc', '-l', 'com1,{0}'.format(self.nmdm[0]),
            self.name
        ]

        self.set_state(VirtualMachineState.RUNNING)
        self.bhyve_process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
        out, err = self.bhyve_process.communicate()
        self.logger.debug('bhyve: {0}'.format(out))

    def console_worker(self):
        BUFSIZE = 1024
        self.console_fd = open(self.nmdm[1], 'r+b')
        tty.setraw(self.console_fd.fileno())
        while True:
            data = gevent.os.tp_read(self.console_fd.fileno(), BUFSIZE)
            self.logger.debug('Read: {0}'.format(data))
            if not data:
                self.logger.info('Reopening {0} device'.format(self.nmdm[1]))
                self.console_fd.close()
                gevent.sleep(1)
                self.console_fd = open(self.nmdm[1], 'r+b')
                tty.setraw(self.console_fd.fileno())
                continue

            self.scrollback.write(data)
            try:
                self.console_channel.put(data, block=False)
            except:
                pass

    def console_write(self, data):
        self.logger.debug('Write: {0}'.format(data))
        self.console_fd.write(data)
        self.console_fd.flush()
Example #23
0
	def init(self,dest):
		self.q = Channel()
		self.end = dest
		self.start_job("job",self._job)
Example #24
0
 def __init__(self):
     self.done = Channel()
Example #25
0
 def __init__(self, sock, addr):
     Endpoint.__init__(self, sock, addr)
     Greenlet.__init__(self)
     self.ctlcmds = Channel()
     self.gamedata = Gamedata(recording=True)
Example #26
0
class tokizer(Jobber):
    def __init__(self, output, parent=None, endput=None):
        super(tokizer, self).__init__()
        self._output = output
        self._endput = endput
        self.parent = parent
        self.q = Channel()
        self.init()
        self.start_job("job", self._job)
        self.job.link(self._end)

    def init(self):
        self.lnum = self.parenlev = self.continued = 0
        self.contstr, self.needcont = '', 0
        self.contline = None
        self.indents = [0]

    def exit(self):
        self.stop_job("job")

    def output(self, *a):
        log("token", TRACE, repr(a))
        self._output(*a)

    def feed_end(self):
        for indent in self.indents[1:]:  # pop remaining indent levels
            self.output(DEDENT, '', (self.lnum, 0), (self.lnum, 0), '')

    def feed(self, line):
        if self.q:
            if line is None:
                q, self.q = self.q, None
                if gevent.getcurrent() is not self.job:
                    q.put(None)
            else:
                self.q.put(line)
        else:
            raise StopParsing

    def _end(self, res):
        self.feed_end()
        if self.q:
            q, self.q = self.q, None
            try:
                q.get_nowait()
            except Empty:
                pass
        if self._endput:
            self._endput()

    def _job(self):
        line = "x"
        while line:
            if self.q:
                line = self.q.get()
            else:
                line = None
            self._do_line(line)

    def _do_line(self, line):
        try:
            try:
                if line is None:
                    raise StopIteration
                log("token", TRACE, "IN", line)
            except StopIteration:
                line = ''
                log("token", TRACE, "IN_END")
            self.lnum = self.lnum + 1
            pos, max = 0, len(line)

            if self.contstr:  # continued string
                if not line:
                    raise TokenError("EOF in multi-line string", strstart)
                endmatch = endprog.match(line)
                if endmatch:
                    pos = end = endmatch.end(0)
                    self.output(STRING, self.contstr + line[:end], strstart,
                                (self.lnum, end), self.contline + line)
                    self.contstr, self.needcont = '', 0
                    self.contline = None
                elif self.needcont and line[-2:] != '\\\n' and line[
                        -3:] != '\\\r\n':
                    self.output(ERRORTOKEN, self.contstr + line, strstart,
                                (self.lnum, len(line)), self.contline)
                    self.contstr = ''
                    self.contline = None
                    return
                else:
                    self.contstr += line
                    self.contline += line
                    return

            elif self.parenlev == 0 and not self.continued:  # new statement
                if not line:
                    self.feed_end()
                    self.output(ENDMARKER, '', (self.lnum, 0), (self.lnum, 0),
                                '')
                    return
                column = 0
                while pos < max:  # measure leading whitespace
                    if line[pos] == ' ': column = column + 1
                    elif line[pos] == '\t':
                        column = (column / tabsize + 1) * tabsize
                    elif line[pos] == '\f':
                        column = 0
                    else:
                        break
                    pos = pos + 1
                if pos == max:
                    self.feed_end()
                    return

                if line[pos] in '#\r\n':  # skip comments or blank lines
                    self.output((NL, COMMENT)[line[pos] == '#'], line[pos:],
                                (self.lnum, pos), (self.lnum, len(line)), line)
                    return

                if column > self.indents[-1]:  # count indents or dedents
                    self.indents.append(column)
                    self.output(INDENT, line[:pos], (self.lnum, 0),
                                (self.lnum, pos), line)
                if column < self.indents[-1]:
                    while column < self.indents[-1]:
                        self.indents.pop()
                        self.output(DEDENT, '', (self.lnum, pos),
                                    (self.lnum, pos), line)

                    if column != self.indents[-1]:
                        raise IndentationError(
                            "unindent does not match any outer indentation level",
                            (u"‹tokenize›", self.lnum, pos, line))

            else:  # continued statement
                if not line:
                    raise TokenError("EOF in multi-line statement",
                                     (self.lnum, 0))
                self.continued = 0

            while pos < max:
                pseudomatch = pseudoprog.match(line, pos)
                if pseudomatch:  # scan for tokens
                    start, end = pseudomatch.span(1)
                    spos, epos, pos = (self.lnum, start), (self.lnum, end), end
                    token, initial = line[start:end], line[start]

                    if num.match(token) or \
                    (initial == '.' and token != '.'):      # ordinary number
                        self.output(NUMBER, token, spos, epos, line)
                    elif initial in '\r\n':
                        self.output(NL if self.parenlev > 0 else NEWLINE,
                                    token, spos, epos, line)
                    elif initial == '#':
                        self.output(COMMENT, token, spos, epos, line)
                    elif token in triple_quoted:
                        endprog = endprogs[token]
                        endmatch = endprog.match(line, pos)
                        if endmatch:  # all on one line
                            pos = endmatch.end(0)
                            token = line[start:pos]
                            self.output(STRING, token, spos, (self.lnum, pos),
                                        line)
                        else:
                            strstart = (self.lnum, start)  # multiple lines
                            self.contstr = line[start:]
                            self.contline = line
                            break
                    elif initial in single_quoted or \
                     token[:2] in single_quoted or \
                     token[:3] in single_quoted:
                        if token[-1] == '\n':  # continued string
                            strstart = (self.lnum, start)
                            endprog = (endprogs[initial] or endprogs[token[1]]
                                       or endprogs[token[2]])
                            self.contstr, self.needcont = line[start:], 1
                            self.contline = line
                            break
                        else:  # ordinary string
                            self.output(STRING, token, spos, epos, line)
                    elif namestart.match(initial):  # ordinary name
                        self.output(NAME, token, spos, epos, line)
                    elif initial == '\\':  # continued stmt
                        self.continued = 1
                    else:
                        if initial in '([{': self.parenlev = self.parenlev + 1
                        elif initial in ')]}':
                            self.parenlev = self.parenlev - 1
                        self.output(OP, token, spos, epos, line)
                else:
                    self.output(ERRORTOKEN, line[pos], (self.lnum, pos),
                                (self.lnum, pos + 1), line)
                    pos = pos + 1

        except StopParsing as e:
            fix_exception(e)
            self.q = None
            if self.parent:
                self.parent.kill(e)
            return
Example #27
0
 def __init__(self):
     self.done = Channel()
Example #28
0
class Waiter(Collected,Jobber):
	"""This is the thing that waits."""
	force = False
	storage = Waiters.storage
	_plinger = None
	_running = False
	q = None

	def __init__(self,parent,name,force):
		self.ctx = parent.ctx
		self.start = now()
		self.force = force
		try:
			self.parent = parent.parent
		except AttributeError:
			pass
		super(Waiter,self).__init__(name)
	
	def list(self):
		end=now()+dt.timedelta(0,self.value)
		yield super(Waiter,self)
		yield("start",self.start)
		yield("end",end)
		yield("total", humandelta(end-self.start))
		w = self
		while True:
			w = getattr(w,"parent",None)
			if w is None: break
			n = getattr(w,"displayname",None)
			if n is not None:
				if not isinstance(n,basestring):
					n = " ".join(unicode(x) for x in n)
			else:
				try:
					if w.args:
						n = unicode(w.args)
				except AttributeError:
					pass
				if n is None:
					try:
						if isinstance(w.name,basestring):
							n = w.name
						else:
							n = " ".join(unicode(x) for x in w.name)
					except AttributeError:
						n = w.__class__.__name__
			if n is not None:
				yield("in",n)

	def info(self):
		return str(self.value)

	def __repr__(self):
		return u"‹%s %s %s›" % (self.__class__.__name__, self.name,self.value)

	def _pling(self):
		self._plinger = None
		self._cmd("timeout")

	def _set_pling(self):
		timeout = unixtime(self.end) - unixtime(now(self.force))
		if timeout <= 0.1:
			timeout = 0.1
		if self._plinger:
			self._plinger.cancel()
		self._plinger = callLater(self.force, timeout, self._pling)
		
	def _job(self):
		try:
			self._set_pling()
			self._running = True
			while True:
				cmd = self.q.get()
	
				q = cmd[0]
				a = cmd[2] if len(cmd)>2 else None
				cmd = cmd[1]
				if cmd == "timeout":
					assert self._plinger is None
					q.put(None)
					return True
				elif cmd == "cancel":
					if self._plinger:
						self._plinger.cancel()
						self._plinger = None
					q.put(None)
					return False
				elif cmd == "update":
					q.put(None)
					self.end = a
					self._set_pling()
				elif cmd == "remain":
					q.put(unixtime(self.end)-unixtime(now(self.force)))
				else:
					q.put(RuntimeError('Unknown command: '+cmd))
		finally:
			q,self.q = self.q,None
			super(Waiter,self).delete()
			if q is not None:
				while not q.empty():
					q.get()[0].put(StopIteration())

	def init(self,dest):
		self.q = Channel()
		self.end = dest
		self.start_job("job",self._job)

	def _cmd(self,cmd,*a):
		if self.q is None:
			raise DelayDone(self)

		q = Channel()
		self.q.put((q,cmd)+tuple(a))
		res = q.get()
		if isinstance(res,BaseException):
			raise res
		return res

	@property
	def value(self):
		if self.q is None:
			return 0
		if not self._running:
			return "??"
		res = self._cmd("remain")
		if TESTING:
			res = "%.1f" % (res,)
		return res

	def delete(self,ctx=None):
		self._cmd("cancel")

	def cancel(self, err=DelayCancelled):
		"""Cancel a waiter."""
		process_event(Event(self.ctx(loglevel=TRACE),"wait","cancel",ixtime(self.end,self.force),*self.name))
		self._cmd("cancel")

	def retime(self, dest):
		process_event(Event(self.ctx(loglevel=TRACE),"wait","update",dest,*self.name))
		self._cmd("update",dest)