Ejemplo n.º 1
0
def shoot(cfgs, status_callback):
    """
    Performs multi-tank multi-config test.
    Accepts one or more config dicts.
    Returns list of session ID's.
    Raises TankLocked and TestFailed.
    """

    try:
        sessions = [
            SessionWrapper(ft.partial(status_callback, i), **cfg) for i, cfg in enumerate(cfgs)  # pylint: disable=W0142
        ]
    except Exception:
        logger.exception("Failed to initialize session objects, config:\n%s", yaml.safe_dump(cfgs))
        raise
    prepares = []
    runs = []
    stops = []
    try:
        try:
            prepares = [async(session.prepare()) for session in sessions]
            yield From(gather(*prepares))  # pylint: disable=W0142
            logger.info("All tanks are prepared. STARTING TO SHOOT.")
            runs = [async(session.run_until_finish()) for session in sessions]
            yield From(gather(*runs))  # pylint: disable=W0142
        except KeyboardInterrupt:
            logger.info("Test interrupted")
            raise
        except CancelledError:
            logger.info("Test cancelled")
            raise
        except TestFailed:
            logger.info("Test failed")
            raise
        except Exception:
            logger.exception("Exception occured in Test.run_until_finish()")
            raise
        except BaseException:
            logger.exception("Something strange caught by Test.run_until_finish()")
            raise
    except BaseException as ex:
        logger.info("Stopping remaining tank sessions...")
        stops = [async(session.stop()) for session in sessions if not session.finished]
        yield From(gather(*stops, return_exceptions=True))  # pylint: disable=W0142
        raise ex
    finally:
        for task in prepares + runs + stops:
            task.cancel()
    logger.info("All tanks are done.")
    raise Return([session.session.s_id if session.session is not None else None for session in sessions])
Ejemplo n.º 2
0
Archivo: worker.py Proyecto: gridl/dist
    def go(self):
        coroutines = [
            work(self.work_q, self.send_q, self.data, self.metadata_addr,
                 self.address, self.loop),
            control(
                self.control_q, {
                    'compute': self.work_q,
                    'send': self.send_q,
                    'get-data': self.data_q,
                    'put-data': self.data_q,
                    'delete-data': self.data_q,
                    'ping': self.pingpong_q
                }),
            send(self.send_q, self.outgoing_q, self.signal_q),
            pingpong(self.pingpong_q, self.send_q),
            comm(self.ip, self.port, self.bind_ip, self.signal_q,
                 self.control_q, self.outgoing_q, self.loop, self.context),
            manage_data(self.data_q, self.send_q, self.data,
                        self.metadata_addr, self.address)
        ]

        try:
            yield From(
                asyncio.wait(coroutines, return_when=asyncio.FIRST_COMPLETED))
        except:
            import pdb
            pdb.set_trace()
        finally:
            self.close()

        print("Closing")

        yield From(asyncio.gather(*coroutines))
Ejemplo n.º 3
0
 def config(self, key=None, **kw):
     contents = yield From(self.list_backend(prefix=self.prefixify(key)))
     contents = contents.get('Contents', [])
     tasks = [self.get(key=obj['Key'], **kw) for obj in contents]
     results = yield From(asyncio.gather(*tasks))
     keys = [k['Key'].split('/')[-1] for k in contents]
     raise Return(OrderedDict(zip(keys, results)))
Ejemplo n.º 4
0
def test_compute():
    with everything() as (loop, c, w, sock):
        w.data['x'] = 123
        c.who_has['x'].add(w.address)
        c.has_what[w.address].add('x')

        @asyncio.coroutine
        def f():
            msg = {
                'op': 'compute',
                'key': 'y',
                'function': add,
                'args': ('x', 10),
                'needed': ['x'],
                'reply': True
            }
            for i in range(3):
                sock.send(dumps(msg))
                result = yield From(delay(loop, sock.recv))
                assert loads(result) == {
                    'op': 'computation-finished',
                    'key': 'y'
                }

            w.close()
            c.close()

        loop.run_until_complete(asyncio.gather(w.go(), c.go(), f()))
Ejemplo n.º 5
0
def test_remote_gather():
    with Loop() as loop, center(loop=loop) as c, worker(
            metadata_port=c.port, port=1234,
            loop=loop) as a, worker(metadata_port=c.port, port=4321,
                                    loop=loop) as b, dealer(a.address) as sock:

        # Put 'x' in b's data.  Register with metadata store
        b.data['x'] = 123
        c.who_has['x'].add(b.address)
        c.has_what[b.address].add('x')

        @asyncio.coroutine
        def f():
            msg = {
                'op': 'compute',
                'key': 'y',
                'function': add,
                'args': ('x', 10),
                'needed': ['x'],
                'reply': True
            }

            sock.send(dumps(msg))  # send to a, will need to get from b
            result = yield From(delay(loop, sock.recv))
            assert loads(result) == {'op': 'computation-finished', 'key': 'y'}

            a.close()
            b.close()
            c.close()

        loop.run_until_complete(asyncio.gather(a.go(), b.go(), c.go(), f()))
        assert a.data['y'] == 10 + 123
        assert c.who_has['y'] == set([a.address])
        assert 'y' in c.has_what[a.address]
Ejemplo n.º 6
0
def loop_ips(ip, log=False):
    """ looping through the ips from ch_ip till 255 and detect using
    det_ccast and returning a list of potential chromecasts """
    active_ccasts = []  # detected chrome casts stored here
    loop = asyncio.get_event_loop()
    if version.startswith('3'):  # if python3 import it, to avoid syntax issues
        from chrome_cut.fix3 import det_ccast as det_ccast3
        tasks = [
            det_ccast3('.'.join(ip.split('.')[0:-1]) + '.' + str(i), log)
            for i in range(1, 256)
        ]
    else:
        tasks = [
            det_ccast(  # fetching the range of ips to async function
                '.'.join(ip.split('.')[0:-1]) + '.' + str(i), log)
            for i in range(1, 256)
        ]
    results = loop.run_until_complete(asyncio.gather(asyncio.wait(tasks)))
    #  loop.close() should be stopped in the before exist
    #  FIXME: register loop.close() to before exit
    register(loop.close)
    for result in results[0][0]:  # looking for successful ones
        global counter
        counter = 0  # clearing up the global counter
        if result.result()[0]:
            active_ccasts.append(result.result()[1])
    return active_ccasts if len(active_ccasts) >= 1 else None
Ejemplo n.º 7
0
    def set_waveform(key, value):
        exception = None
        result = signals.signal('set-%s' % key).send(value)
        if result:
            try:
                receivers, co_callbacks = zip(*result)
                if receivers:
                    results = yield asyncio.From(asyncio.gather(*co_callbacks))
            except asyncio.CancelledError:
                raise
            except Exception as exception:
                pass
            else:
                if receivers:
                    raise asyncio.Return(zip(receivers, results))

        if exception is not None:
            message = ('Error setting **%s**: `%s`' % (key, exception))
        else:
            message = ('No waveform generators available to set **%s**.' % key)

        yield asyncio.From(
            _warning(signals.signal('warning'),
                     message,
                     title='Warning: failed to set %s' % key,
                     key='waveform-%s' % key))
Ejemplo n.º 8
0
 def config(self, key=None, **kw):
     contents = yield From(self.list_backend(prefix=self.prefixify(key)))
     contents = contents.get('Contents', [])
     kw.setdefault('in_group_check', True)
     tasks = [self.get(key=obj['Key'], **kw) for obj in contents]
     try:
         results = yield From(asyncio.gather(*tasks))
     except ValueError as e:
         raise Return({})
     keys = [k['Key'].split('/')[-1] for k in contents]
     raise Return(OrderedDict(zip(keys, results)))
Ejemplo n.º 9
0
def test_get_data():
    with everything() as (loop, c, w, sock):
        w.data['x'] = 123

        @asyncio.coroutine
        def f():
            msg = {'op': 'get-data', 'keys': ['x']}
            for i in range(3):
                sock.send(dumps(msg))
                result = yield From(delay(loop, sock.recv))
                assert loads(result) == {'x': 123}
            w.close()
            c.close()

        loop.run_until_complete(asyncio.gather(w.go(), c.go(), f()))
Ejemplo n.º 10
0
def _main_task_done(main_task):
    """Callback to be called when the top-level coroutine dies"""
    if main_task.cancelled():
        logger.info("Main task cancelled")
    else:
        ex = main_task.exception()
        if ex is None:
            logger.info("Main task finished")
        else:
            logger.error("Main task failed")
    logger.info("Finding remaining tasks...")
    pending = set(task for task in asyncio.Task.all_tasks() if not task.done())
    logger.info("Waiting for %s remaining tasks to terminate", len(pending))
    gth = asyncio.gather(*pending, return_exceptions=True)
    gth.add_done_callback(_stop_loop)
Ejemplo n.º 11
0
def test_Worker():
    with Loop() as loop, center(loop) as c, worker(metadata_port=c.port,
                                                   loop=loop) as w, dealer(
                                                       w.address) as sock:

        @asyncio.coroutine
        def f():
            msg = {'op': 'ping'}
            for i in range(3):
                sock.send(dumps(msg))
                result = yield From(delay(loop, sock.recv))
                assert result == b'pong'

            w.close()

        loop.run_until_complete(asyncio.gather(w.go(), f()))
Ejemplo n.º 12
0
Archivo: worker.py Proyecto: gridl/dist
def get_remote_data(loop, keys, metadata_addr):
    msg = {'op': 'who-has', 'keys': keys}
    who_has = yield From(dealer_send_recv(loop, metadata_addr, msg))

    lost = set(keys) - set(k for k, v in who_has.items() if v)
    if lost:
        raise KeyError("Missing keys {%s}" % ', '.join(map(str, lost)))

    # get those keys from remote sources
    print("Collecting %s" % who_has)
    coroutines = [
        get_datum(loop, random.choice(list(who_has[k])), [k]) for k in keys
    ]
    result = yield From(asyncio.gather(*coroutines))

    raise Return(merge(result))
Ejemplo n.º 13
0
def test_center():
    with Loop() as loop:
        with center(loop) as c:
            with dealer(c.address) as sock:

                @asyncio.coroutine
                def f():
                    msg = {
                        'op': 'register',
                        'address': 'hank',
                        'keys': ['x', 'y'],
                        'reply': True
                    }
                    sock.send(dumps(msg))
                    ack = yield From(delay(loop, sock.recv))
                    assert ack == b'OK'
                    assert 'hank' in c.who_has['x']
                    assert 'hank' in c.who_has['y']
                    assert c.has_what['hank'] == set(['x', 'y'])

                    msg = {'op': 'who-has', 'keys': ['x']}
                    sock.send(dumps(msg))
                    result = yield From(delay(loop, sock.recv))
                    assert loads(result) == {'x': set(['hank'])}

                    msg = {'op': 'list', 'number': 0}
                    sock.send(dumps(msg))
                    result = yield From(delay(loop, sock.recv))
                    assert loads(result) == set(['hank'])

                    msg = {
                        'op': 'unregister',
                        'address': 'hank',
                        'keys': ['x'],
                        'reply': True
                    }
                    sock.send(dumps(msg))
                    result = yield From(delay(loop, sock.recv))
                    assert c.who_has['x'] == set()
                    assert c.who_has['y'] == set(['hank'])
                    assert c.has_what['hank'] == set(['y'])

                    c.close()

                loop.run_until_complete(asyncio.gather(c.go(), f()))
Ejemplo n.º 14
0
def main():
    asyncio.set_event_loop(None)
    if args.iocp:
        from trollius.windows_events import ProactorEventLoop
        loop = ProactorEventLoop()
    else:
        loop = asyncio.new_event_loop()
    sslctx = None
    if args.tls:
        sslctx = test_utils.dummy_ssl_context()
    cache = CacheClient(args.host, args.port, sslctx=sslctx, loop=loop)
    try:
        loop.run_until_complete(
            asyncio.gather(
                *[testing(i, cache, loop) for i in range(args.ntasks)],
                loop=loop))
    finally:
        loop.close()
Ejemplo n.º 15
0
 def config(self, key=None, **kw):
     args = prepare()
     contents = yield From(self.list_backend(prefix=self.prefixify(key)))
     contents = contents.get('Contents', [])
     kw.setdefault('in_group_check', False)
     kw.setdefault('no_files', True)
     tasks = [self._get(key=obj['Key'], **kw) for obj in contents]
     try:
         results = yield From(asyncio.gather(*tasks))
     except ValueError as e:
         raise Return({})
     result = OrderedDict()
     project_env = "{}/{}".format(args.project, args.env)
     for k, v in enumerate(results):
         if kw.get('in_group_check'):  # for grouped keys strip prefixes
             key_ = contents[k]['Key'].split('/')[-1]
         else:  # otherwise full key
             key_ = contents[k]['Key'].replace(project_env, '').lstrip('/')
         if args.skip_files and v['is_file']:
             continue
         result[key_] = v['data']
     raise Return(result)
Ejemplo n.º 16
0
 def config(self, key=None, **kw):
     args = prepare()
     contents = yield From(self.list_backend(prefix=self.prefixify(key)))
     contents = contents.get('Contents', [])
     kw.setdefault('in_group_check', False)
     kw.setdefault('no_files', True)
     tasks = [self._get(key=obj['Key'], **kw) for obj in contents]
     try:
         results = yield From(asyncio.gather(*tasks))
     except ValueError as e:
         raise Return({})
     result = OrderedDict()
     project_env = "{}/{}".format(args.project, args.env)
     for k, v in enumerate(results):
         if kw.get('in_group_check'): # for grouped keys strip prefixes
             key_ = contents[k]['Key'].split('/')[-1]
         else: # otherwise full key
             key_ = contents[k]['Key'].replace(project_env, '').lstrip('/')
         if args.skip_files and v['is_file']:
             continue
         result[key_] = v['data']
     raise Return(result)
Ejemplo n.º 17
0
    def run():
        connection = yield From(
            trollius_redis.Pool.create(host=u'localhost',
                                       port=6379,
                                       poolsize=50))

        try:
            # === Benchmark 1 ==
            print(u'1. How much time does it take to set 10,000 values '
                  u'in Redis? (without pipelining)')
            print(u'Starting...')
            start = time.time()

            # Do 10,000 set requests
            for i in range(10 * 1000):
                # By using yield from here, we wait for the answer.
                yield From(connection.set(u'key', u'value'))

            print(u'Done. Duration=', time.time() - start)
            print()

            # === Benchmark 2 (should be at least 3x as fast) ==

            print(u'2. How much time does it take if we use asyncio.gather, '
                  u'and pipeline requests?')
            print(u'Starting...')
            start = time.time()

            # Do 10,000 set requests
            futures = [
                asyncio.Task(connection.set(u'key', u'value'))
                for x in range(10 * 1000)
            ]
            yield From(asyncio.gather(*futures))

            print(u'Done. Duration=', time.time() - start)

        finally:
            connection.close()
Ejemplo n.º 18
0
Archivo: center.py Proyecto: gridl/dist
    def go(self):
        coroutines = [
                control(self.control_q, {'who-has': self.metadata_q,
                                         'register': self.metadata_q,
                                         'unregister': self.metadata_q,
                                         'list': self.metadata_q,
                                         'ping': self.pingpong_q,
                                         'send': self.send_q}),
                send(self.send_q, self.outgoing_q, self.signal_q),
                pingpong(self.pingpong_q, self.send_q),
                comm(self.ip, self.port, self.bind_ip, self.signal_q,
                     self.control_q, self.outgoing_q, self.loop, self.context),
                metadata(self.metadata_q, self.send_q,
                         self.who_has, self.has_what)
            ]

        first, other = yield From(asyncio.wait(coroutines,
                                return_when=asyncio.FIRST_COMPLETED))

        print("Closing")

        yield From(asyncio.gather(*other))
Ejemplo n.º 19
0
def _warning(signal, message, **kwargs):
    '''
    XXX Coroutine XXX

    Send warning signal and gather response from receivers.

    If no receivers are available or any receiver raises an exception, raise
    the specified warning message as a `RuntimeError`.

    Raises
    ------
    RuntimeError
        If no receivers are available or any receiver raises an exception.
    '''
    responses = signal.send(message, **kwargs)

    try:
        receivers, co_callbacks = zip(*responses)
        results = yield asyncio.From(asyncio.gather(*co_callbacks))
    except Exception:
        raise RuntimeError(message)
    else:
        raise asyncio.Return(zip(receivers, results))
Ejemplo n.º 20
0
def test_no_data_found():
    with everything() as (loop, c, w, sock):

        @asyncio.coroutine
        def f():
            msg = {
                'op': 'compute',
                'key': 'y',
                'function': add,
                'args': ('asdf', 10),
                'needed': ['asdf'],
                'reply': True
            }
            sock.send(dumps(msg))
            result = yield From(delay(loop, sock.recv))
            result = loads(result)
            assert result['op'] == 'computation-failed'
            assert isinstance(result['error'], KeyError)
            assert 'asdf' in str(result['error'])

            w.close()
            c.close()

        loop.run_until_complete(asyncio.gather(w.go(), c.go(), f()))
Ejemplo n.º 21
0
    def on_actuation_request(electrode_states, duration_s=0):
        '''
        XXX Coroutine XXX

        Actuate electrodes according to specified states.

        Parameters
        ----------
        electrode_states : pandas.Series
        duration_s : float, optional
            If ``volume_threshold`` step option is set, maximum duration before
            timing out.  Otherwise, time to actuate before actuation is
            considered completed.

        Returns
        -------
        actuated_electrodes : list
            List of actuated electrode IDs.
        '''
        try:
            result = yield asyncio\
                .From(actuate(proxy, dmf_device,
                              electrode_states, duration_s=duration_s,
                              volume_threshold=volume_threshold,
                              c_unit_area=c_unit_area))
            # Notify other plugins that actuation completed.
            responses = signals.signal('actuation-completed')\
                .send('dropbot_plugin', **result)
            yield asyncio.From(asyncio.gather(*(r[1] for r in responses)))
        except asyncio.CancelledError:
            raise
        except:
            logging.info('on_actuation_request', exc_info=True)
            raise
        else:
            raise asyncio.Return(result['actuated_electrodes'])
Ejemplo n.º 22
0
    def run():
        connection = yield From(trollius_redis.Pool.create(
            host=u'localhost', port=6379, poolsize=50))

        try:
            # === Benchmark 1 ==
            print(
                u'1. How much time does it take to set 10,000 values '
                u'in Redis? (without pipelining)')
            print(u'Starting...')
            start = time.time()

            # Do 10,000 set requests
            for i in range(10 * 1000):
                # By using yield from here, we wait for the answer.
                yield From(connection.set(u'key', u'value'))

            print(u'Done. Duration=', time.time() - start)
            print()

            # === Benchmark 2 (should be at least 3x as fast) ==

            print(u'2. How much time does it take if we use asyncio.gather, '
                  u'and pipeline requests?')
            print(u'Starting...')
            start = time.time()

            # Do 10,000 set requests
            futures = [asyncio.Task(connection.set(u'key', u'value')) for x
                       in range(10*1000)]
            yield From(asyncio.gather(*futures))

            print(u'Done. Duration=', time.time() - start)

        finally:
            connection.close()
Ejemplo n.º 23
0
 def test_gather(self):
     fut = asyncio.Future()
     fut.set_result("ok")
     results = self.loop.run_until_complete(trollius.gather(fut))
     self.assertEqual(results, ["ok"])
Ejemplo n.º 24
0
def execute_actuation(signals, static_states, dynamic_states, voltage,
                      frequency, duration_s):
    '''
    XXX Coroutine XXX

    Execute specified *static* and *dynamic* electrode actuations.

    Parameters
    ----------
    signals : blinker.Namespace
        Signals namespace.
    static_states : pandas.Series
        Static electrode actuation states, indexed by electrode ID, (e.g.,
        `"electrode001"`).
    dynamic_states : pandas.Series
        Dynamic electrode actuation states, indexed by electrode ID.
    voltage : float
        Actuation amplitude as RMS AC voltage (in volts).
    frequency : float
        Actuation frequency (in Hz).
    duration_s : float
        Actuation duration (in seconds).  If not specified, use value from
        step options.

    Returns
    -------
    dict
        Response with fields:

        - ``start``: actuation start timestamp (`datetime.datetime`).
        - ``end``: actuation start timestamp (`datetime.datetime`).
        - ``actuated_electrodes``: actuated electrode IDs (`list`).

    See Also
    --------
    execute_actuations


    .. versionchanged:: 2.25
        Still apply for specified duration even if _no electrodes_ are
        specified for actuation.

    .. versionchanged:: 2.28.2
        Allow user to optionally ignore failed actuations.

    .. versionchanged:: 2.30
        Add `signals`, `voltage` and `frequency` parameters.  Refactor the
        to set waveform parameters using ``signals`` namespace instead.

    .. versionchanged:: 2.30
        Refactor electrode actuation requests to use :data:`signals`
        interface instead of using pyutilib :func:`emit_signal()`.

    .. versionchanged:: 2.31.1
        Prevent error dialog prompt if coroutine is cancelled while calling
        ``set_waveform()`` callbacks.
    '''
    # Notify other plugins that dynamic electrodes states have changed.
    responses = (signals.signal('dynamic-electrode-states-changed').send(
        NAME, electrode_states=dynamic_states))
    yield asyncio.From(asyncio.gather(*(r[1] for r in responses)))

    static_electrodes_to_actuate = set(static_states[static_states > 0].index)
    dynamic_electrodes_to_actuate = set(
        dynamic_states[dynamic_states > 0].index)

    electrodes_to_actuate = (dynamic_electrodes_to_actuate
                             | static_electrodes_to_actuate)

    # Execute `set_electrode_states` command through ZeroMQ plugin
    # API to notify electrode actuator plugins (i.e., plugins
    # implementing the `IElectrodeActuator` interface) of the
    # electrodes to actuate.
    s_electrodes_to_actuate = \
        pd.Series(True, index=sorted(electrodes_to_actuate))

    @asyncio.coroutine
    def set_waveform(key, value):
        exception = None
        result = signals.signal('set-%s' % key).send(value)
        if result:
            try:
                receivers, co_callbacks = zip(*result)
                if receivers:
                    results = yield asyncio.From(asyncio.gather(*co_callbacks))
            except asyncio.CancelledError:
                raise
            except Exception as exception:
                pass
            else:
                if receivers:
                    raise asyncio.Return(zip(receivers, results))

        if exception is not None:
            message = ('Error setting **%s**: `%s`' % (key, exception))
        else:
            message = ('No waveform generators available to set **%s**.' % key)

        yield asyncio.From(
            _warning(signals.signal('warning'),
                     message,
                     title='Warning: failed to set %s' % key,
                     key='waveform-%s' % key))

    for key, value, unit in (('frequency', frequency, 'Hz'), ('voltage',
                                                              voltage, 'V')):
        waveform_result = yield asyncio.From(set_waveform(key, value))

        if waveform_result:
            _L().info('%s set to %s%s (receivers: `%s`)', key,
                      si.si_format(value), unit,
                      zip(*waveform_result)[0])

    electrode_actuators = signals.signal('on-actuation-request')\
        .send(s_electrodes_to_actuate, duration_s=duration_s)

    if not electrode_actuators:
        title = 'Warning: failed to actuate all electrodes'
        message = ('No electrode actuators registered to **actuate**: `%s`' %
                   list(electrodes_to_actuate))

        yield asyncio.From(
            _warning(signals.signal('warning'),
                     message,
                     title=title,
                     key='no-actuators'))

        # Simulate actuation by waiting for specified duration.
        yield asyncio.From(asyncio.sleep(duration_s))
    else:
        actuation_tasks = zip(*electrode_actuators)[1]

        # Wait for actuations to complete.
        start = dt.datetime.now()
        done, pending = yield asyncio.From(asyncio.wait(actuation_tasks))
        end = dt.datetime.now()

        actuated_electrodes = set()

        exceptions = []

        for d in done:
            try:
                actuated_electrodes.update(d.result())
            except Exception as exception:
                # Actuation error occurred.  Save exception and check
                # remaining responses from actuators.
                exceptions.append(exception)

        if (electrodes_to_actuate - actuated_electrodes) or exceptions:

            def _error_message():
                missing_electrodes = (electrodes_to_actuate -
                                      actuated_electrodes)
                messages = []

                if missing_electrodes:
                    messages.append('Failed to actuate the following '
                                    'electrodes: %s' %
                                    ', '.join('`%s`' % e
                                              for e in missing_electrodes))
                if len(exceptions) == 1:
                    messages.append('**Actuation error:** `%s`' %
                                    exceptions[0])
                elif exceptions:
                    messages.append('**Actuation errors:**\n%s' %
                                    '\n'.join(' - ' + '`%s`' % e
                                              for e in exceptions))
                return '\n\n'.join(messages)

            # Send `'warning'` signal to give other plugins an opportunity
            # to handle the warning.
            yield asyncio.From(
                _warning(signals.signal('warning'),
                         _error_message(),
                         title='Warning:'
                         ' actuation error',
                         key='actuation-error'))
            _L().info('Ignored actuation error(s): `%s`', exceptions)
            # Simulate actuation by waiting for remaining duration.
            remaining_duration = (duration_s -
                                  (dt.datetime.now() - start).total_seconds())
            if remaining_duration > 0:
                yield asyncio.From(asyncio.sleep(remaining_duration))
        else:
            # Requested actuations were completed successfully.
            _L().info('actuation completed (actuated electrodes: %s)',
                      actuated_electrodes)

        raise asyncio.Return({
            'start': start,
            'end': end,
            'actuated_electrodes': sorted(actuated_electrodes)
        })
Ejemplo n.º 25
0
 def co_callback(message):
     listeners = signals.signal(name).send(
         'keep_alive', **message)
     yield asyncio.From(
         asyncio.gather(*(l[1] for l in listeners)))
Ejemplo n.º 26
0
def main():
    def make_stream(sources):
        bug_compat = spead2.BUG_COMPAT_PYSPEAD_0_5_2 if args.pyspead else 0
        stream = spead2.recv.trollius.Stream(thread_pool, bug_compat,
                                             args.heaps, args.ring_heaps)
        if memory_pool is not None:
            stream.set_memory_allocator(memory_pool)
        if args.memcpy_nt:
            stream.set_memcpy(spead2.MEMCPY_NONTEMPORAL)
        ibv_endpoints = []
        for source in sources:
            try:
                if ':' in source:
                    host, port = source.rsplit(':', 1)
                    port = int(port)
                else:
                    host = ''
                    port = int(source)
            except ValueError:
                try:
                    stream.add_udp_pcap_file_reader(source)
                except AttributeError:
                    raise RuntimeError(
                        'spead2 was compiled without pcap support')
            else:
                if args.tcp:
                    stream.add_tcp_reader(port, args.packet, args.buffer, host)
                elif 'ibv' in args and args.ibv:
                    if host is None:
                        raise ValueError(
                            'a multicast group is required when using --ibv')
                    ibv_endpoints.append((host, port))
                elif args.bind and host:
                    stream.add_udp_reader(host, port, args.packet, args.buffer,
                                          args.bind)
                else:
                    stream.add_udp_reader(port, args.packet, args.buffer, host)
        if ibv_endpoints:
            stream.add_udp_ibv_reader(ibv_endpoints, args.bind, args.packet,
                                      args.buffer, args.ibv_vector,
                                      args.ibv_max_poll)
        return stream

    def make_coro(sources):
        stream = make_stream(sources)
        return run_stream(stream, sources[0], args), stream

    def stop_streams():
        for stream in streams:
            stream.stop()

    args = get_args()
    logging.basicConfig(level=getattr(logging, args.log.upper()))

    if args.affinity is not None and len(args.affinity) > 0:
        spead2.ThreadPool.set_affinity(args.affinity[0])
        thread_pool = spead2.ThreadPool(args.threads,
                                        args.affinity[1:] + args.affinity[:1])
    else:
        thread_pool = spead2.ThreadPool(args.threads)
    memory_pool = None
    if args.mem_pool:
        memory_pool = spead2.MemoryPool(args.mem_lower, args.mem_upper,
                                        args.mem_max_free, args.mem_initial)
    if args.joint:
        coros_and_streams = [make_coro(args.source)]
    else:
        coros_and_streams = [make_coro([source]) for source in args.source]
    coros, streams = zip(*coros_and_streams)
    main_task = trollius.ensure_future(trollius.gather(*coros))
    loop = trollius.get_event_loop()
    loop.add_signal_handler(signal.SIGINT, stop_streams)
    try:
        loop.run_until_complete(main_task)
    except trollius.CancelledError:
        pass
    loop.close()
Ejemplo n.º 27
0
def monitor(signals):
    '''
    Establish and maintain a DropBot connection.

    XXX Coroutine XXX

    If no DropBot is available or if the connection is lost, wait until a
    DropBot is detected on one of the available serial ports and (re)connect.

    DropBot signals are forwarded to the supplied :data:`signals` namespace,
    avoiding the need to manually connect signals after DropBot is
    (re)connected.

    DropBot connection is automatically closed when coroutine exits, e.g., when
    cancelled.

    Notes
    -----
    On Windows **MUST** be run using a `asyncio.ProactorEventLoop`.

    Parameters
    ----------
    signals : blinker.Namespace
        Namespace for DropBot monitor signals.

    Sends
    -----
    connected
        When DropBot connection is established, with kwargs::
        - ``dropbot``: reference to DropBot proxy instance.
    disconnected
        When DropBot connection is lost.
    chip-inserted
        When DropBot detects a chip has been inserted.  Also sent upon
        connection to DropBot if a chip is present.
    chip-removed
        When DropBot detects a chip has been removed.  Also sent upon
        connection to DropBot if a chip is **not** present.

    Example
    -------

    >>> import blinker
    >>>
    >>> signals = blinker.Namespace()
    >>>
    >>> @asyncio.coroutine
    >>> def dump(*args, **kwargs):
    >>>     print('args=`%s`, kwargs=`%s`' % (args, kwargs))
    >>>
    >>> signals.signal('chip-inserted').connect(dump, weak=False)
    >>> loop = asyncio.ProactorEventLoop()
    >>> asyncio.set_event_loop(loop)
    >>> task = loop.create_task(db.monitor.dropbot_monitor(signals))
    >>> # Stop monitor after 15 seconds.
    >>> loop.call_later(15, task.cancel)
    >>> loop.run_until_complete(task)


    .. versionchanged:: 1.67.1
        Upon connection, send `'chip-inserted'` if chip is inserted or send
        `'chip-removed'` if no chip is inserted.

    .. versionchanged:: 1.68
        Send `'no-power'` signal if 12V power supply not connected.  Receivers
        may return `'ignore'` to attempt to connect anyway.
    '''
    loop = asyncio.get_event_loop()
    dropbot = None

    @asyncio.coroutine
    def co_flash_firmware():
        if dropbot is not None:
            dropbot.terminate()
        db.bin.upload.upload()
        time.sleep(.5)

    def flash_firmware(dropbot):
        loop.create_task(co_flash_firmware())

    signals.signal('flash-firmware') \
        .connect(lambda *args: loop.call_soon_threadsafe(flash_firmware,
                                                         dropbot), weak=False)

    def reboot(dropbot):
        if dropbot is not None:
            dropbot._reboot()

    signals.signal('reboot') \
        .connect(lambda *args: loop.call_soon_threadsafe(reboot, dropbot),
                 weak=False)

    def reconnect(dropbot):
        if dropbot is not None:
            dropbot.terminate()

    signals.signal('reconnect') \
        .connect(lambda *args: loop.call_soon_threadsafe(reconnect, dropbot),
                 weak=False)

    try:
        while True:
            # Multiple DropBot devices were found.
            # Get list of available devices.
            df_comports = yield asyncio.From(
                bnr. async ._available_devices(timeout=.1))

            if 'device_name' not in df_comports or not df_comports.shape[0]:
                yield asyncio.From(asyncio.sleep(.1))
                continue

            # Automatically select DropBot with highest version, with ties
            # going to the lowest port name (i.e., `COM1` before `COM2`).
            df_comports = df_comports.loc[df_comports.device_name ==
                                          'dropbot'].copy()
            df_comports.reset_index(inplace=True)

            df_comports.sort_values(['device_version', 'port'],
                                    ascending=[False, True],
                                    inplace=True)
            df_comports.set_index('port', inplace=True)
            port = df_comports.index[0]

            @asyncio.coroutine
            def _attempt_connect(**kwargs):
                ignore = kwargs.pop('ignore', [])
                try:
                    # Attempt to connect to automatically selected port.
                    dropbot = db.SerialProxy(port=port,
                                             ignore=ignore,
                                             **kwargs)
                    raise asyncio.Return(dropbot)
                except db.proxy.NoPower as exception:
                    # No 12V power supply detected on DropBot.
                    _L().debug('No 12V power supply detected.')
                    responses = signals.signal('no-power').send('keep_alive')

                    for t in asyncio.as_completed(
                        [loop.create_task(r[1]) for r in responses]):
                        response = yield asyncio.From(t)
                        if response == 'ignore':
                            ignore.append(db.proxy.NoPower)
                            break
                    else:
                        raise exception
                except bnr.proxy.DeviceVersionMismatch as exception:
                    # Firmware version does not match driver version.
                    _L().debug(
                        'Driver version (`%s`) does not match firmware '
                        'version (`%s`)', db.__version__,
                        exception.device_version)
                    responses = signals.signal('version-mismatch')\
                        .send('keep_alive', driver_version=db.__version__,
                            firmware_version=exception.device_version)

                    update = False

                    for t in asyncio.as_completed(
                        [loop.create_task(r[1]) for r in responses]):
                        response = yield asyncio.From(t)
                        if response == 'ignore':
                            ignore.append(bnr.proxy.DeviceVersionMismatch)
                            break
                        elif response == 'update':
                            update = True
                            break
                    else:
                        raise

                    if update:
                        # Flash firmware and retry connection.
                        _L().info('Flash firmware and retry connection.')
                        yield asyncio.From(co_flash_firmware())

                dropbot = yield asyncio.From(
                    _attempt_connect(ignore=ignore, **kwargs))
                raise asyncio.Return(dropbot)

            try:
                dropbot = yield asyncio.From(_attempt_connect())
            # except bnr.proxy.DeviceNotFound:
            except asyncio.CancelledError:
                raise
            except Exception:
                _L().debug('Error connecting to DropBot.', exc_info=True)
                yield asyncio.From(asyncio.sleep(.1))
                continue

            def co_connect(name):
                def _wrapped(sender, **message):
                    @asyncio.coroutine
                    def co_callback(message):
                        listeners = signals.signal(name).send(
                            'keep_alive', **message)
                        yield asyncio.From(
                            asyncio.gather(*(l[1] for l in listeners)))

                    return loop.call_soon_threadsafe(
                        loop.create_task, co_callback(sender, **message))

                return _wrapped

            for name_i in DROPBOT_SIGNAL_NAMES:
                dropbot.signals.signal(name_i).connect(co_connect(name_i),
                                                       weak=False)

            dropbot.signals.signal('output_enabled')\
                .connect(co_connect('chip-inserted'), weak=False)
            dropbot.signals.signal('output_disabled')\
                .connect(co_connect('chip-removed'), weak=False)

            responses = signals.signal('connected').send('keep_alive',
                                                         dropbot=dropbot)
            yield asyncio.From(asyncio.gather(*(r[1] for r in responses)))

            OUTPUT_ENABLE_PIN = 22
            # Chip may have been inserted before connecting, so `chip-inserted`
            # event may have been missed.
            # Explicitly check if chip is inserted by reading **active low**
            # `OUTPUT_ENABLE_PIN`.
            if dropbot.digital_read(OUTPUT_ENABLE_PIN):
                co_connect('chip-removed')({})
            else:
                co_connect('chip-inserted')({})

            disconnected = asyncio.Event()

            dropbot.serial_signals.signal('disconnected')\
                .connect(lambda *args:
                         loop.call_soon_threadsafe(disconnected.set),
                         weak=False)

            yield asyncio.From(disconnected.wait())

            dropbot.terminate()

            responses = signals.signal('disconnected').send('keep_alive')
            yield asyncio.From(asyncio.gather(*(r[1] for r in responses)))
    finally:
        signals.signal('closed').send('keep_alive')
        if dropbot is not None:
            dropbot.terminate()
Ejemplo n.º 28
0
def shoot(cfgs, status_callback):
    """
    Performs multi-tank multi-config test.
    Accepts one or more config dicts.
    Returns list of session ID's.
    Raises TankLocked and TestFailed.
    """

    try:
        sessions = [
            SessionWrapper(
                ft.partial(  # pylint: disable=W0142
                    status_callback, i),
                **cfg) for i, cfg in enumerate(cfgs)
        ]
    except Exception:
        logger.exception("Failed to initialize session objects, config:\n%s",
                         yaml.safe_dump(cfgs))
        raise
    prepares = []
    runs = []
    stops = []
    try:
        try:
            prepares = [async (session.prepare()) for session in sessions]
            yield From(gather(*prepares))  # pylint: disable=W0142
            logger.info("All tanks are prepared. STARTING TO SHOOT.")
            runs = [async (session.run_until_finish()) for session in sessions]
            yield From(gather(*runs))  # pylint: disable=W0142
        except KeyboardInterrupt:
            logger.info("Test interrupted")
            raise
        except CancelledError:
            logger.info("Test cancelled")
            raise
        except TestFailed:
            logger.info("Test failed")
            raise
        except Exception:
            logger.exception("Exception occured in Test.run_until_finish()")
            raise
        except BaseException:
            logger.exception(
                "Something strange caught by Test.run_until_finish()")
            raise
    except BaseException as ex:
        logger.info("Stopping remaining tank sessions...")
        stops = [
            async (session.stop()) for session in sessions
            if not session.finished
        ]
        yield From(gather(*stops, return_exceptions=True))  # pylint: disable=W0142
        raise ex
    finally:
        for task in prepares + runs + stops:
            task.cancel()

    logger.info("All tanks are done.")
    raise Return([
        session.session.s_id if session.session is not None else None
        for session in sessions
    ])
Ejemplo n.º 29
0
def main():
    def make_stream(sources):
        bug_compat = spead2.BUG_COMPAT_PYSPEAD_0_5_2 if args.pyspead else 0
        stream = spead2.recv.trollius.Stream(thread_pool, bug_compat, args.heaps, args.ring_heaps)
        if memory_pool is not None:
            stream.set_memory_allocator(memory_pool)
        if args.memcpy_nt:
            stream.set_memcpy(spead2.MEMCPY_NONTEMPORAL)
        ibv_endpoints = []
        for source in sources:
            try:
                if ':' in source:
                    host, port = source.rsplit(':', 1)
                    port = int(port)
                else:
                    host = ''
                    port = int(source)
            except ValueError:
                try:
                    stream.add_udp_pcap_file_reader(source)
                except AttributeError:
                    raise RuntimeError('spead2 was compiled without pcap support')
            else:
                if args.tcp:
                    stream.add_tcp_reader(port, args.packet, args.buffer, host)
                elif 'ibv' in args and args.ibv:
                    if host is None:
                        raise ValueError('a multicast group is required when using --ibv')
                    ibv_endpoints.append((host, port))
                elif args.bind and host:
                    stream.add_udp_reader(host, port, args.packet, args.buffer, args.bind)
                else:
                    stream.add_udp_reader(port, args.packet, args.buffer, host)
        if ibv_endpoints:
            stream.add_udp_ibv_reader(ibv_endpoints, args.bind, args.packet,
                                      args.buffer, args.ibv_vector, args.ibv_max_poll)
        return stream

    def make_coro(sources):
        stream = make_stream(sources)
        return run_stream(stream, sources[0], args), stream

    def stop_streams():
        for stream in streams:
            stream.stop()

    args = get_args()
    logging.basicConfig(level=getattr(logging, args.log.upper()))

    if args.affinity is not None and len(args.affinity) > 0:
        spead2.ThreadPool.set_affinity(args.affinity[0])
        thread_pool = spead2.ThreadPool(args.threads, args.affinity[1:] + args.affinity[:1])
    else:
        thread_pool = spead2.ThreadPool(args.threads)
    memory_pool = None
    if args.mem_pool:
        memory_pool = spead2.MemoryPool(args.mem_lower, args.mem_upper,
                                        args.mem_max_free, args.mem_initial)
    if args.joint:
        coros_and_streams = [make_coro(args.source)]
    else:
        coros_and_streams = [make_coro([source]) for source in args.source]
    coros, streams = zip(*coros_and_streams)
    main_task = trollius.ensure_future(trollius.gather(*coros))
    loop = trollius.get_event_loop()
    loop.add_signal_handler(signal.SIGINT, stop_streams)
    try:
        loop.run_until_complete(main_task)
    except trollius.CancelledError:
        pass
    loop.close()
Ejemplo n.º 30
0
    def execute_queries_async(
        self,
        queries,
        definitions=None,
        meta_filter=None,
        exclude_modified=False):
        """Runs the given queries on the corpus asynchronously.

        Arguments:
            queries ([Query]): The queries to run.
            definitions (dict): A dictionary defining query terms.
            meta_filter (dict -> bool): A function taking file meta data and
                returning whether the file should be queried.
            exclude_modified (bool): Whether to exclude modified files from
                the query.

        Returns:
            [Result]: An iterator producing the results of the query.
        """
        log = logging.getLogger('uweclang.corpus.manager')
        results = []

        # Get filtered files from corpus.
        try:
            files = self.files(
                meta_filter=meta_filter,
                exclude_modified=exclude_modified)
        except Exception as e:
            raise CorpusException(e)

        status = {
            'completed' : 0,
            'total': 0,
        } # Dictionary needed since `nonlocal` is not in Python 2.7.

        log.debug('Executing query batch (async.)')
        # Function for searching a single file.
        def query_file(meta, tagged, index):
            results = []
            # Extract TaggedToken list from file.
            text = list(chain.from_iterable(uweclang.read_tagged_string(tagged)))
            # Execute search.
            try:
                for i, query in enumerate(queries):
                    res = query.match(text, source_id=index, definitions=definitions)
                    if res:
                        results.extend(res)
            except Exception as e:
                raise QueryExecutionError(e)

            # Update status variables.
            status['completed'] += 1
            log.debug('Completed file %d', index)
            percent = int(status['completed'] / status['total'] * 100)
            log.info('%d%% complete', percent)
            return results

        # Worker function for running a file search.
        @asyncio.coroutine
        def worker(meta, tagged, index):
            log.debug('Starting file %d', index)
            return loop.run_in_executor(None, query_file, meta, tagged, index)

        # Create asynchronous task list.
        loop = asyncio.get_event_loop()
        tasks = []
        for index, (meta, tagged) in enumerate(files):
            log.debug('Added task %d', index)
            tasks.append(asyncio.ensure_future(worker(meta, tagged, index)))

        # Run tasks.
        status['total'] = len(tasks)
        log.info('Starting %d tasks.', status['total'])
        data = loop.run_until_complete(asyncio.gather(*tuple(tasks)))

        # Shutdown event loop and logger.
        loop.close()
        logging.shutdown()

        results = (task.result() for task in tasks if task.result())
        return chain.from_iterable(results)
Ejemplo n.º 31
0
def execute_steps(steps, signals=None):
    '''
    .. versionadded:: 2.32

    Parameters
    ----------
    steps : list[dict]
        List of plugin keyword argument dictionaries.
    signals : blinker.Namespace, optional
        Signals namespace where signals are sent through.

    Signals
    -------
    step-started
        Parameters::
        - ``i``: step index
        - ``plugin_kwargs``: plugin keyword arguments
        - ``steps_count``: total number of steps
    step-completed
        Parameters::
        - ``i``: step index
        - ``plugin_kwargs``: plugin keyword arguments
        - ``steps_count``: total number of steps
        - ``result``: list of plugin step return values
    '''
    if signals is None:
        signals = blinker.Namespace()

    for i, step_i in enumerate(steps):
        # Send notification that step has completed.
        responses = signals.signal('step-started')\
            .send('execute_steps', i=i, plugin_kwargs=step_i,
                  steps_count=len(steps))
        yield asyncio.From(asyncio.gather(*(r[1] for r in responses)))
        # XXX Execute `on_step_run` coroutines in background thread
        # event-loop.
        try:
            done, pending = yield asyncio.From(execute_step(step_i))

            exceptions = []

            for d in done:
                try:
                    d.result()
                except Exception as exception:
                    exceptions.append(exception)
                    _L().debug('Error: %s', exception, exc_info=True)

            if exceptions:
                use_markup = False
                monospace_format = '<tt>%s</tt>' if use_markup else '%s'

                if len(exceptions) == 1:
                    message = (' ' + monospace_format % exceptions[0])
                elif exceptions:
                    message = ('\n%s' % '\n'.join(' - ' + monospace_format % e
                                                  for e in exceptions))
                raise RuntimeError('Error executing step:%s' % message)
        except asyncio.CancelledError:
            _L().debug('Cancelling protocol.', exc_info=True)
            raise
        except Exception as exception:
            _L().debug('Error executing step: `%s`', exception, exc_info=True)
            raise
        else:
            # All plugins have completed the step.
            # Send notification that step has completed.
            responses = signals.signal('step-completed')\
                .send('execute_steps', i=i, plugin_kwargs=step_i,
                      result=[r.result() for r in done],
                      steps_count=len(steps))
            yield asyncio.From(asyncio.gather(*(r[1] for r in responses)))