Esempio n. 1
0
class Notifier:
    def __init__(self, logger: Logger):
        self._logger = logger
        self._q = LightQueue(maxsize=1000)

        # ensure no one is writing to the same broker connection concurrently
        self._sio = storage.utils.SIOManager.create(write_only=True)

    def _process(self, ar: AttackResult):
        flag_data = ar.get_flag_notification()
        self._logger.debug('Sending notification with %s', flag_data)
        self._sio.emit(
            event='flag_stolen',
            data={'data': flag_data},
            namespace='/live_events',
        )

    def add(self, ar: AttackResult) -> bool:
        try:
            self._q.put_nowait(ar)
            return True
        except Full:
            return False

    def __call__(self) -> None:
        while True:
            try:
                ar = self._q.get(block=True, timeout=3)
            except Empty:
                eventlet.sleep(0.5)
            else:
                self._process(ar)
Esempio n. 2
0
 def get(self, block=True, timeout=None):
     event = LightQueue(1)
     if flask.has_request_context():
         emit('{}#get'.format(self._uuid), callback=lambda x: event.put(x))
     else:
         sio = flask.current_app.extensions['socketio']
         sio.emit('{}#get'.format(self._uuid),
                  callback=lambda x: event.put(x))
     return event.get(timeout=1)
Esempio n. 3
0
def main(myid, queue, concurrency, delay=5.0, duration=DURATION):
    counter = 0
    created = list()
    results = LightQueue(concurrency * 10)
    pool = GreenPool(concurrency)
    api = AccountClient({'namespace': NS}, pool_maxsize=concurrency+1)
    now = start = checkpoint = time.time()
    pool.starmap(create_loop, [(api, 'buck-%d-%d' % (myid, n), results)
                               for n in range(concurrency)])
    while now - start < duration:
        try:
            res = results.get(timeout=delay)
            created.append(res)
            counter += 1
        except Empty:
            pass
        if now - checkpoint > delay:
            print("Proc %d: %d updates in %fs, %f updates per second." % (
                  myid, counter, now - checkpoint,
                  counter / (now - checkpoint)))
            counter = 0
            checkpoint = now
        now = time.time()
    for coro in pool.coroutines_running:
        coro.kill()
    while not results.empty():
        created.append(results.get(block=False))
    end = time.time()
    rate = len(created) / (end - start)
    print("Proc %d: end. %d updates in %fs, %f updates per second." % (
          myid, len(created), end - start, rate))
    time.sleep(2)
    print("Proc %d: cleaning..." % myid)
    del_req = {'dtime': time.time()}
    # Do not delete twice (or an exception is raised)
    uniq_ct = set(created)
    for _ in pool.starmap(api.container_update,
                          [(ACCOUNT, n, del_req) for n in uniq_ct]):
        pass
    pool.waitall()
    queue.put(rate)
    return 0
Esempio n. 4
0
 def get(self, timeout=10):
     name = getter.__name__
     # pylint: disable=protected-access
     signal = '{uuid}#{event}'.format(uuid=self._uuid, event=name)
     event = LightQueue(1)
     if flask.has_request_context():
         emit(signal, callback=lambda x: event.put(unpack(x)))
     else:
         sio = flask.current_app.extensions['socketio']
         sio.emit(signal, callback=lambda x: event.put(unpack(x)))
     data = event.get(timeout=timeout)
     return getter(self, data)
Esempio n. 5
0
 def get(self, timeout=10):  # pylint: disable=missing-docstring
     name = getter.__name__
     signal = '{uuid}{sep}{event}'.format(
         uuid=self._uuid,  # pylint: disable=protected-access
         sep=SEPARATOR,
         event=name
     )
     event = LightQueue(1)
     if flask.has_request_context():
         emit(signal, callback=lambda x: event.put(unpack(x)))
     else:
         sio = flask.current_app.extensions['socketio']
         sio.emit(signal, callback=lambda x: event.put(unpack(x)))
     data = event.get(timeout=timeout)
     return getter(self, data)
Esempio n. 6
0
File: mailbox.py Progetto: i2y/mochi
class LocalMailbox(Mailbox):
    __slots__ = ['_queue']

    def __init__(self):
        self._queue = LightQueue()

    def put(self, message):
        self._queue.put(message)

    def get(self):
        return self._queue.get(block=True)

    def encode(self):
        raise NotImplementedError

    @staticmethod
    def decode(params):
        raise NotImplementedError
Esempio n. 7
0
def fail_fast_imap(pool, call, items):
    """ Run a function against each item in a given list, yielding each
    function result in turn, where the function call is handled in a
    :class:`~eventlet.greenthread.GreenThread` spawned by the provided pool.

    If any function raises an exception, all other ongoing threads are killed,
    and the exception is raised to the caller.

    This function is similar to :meth:`~eventlet.greenpool.GreenPool.imap`.

    :param pool: Pool to spawn function threads from
    :type pool: eventlet.greenpool.GreenPool
    :param call: Function call to make, expecting to receive an item from the
        given list
    """
    result_queue = LightQueue(maxsize=len(items))
    spawned_threads = set()

    def handle_result(finished_thread):
        try:
            thread_result = finished_thread.wait()
            spawned_threads.remove(finished_thread)
            result_queue.put((thread_result, None))
        except Exception:
            spawned_threads.remove(finished_thread)
            result_queue.put((None, sys.exc_info()))

    for item in items:
        gt = pool.spawn(call, item)
        spawned_threads.add(gt)
        gt.link(handle_result)

    while spawned_threads:
        result, exc_info = result_queue.get()
        if exc_info is not None:
            # Kill all other ongoing threads
            for ongoing_thread in spawned_threads:
                ongoing_thread.kill()
            # simply raising here (even raising a full exc_info) isn't
            # sufficient to preserve the original stack trace.
            # greenlet.throw() achieves this.
            eventlet.getcurrent().throw(*exc_info)
        yield result
Esempio n. 8
0
File: utils.py Progetto: ahmb/nameko
def fail_fast_imap(pool, call, items):
    """ Run a function against each item in a given list, yielding each
    function result in turn, where the function call is handled in a
    :class:`~eventlet.greenthread.GreenThread` spawned by the provided pool.

    If any function raises an exception, all other ongoing threads are killed,
    and the exception is raised to the caller.

    This function is similar to :meth:`~eventlet.greenpool.GreenPool.imap`.

    :param pool: Pool to spawn function threads from
    :type pool: eventlet.greenpool.GreenPool
    :param call: Function call to make, expecting to receive an item from the
        given list
    """
    result_queue = LightQueue(maxsize=len(items))
    spawned_threads = set()

    def handle_result(finished_thread):
        try:
            thread_result = finished_thread.wait()
            spawned_threads.remove(finished_thread)
            result_queue.put((thread_result, None))
        except Exception:
            spawned_threads.remove(finished_thread)
            result_queue.put((None, sys.exc_info()))

    for item in items:
        gt = pool.spawn(call, item)
        spawned_threads.add(gt)
        gt.link(handle_result)

    while spawned_threads:
        result, exc_info = result_queue.get()
        if exc_info is not None:
            # Kill all other ongoing threads
            for ongoing_thread in spawned_threads:
                ongoing_thread.kill()
            # simply raising here (even raising a full exc_info) isn't
            # sufficient to preserve the original stack trace.
            # greenlet.throw() achieves this.
            eventlet.getcurrent().throw(*exc_info)
        yield result
Esempio n. 9
0
def fail_fast_imap(pool, call, items):
    """ Run a function against each item in a given list, yielding each
    function result in turn, where the function call is handled in a
    :class:`~eventlet.greenthread.GreenThread` spawned by the provided pool.

    If any function raises an exception, all other ongoing threads are killed,
    and the exception is raised to the caller.

    This function is similar to :meth:`~eventlet.greenpool.GreenPool.imap`.

    :param pool: Pool to spawn function threads from
    :type pool: eventlet.greenpool.GreenPool
    :param call: Function call to make, expecting to receive an item from the
        given list
    """
    result_queue = LightQueue(maxsize=len(items))
    spawned_threads = set()

    def handle_result(finished_thread):
        try:
            thread_result = finished_thread.wait()
            spawned_threads.remove(finished_thread)
            result_queue.put((thread_result, None))
        except Exception as e:
            spawned_threads.remove(finished_thread)
            result_queue.put((None, e))

    for item in items:
        gt = pool.spawn(call, item)
        spawned_threads.add(gt)
        gt.link(handle_result)

    while spawned_threads:
        result, raised_exc = result_queue.get()
        if raised_exc is not None:
            # Kill all other ongoing threads
            for ongoing_thread in spawned_threads:
                ongoing_thread.kill()
            raise raised_exc
        yield result
Esempio n. 10
0
class EventletConnectionPool(ConnectionPool):
    def __init__(self, connection_class=Connection, max_connections=None,
                 **connection_kwargs):
        self.pid = os.getpid()
        self.connection_class = connection_class
        self.connection_kwargs = connection_kwargs
        self.max_connections = max_connections or 2 ** 31
        self._created_connections = 0
        self._available_connections = LightQueue()
        self._in_use_connections = set()
    def get_connection(self, command_name, *keys, **options):
        "Get a connection from the pool"
        try:
            connection = self._available_connections.get_nowait()
        except Empty:
            if self._created_connections < self.max_connections:
                connection = self.make_connection()
            else:
                try:
                    connection = self._available_connections.get()
                except Empty:
                    raise ConnectionError("Couldn't find a free connection")
        self._in_use_connections.add(connection)
        return connection
    def release(self, connection):
        "Releases the connection back to the pool"
        self._checkpid()
        if connection.pid == self.pid:
            self._in_use_connections.remove(connection)
            self._available_connections.put_nowait(connection)
    def disconnect(self):
        "Disconnects all connections in the pool"
        while True:
            try:
                self._available_connections.get_nowait().disconnect()
            except Empty:
                break
        for connection in self._in_use_connections:
            connection.disconnect()
Esempio n. 11
0
def load(key):
    """Load the value stored with the key.

    Parameters
    ----------
    key : object
        The key to lookup the value stored.

    Returns
    -------
    object
        The value if the key exists in the cache, otherwise None.

    """
    signal = 'cache_load'
    event = LightQueue(1)
    if flask.has_request_context():
        emit(signal, {'data': pack(key)}, callback=event.put)
    else:
        sio = flask.current_app.extensions['socketio']
        sio.emit(signal, {'data': pack(key)}, callback=event.put)
    return msgpack.unpackb(bytes(event.get(timeout=10)), encoding='utf8')
Esempio n. 12
0
    def __getitem__(self, key):
        """Load the value stored with the key.

        Parameters
        ----------
        key : str
            The key to lookup the value stored.

        Returns
        -------
        object
            The value if the key exists in the cache, otherwise None.

        """
        validate(key)
        signal = 'cache_load'
        event = LightQueue(1)
        if flask.has_request_context():
            emit(signal, {'data': pack(key)}, callback=event.put)
        else:
            sio = flask.current_app.extensions['socketio']
            sio.emit(signal, {'data': pack(key)}, callback=event.put)
        return msgpack.unpackb(bytes(event.get(timeout=10)), encoding='utf8')
Esempio n. 13
0
class VCenter(object):
    # PropertyCollector discovers changes on vms and their hardware and produces
    #    (mac, switch, portKey, portGroupKey, connectable.connected, connectable.status)
    #    internally, it keeps internally vm and key for identifying updates
    # Subsequently, the mac has to be identified with a port
    #

    def __init__(self, config=None, pool=None, agent=None):
        self.pool = pool
        self.agent = agent
        self.config = config or CONF.ML2_VMWARE
        self.connection = _create_session(self.config)
        self.context = agent.context
        self._monitor_process = VCenterMonitor(self,
                                               self.config,
                                               connection=self.connection,
                                               pool=self.pool)
        self.queue = Queue(None)

        self.uuid_port_map = {}
        self.mac_port_map = {}

        self.uuid_dvs_map = {}
        self.network_dvs_map = {}

        for network, dvs in six.iteritems(
                dvs_util.create_network_map_from_config(
                    self.config, connection=self.connection, pool=pool)):
            self.network_dvs_map[network] = dvs
            self.uuid_dvs_map[dvs.uuid] = dvs

    def vcenter_port_changes(self, changed):
        ports_by_mac = defaultdict(dict)

        for port_desc in changed:
            port = {
                'port_desc': port_desc,
                'port': {
                    'binding:vif_details': {
                        'dvs_port_key': port_desc.port_key,
                        'dvs_uuid': port_desc.dvs_uuid,
                    },
                    'mac_address': port_desc.mac_address
                }
            }

            dvs = self.get_dvs_by_uuid(port_desc.dvs_uuid)
            if not dvs:
                continue

            if port_desc.status != 'deleted':
                dvs.ports_by_key[port_desc.port_key] = port
                ports_by_mac[port_desc.mac_address] = port
            else:
                dvs.ports_by_key.pop(port_desc.port_key, None)
                ports_by_mac.pop(port_desc.mac_address, None)

        # self.read_dvs_ports(ports_by_mac) Skip that
        macs = set(six.iterkeys(ports_by_mac))

        port_list = []
        for port_id, mac, status, admin_state_up, network_id, network_type, segmentation_id in self.get_ports_by_mac(
                macs):
            macs.discard(mac)
            port_info = ports_by_mac[mac]
            neutron_info = {
                "port_id": port_id,
                "id": port_id,
                "device": port_id,
                "mac_address": mac,
                "admin_state_up": admin_state_up,
                "status": status,
                "network_id": network_id,
                "network_type": network_type,
                "segmentation_id": segmentation_id,
            }

            port_info["port"]["id"] = port_id
            c_util.dict_merge(port_info, neutron_info)
            self.uuid_port_map[port_id] = port_info
            port_list.append(port_info)

        if macs:
            LOG.warning(
                _LW("Could not find the following macs: {}").format(macs))

        LOG.debug("Got port information from db for %d ports", len(port_list))
        for port in port_list:
            self.queue.put(port)

    def start(self):
        self._monitor_process.start()

    @staticmethod
    def update_port_desc(port, port_info):
        # Validate connectionCookie, so we still have the same instance behind that portKey
        port_desc = port['port_desc']
        connection_cookie = _cast(getattr(port_info, 'connectionCookie', None))

        if port_desc.connection_cookie != connection_cookie:
            LOG.error("Cookie mismatch {} {} {} <> {}".format(
                port_desc.mac_address, port_desc.port_key,
                port_desc.connection_cookie, connection_cookie))
            return False

        for k, v in six.iteritems(_DVSPortDesc.from_dvs_port(port_info)):
            setattr(port_desc, k, v)
        return True

    def ports_by_switch_and_key(self, ports):
        ports_by_switch_and_key = defaultdict(dict)
        for port in ports:
            port_desc = port['port_desc']
            dvs = self.get_dvs_by_uuid(port_desc.dvs_uuid)

            if dvs:
                ports_by_switch_and_key[dvs][port_desc.port_key] = port

        return ports_by_switch_and_key

    @c_util.stats.timed()
    def bind_ports(self, ports, callback=None):
        ports_by_switch_and_key = self.ports_by_switch_and_key(ports)

        for dvs, ports_by_key in six.iteritems(ports_by_switch_and_key):
            specs = []
            for port in six.itervalues(ports_by_key):
                if (port["network_type"] == "vlan" and not port["segmentation_id"] is None) \
                        or port["network_type"] == "flat":
                    spec = builder.neutron_to_port_config_spec(port)
                    if not CONF.AGENT.dry_run:
                        specs.append(spec)
                    else:
                        LOG.debug(spec)

            dvs.queue_update_specs(specs, callback=callback)

    def get_dvs_by_uuid(self, uuid):
        return self.uuid_dvs_map.get(uuid, None)

    def get_port_by_uuid(self, uuid):
        return self.uuid_port_map.get(uuid, None)

    def fetch_ports_by_mac(self, portgroup_key=None, mac_addr=None):
        for dvs in six.itervalues(self.uuid_dvs_map):
            port_keys = dvs._dvs.FetchDVPortKeys(
                dvs._dvs, criteria=builder.port_criteria())
            ports = dvs._dvs.FetchDVPorts(criteria=builder.port_criteria(
                port_group_key=portgroup_key, port_key=port_keys))

        for port in ports:
            if hasattr(port, 'state'):
                if hasattr(port.state, 'runtimeInfo'):
                    if mac_addr == port.state.runtimeInfo.macAddress:
                        return port
                    else:
                        continue

            raise Exception('DVS port not found!')

    def get_new_ports(self, block=False, timeout=1.0, max_ports=None):
        ports_by_mac = defaultdict(dict)
        try:
            while max_ports is None or len(ports_by_mac) < max_ports:
                new_port = self.queue.get(block=block, timeout=timeout)
                port_desc = new_port['port_desc']
                block = False  # Only block on the first item
                if port_desc.status == 'deleted':
                    ports_by_mac.pop(port_desc.mac_address, None)
                    port = self.mac_port_map.pop(port_desc.mac_address, None)
                    if port:
                        port_desc = port['port_desc']
                        self.uuid_port_map.pop(port['id'], None)
                        dvs = self.get_dvs_by_uuid(port_desc.dvs_uuid)
                        dvs.ports_by_key.pop(port_desc.port_key, None)
                else:
                    port = self.mac_port_map.get(port_desc.mac_address, {})
                    port.update(dict(new_port))
                    ports_by_mac[port_desc.mac_address] = port
                    dvs = self.get_dvs_by_uuid(port_desc.dvs_uuid)
                    if dvs:
                        dvs.ports_by_key[port_desc.port_key] = port
        except Empty:
            pass
        return ports_by_mac

    def read_dvs_ports(self, ports_by_mac):
        ports_by_switch_and_key = self.ports_by_switch_and_key(
            six.itervalues(ports_by_mac))
        # This loop can get very slow, if get_port_info_by_portkey gets port keys passed of instances, which are only
        # partly connected, meaning: the instance is associated, but the link is not quite up yet
        for dvs, ports_by_key in six.iteritems(ports_by_switch_and_key):
            for port_info in dvs.get_port_info_by_portkey(
                    list(
                        six.iterkeys(ports_by_key))):  # View is not sufficient
                port = ports_by_key[port_info.key]
                if not VCenter.update_port_desc(port, port_info):
                    port_desc = port['port_desc']
                    ports_by_mac.pop(port_desc.mac_address)
        LOG.debug("Read all ports")

    @enginefacade.reader
    def get_ports_by_mac(self, mac_addresses):
        if not mac_addresses:
            return []

        if not self.context:
            self.context = neutron.context.get_admin_context()

        session = self.context.session
        with session.begin(subtransactions=True):
            return session.query(models_v2.Port.id, models_v2.Port.mac_address, models_v2.Port.status, models_v2.Port.admin_state_up,
                                 models_ml2.NetworkSegment.network_id,
                                 models_ml2.NetworkSegment.network_type,
                                 models_ml2.NetworkSegment.segmentation_id).\
                 join(models_ml2.PortBindingLevel, models_v2.Port.id == models_ml2.PortBindingLevel.port_id).\
                 join(models_ml2.NetworkSegment, models_ml2.PortBindingLevel.segment_id == models_ml2.NetworkSegment.id).\
                 filter(models_ml2.PortBindingLevel.host == self.agent.conf.host,
                        models_ml2.PortBindingLevel.driver == constants.DVS,
                        models_v2.Port.mac_address.in_(mac_addresses),
                        ).all()

    def stop(self):
        self._monitor_process.stop()

        try:
            while True:
                self.queue.get_nowait()
        except Empty:
            pass