class Notifier:
def __init__(self, logger: Logger):
self._logger = logger
self._q = LightQueue(maxsize=1000)
# ensure no one is writing to the same broker connection concurrently
self._sio = storage.utils.SIOManager.create(write_only=True)
def _process(self, ar: AttackResult):
flag_data = ar.get_flag_notification()
self._logger.debug('Sending notification with %s', flag_data)
self._sio.emit(
event='flag_stolen',
data={'data': flag_data},
namespace='/live_events',
)
def add(self, ar: AttackResult) -> bool:
try:
self._q.put_nowait(ar)
return True
except Full:
return False
def __call__(self) -> None:
while True:
try:
ar = self._q.get(block=True, timeout=3)
except Empty:
eventlet.sleep(0.5)
else:
self._process(ar)
def get(self, block=True, timeout=None):
event = LightQueue(1)
if flask.has_request_context():
emit('{}#get'.format(self._uuid), callback=lambda x: event.put(x))
else:
sio = flask.current_app.extensions['socketio']
sio.emit('{}#get'.format(self._uuid),
callback=lambda x: event.put(x))
return event.get(timeout=1)
def main(myid, queue, concurrency, delay=5.0, duration=DURATION):
counter = 0
created = list()
results = LightQueue(concurrency * 10)
pool = GreenPool(concurrency)
api = AccountClient({'namespace': NS}, pool_maxsize=concurrency+1)
now = start = checkpoint = time.time()
pool.starmap(create_loop, [(api, 'buck-%d-%d' % (myid, n), results)
for n in range(concurrency)])
while now - start < duration:
try:
res = results.get(timeout=delay)
created.append(res)
counter += 1
except Empty:
pass
if now - checkpoint > delay:
print("Proc %d: %d updates in %fs, %f updates per second." % (
myid, counter, now - checkpoint,
counter / (now - checkpoint)))
counter = 0
checkpoint = now
now = time.time()
for coro in pool.coroutines_running:
coro.kill()
while not results.empty():
created.append(results.get(block=False))
end = time.time()
rate = len(created) / (end - start)
print("Proc %d: end. %d updates in %fs, %f updates per second." % (
myid, len(created), end - start, rate))
time.sleep(2)
print("Proc %d: cleaning..." % myid)
del_req = {'dtime': time.time()}
# Do not delete twice (or an exception is raised)
uniq_ct = set(created)
for _ in pool.starmap(api.container_update,
[(ACCOUNT, n, del_req) for n in uniq_ct]):
pass
pool.waitall()
queue.put(rate)
return 0
def get(self, timeout=10):
name = getter.__name__
# pylint: disable=protected-access
signal = '{uuid}#{event}'.format(uuid=self._uuid, event=name)
event = LightQueue(1)
if flask.has_request_context():
emit(signal, callback=lambda x: event.put(unpack(x)))
else:
sio = flask.current_app.extensions['socketio']
sio.emit(signal, callback=lambda x: event.put(unpack(x)))
data = event.get(timeout=timeout)
return getter(self, data)
def get(self, timeout=10): # pylint: disable=missing-docstring
name = getter.__name__
signal = '{uuid}{sep}{event}'.format(
uuid=self._uuid, # pylint: disable=protected-access
sep=SEPARATOR,
event=name
)
event = LightQueue(1)
if flask.has_request_context():
emit(signal, callback=lambda x: event.put(unpack(x)))
else:
sio = flask.current_app.extensions['socketio']
sio.emit(signal, callback=lambda x: event.put(unpack(x)))
data = event.get(timeout=timeout)
return getter(self, data)
class LocalMailbox(Mailbox):
__slots__ = ['_queue']
def __init__(self):
self._queue = LightQueue()
def put(self, message):
self._queue.put(message)
def get(self):
return self._queue.get(block=True)
def encode(self):
raise NotImplementedError
@staticmethod
def decode(params):
raise NotImplementedError
def fail_fast_imap(pool, call, items):
""" Run a function against each item in a given list, yielding each
function result in turn, where the function call is handled in a
:class:`~eventlet.greenthread.GreenThread` spawned by the provided pool.
If any function raises an exception, all other ongoing threads are killed,
and the exception is raised to the caller.
This function is similar to :meth:`~eventlet.greenpool.GreenPool.imap`.
:param pool: Pool to spawn function threads from
:type pool: eventlet.greenpool.GreenPool
:param call: Function call to make, expecting to receive an item from the
given list
"""
result_queue = LightQueue(maxsize=len(items))
spawned_threads = set()
def handle_result(finished_thread):
try:
thread_result = finished_thread.wait()
spawned_threads.remove(finished_thread)
result_queue.put((thread_result, None))
except Exception:
spawned_threads.remove(finished_thread)
result_queue.put((None, sys.exc_info()))
for item in items:
gt = pool.spawn(call, item)
spawned_threads.add(gt)
gt.link(handle_result)
while spawned_threads:
result, exc_info = result_queue.get()
if exc_info is not None:
# Kill all other ongoing threads
for ongoing_thread in spawned_threads:
ongoing_thread.kill()
# simply raising here (even raising a full exc_info) isn't
# sufficient to preserve the original stack trace.
# greenlet.throw() achieves this.
eventlet.getcurrent().throw(*exc_info)
yield result
def fail_fast_imap(pool, call, items):
""" Run a function against each item in a given list, yielding each
function result in turn, where the function call is handled in a
:class:`~eventlet.greenthread.GreenThread` spawned by the provided pool.
If any function raises an exception, all other ongoing threads are killed,
and the exception is raised to the caller.
This function is similar to :meth:`~eventlet.greenpool.GreenPool.imap`.
:param pool: Pool to spawn function threads from
:type pool: eventlet.greenpool.GreenPool
:param call: Function call to make, expecting to receive an item from the
given list
"""
result_queue = LightQueue(maxsize=len(items))
spawned_threads = set()
def handle_result(finished_thread):
try:
thread_result = finished_thread.wait()
spawned_threads.remove(finished_thread)
result_queue.put((thread_result, None))
except Exception:
spawned_threads.remove(finished_thread)
result_queue.put((None, sys.exc_info()))
for item in items:
gt = pool.spawn(call, item)
spawned_threads.add(gt)
gt.link(handle_result)
while spawned_threads:
result, exc_info = result_queue.get()
if exc_info is not None:
# Kill all other ongoing threads
for ongoing_thread in spawned_threads:
ongoing_thread.kill()
# simply raising here (even raising a full exc_info) isn't
# sufficient to preserve the original stack trace.
# greenlet.throw() achieves this.
eventlet.getcurrent().throw(*exc_info)
yield result
def fail_fast_imap(pool, call, items):
""" Run a function against each item in a given list, yielding each
function result in turn, where the function call is handled in a
:class:`~eventlet.greenthread.GreenThread` spawned by the provided pool.
If any function raises an exception, all other ongoing threads are killed,
and the exception is raised to the caller.
This function is similar to :meth:`~eventlet.greenpool.GreenPool.imap`.
:param pool: Pool to spawn function threads from
:type pool: eventlet.greenpool.GreenPool
:param call: Function call to make, expecting to receive an item from the
given list
"""
result_queue = LightQueue(maxsize=len(items))
spawned_threads = set()
def handle_result(finished_thread):
try:
thread_result = finished_thread.wait()
spawned_threads.remove(finished_thread)
result_queue.put((thread_result, None))
except Exception as e:
spawned_threads.remove(finished_thread)
result_queue.put((None, e))
for item in items:
gt = pool.spawn(call, item)
spawned_threads.add(gt)
gt.link(handle_result)
while spawned_threads:
result, raised_exc = result_queue.get()
if raised_exc is not None:
# Kill all other ongoing threads
for ongoing_thread in spawned_threads:
ongoing_thread.kill()
raise raised_exc
yield result
class EventletConnectionPool(ConnectionPool):
def __init__(self, connection_class=Connection, max_connections=None,
**connection_kwargs):
self.pid = os.getpid()
self.connection_class = connection_class
self.connection_kwargs = connection_kwargs
self.max_connections = max_connections or 2 ** 31
self._created_connections = 0
self._available_connections = LightQueue()
self._in_use_connections = set()
def get_connection(self, command_name, *keys, **options):
"Get a connection from the pool"
try:
connection = self._available_connections.get_nowait()
except Empty:
if self._created_connections < self.max_connections:
connection = self.make_connection()
else:
try:
connection = self._available_connections.get()
except Empty:
raise ConnectionError("Couldn't find a free connection")
self._in_use_connections.add(connection)
return connection
def release(self, connection):
"Releases the connection back to the pool"
self._checkpid()
if connection.pid == self.pid:
self._in_use_connections.remove(connection)
self._available_connections.put_nowait(connection)
def disconnect(self):
"Disconnects all connections in the pool"
while True:
try:
self._available_connections.get_nowait().disconnect()
except Empty:
break
for connection in self._in_use_connections:
connection.disconnect()
def load(key):
"""Load the value stored with the key.
Parameters
----------
key : object
The key to lookup the value stored.
Returns
-------
object
The value if the key exists in the cache, otherwise None.
"""
signal = 'cache_load'
event = LightQueue(1)
if flask.has_request_context():
emit(signal, {'data': pack(key)}, callback=event.put)
else:
sio = flask.current_app.extensions['socketio']
sio.emit(signal, {'data': pack(key)}, callback=event.put)
return msgpack.unpackb(bytes(event.get(timeout=10)), encoding='utf8')
def __getitem__(self, key):
"""Load the value stored with the key.
Parameters
----------
key : str
The key to lookup the value stored.
Returns
-------
object
The value if the key exists in the cache, otherwise None.
"""
validate(key)
signal = 'cache_load'
event = LightQueue(1)
if flask.has_request_context():
emit(signal, {'data': pack(key)}, callback=event.put)
else:
sio = flask.current_app.extensions['socketio']
sio.emit(signal, {'data': pack(key)}, callback=event.put)
return msgpack.unpackb(bytes(event.get(timeout=10)), encoding='utf8')
class VCenter(object):
# PropertyCollector discovers changes on vms and their hardware and produces
# (mac, switch, portKey, portGroupKey, connectable.connected, connectable.status)
# internally, it keeps internally vm and key for identifying updates
# Subsequently, the mac has to be identified with a port
#
def __init__(self, config=None, pool=None, agent=None):
self.pool = pool
self.agent = agent
self.config = config or CONF.ML2_VMWARE
self.connection = _create_session(self.config)
self.context = agent.context
self._monitor_process = VCenterMonitor(self,
self.config,
connection=self.connection,
pool=self.pool)
self.queue = Queue(None)
self.uuid_port_map = {}
self.mac_port_map = {}
self.uuid_dvs_map = {}
self.network_dvs_map = {}
for network, dvs in six.iteritems(
dvs_util.create_network_map_from_config(
self.config, connection=self.connection, pool=pool)):
self.network_dvs_map[network] = dvs
self.uuid_dvs_map[dvs.uuid] = dvs
def vcenter_port_changes(self, changed):
ports_by_mac = defaultdict(dict)
for port_desc in changed:
port = {
'port_desc': port_desc,
'port': {
'binding:vif_details': {
'dvs_port_key': port_desc.port_key,
'dvs_uuid': port_desc.dvs_uuid,
},
'mac_address': port_desc.mac_address
}
}
dvs = self.get_dvs_by_uuid(port_desc.dvs_uuid)
if not dvs:
continue
if port_desc.status != 'deleted':
dvs.ports_by_key[port_desc.port_key] = port
ports_by_mac[port_desc.mac_address] = port
else:
dvs.ports_by_key.pop(port_desc.port_key, None)
ports_by_mac.pop(port_desc.mac_address, None)
# self.read_dvs_ports(ports_by_mac) Skip that
macs = set(six.iterkeys(ports_by_mac))
port_list = []
for port_id, mac, status, admin_state_up, network_id, network_type, segmentation_id in self.get_ports_by_mac(
macs):
macs.discard(mac)
port_info = ports_by_mac[mac]
neutron_info = {
"port_id": port_id,
"id": port_id,
"device": port_id,
"mac_address": mac,
"admin_state_up": admin_state_up,
"status": status,
"network_id": network_id,
"network_type": network_type,
"segmentation_id": segmentation_id,
}
port_info["port"]["id"] = port_id
c_util.dict_merge(port_info, neutron_info)
self.uuid_port_map[port_id] = port_info
port_list.append(port_info)
if macs:
LOG.warning(
_LW("Could not find the following macs: {}").format(macs))
LOG.debug("Got port information from db for %d ports", len(port_list))
for port in port_list:
self.queue.put(port)
def start(self):
self._monitor_process.start()
@staticmethod
def update_port_desc(port, port_info):
# Validate connectionCookie, so we still have the same instance behind that portKey
port_desc = port['port_desc']
connection_cookie = _cast(getattr(port_info, 'connectionCookie', None))
if port_desc.connection_cookie != connection_cookie:
LOG.error("Cookie mismatch {} {} {} <> {}".format(
port_desc.mac_address, port_desc.port_key,
port_desc.connection_cookie, connection_cookie))
return False
for k, v in six.iteritems(_DVSPortDesc.from_dvs_port(port_info)):
setattr(port_desc, k, v)
return True
def ports_by_switch_and_key(self, ports):
ports_by_switch_and_key = defaultdict(dict)
for port in ports:
port_desc = port['port_desc']
dvs = self.get_dvs_by_uuid(port_desc.dvs_uuid)
if dvs:
ports_by_switch_and_key[dvs][port_desc.port_key] = port
return ports_by_switch_and_key
@c_util.stats.timed()
def bind_ports(self, ports, callback=None):
ports_by_switch_and_key = self.ports_by_switch_and_key(ports)
for dvs, ports_by_key in six.iteritems(ports_by_switch_and_key):
specs = []
for port in six.itervalues(ports_by_key):
if (port["network_type"] == "vlan" and not port["segmentation_id"] is None) \
or port["network_type"] == "flat":
spec = builder.neutron_to_port_config_spec(port)
if not CONF.AGENT.dry_run:
specs.append(spec)
else:
LOG.debug(spec)
dvs.queue_update_specs(specs, callback=callback)
def get_dvs_by_uuid(self, uuid):
return self.uuid_dvs_map.get(uuid, None)
def get_port_by_uuid(self, uuid):
return self.uuid_port_map.get(uuid, None)
def fetch_ports_by_mac(self, portgroup_key=None, mac_addr=None):
for dvs in six.itervalues(self.uuid_dvs_map):
port_keys = dvs._dvs.FetchDVPortKeys(
dvs._dvs, criteria=builder.port_criteria())
ports = dvs._dvs.FetchDVPorts(criteria=builder.port_criteria(
port_group_key=portgroup_key, port_key=port_keys))
for port in ports:
if hasattr(port, 'state'):
if hasattr(port.state, 'runtimeInfo'):
if mac_addr == port.state.runtimeInfo.macAddress:
return port
else:
continue
raise Exception('DVS port not found!')
def get_new_ports(self, block=False, timeout=1.0, max_ports=None):
ports_by_mac = defaultdict(dict)
try:
while max_ports is None or len(ports_by_mac) < max_ports:
new_port = self.queue.get(block=block, timeout=timeout)
port_desc = new_port['port_desc']
block = False # Only block on the first item
if port_desc.status == 'deleted':
ports_by_mac.pop(port_desc.mac_address, None)
port = self.mac_port_map.pop(port_desc.mac_address, None)
if port:
port_desc = port['port_desc']
self.uuid_port_map.pop(port['id'], None)
dvs = self.get_dvs_by_uuid(port_desc.dvs_uuid)
dvs.ports_by_key.pop(port_desc.port_key, None)
else:
port = self.mac_port_map.get(port_desc.mac_address, {})
port.update(dict(new_port))
ports_by_mac[port_desc.mac_address] = port
dvs = self.get_dvs_by_uuid(port_desc.dvs_uuid)
if dvs:
dvs.ports_by_key[port_desc.port_key] = port
except Empty:
pass
return ports_by_mac
def read_dvs_ports(self, ports_by_mac):
ports_by_switch_and_key = self.ports_by_switch_and_key(
six.itervalues(ports_by_mac))
# This loop can get very slow, if get_port_info_by_portkey gets port keys passed of instances, which are only
# partly connected, meaning: the instance is associated, but the link is not quite up yet
for dvs, ports_by_key in six.iteritems(ports_by_switch_and_key):
for port_info in dvs.get_port_info_by_portkey(
list(
six.iterkeys(ports_by_key))): # View is not sufficient
port = ports_by_key[port_info.key]
if not VCenter.update_port_desc(port, port_info):
port_desc = port['port_desc']
ports_by_mac.pop(port_desc.mac_address)
LOG.debug("Read all ports")
@enginefacade.reader
def get_ports_by_mac(self, mac_addresses):
if not mac_addresses:
return []
if not self.context:
self.context = neutron.context.get_admin_context()
session = self.context.session
with session.begin(subtransactions=True):
return session.query(models_v2.Port.id, models_v2.Port.mac_address, models_v2.Port.status, models_v2.Port.admin_state_up,
models_ml2.NetworkSegment.network_id,
models_ml2.NetworkSegment.network_type,
models_ml2.NetworkSegment.segmentation_id).\
join(models_ml2.PortBindingLevel, models_v2.Port.id == models_ml2.PortBindingLevel.port_id).\
join(models_ml2.NetworkSegment, models_ml2.PortBindingLevel.segment_id == models_ml2.NetworkSegment.id).\
filter(models_ml2.PortBindingLevel.host == self.agent.conf.host,
models_ml2.PortBindingLevel.driver == constants.DVS,
models_v2.Port.mac_address.in_(mac_addresses),
).all()
def stop(self):
self._monitor_process.stop()
try:
while True:
self.queue.get_nowait()
except Empty:
pass