class ProducerPool(object):
Receipt = Receipt
def __init__(self, size=20):
self.size = size
self.inqueue = LightQueue()
self._running = None
self._producers = None
def apply_async(self, task, args, kwargs, callback=None, **options):
if self._running is None:
self._running = spawn_n(self._run)
receipt = self.Receipt(callback)
self.inqueue.put((task, args, kwargs, options, receipt))
return receipt
def _run(self):
self._producers = [spawn_n(self._producer) for _ in xrange(self.size)]
def _producer(self):
connection = current_app.broker_connection()
publisher = current_app.amqp.TaskProducer(connection)
inqueue = self.inqueue
while 1:
task, args, kwargs, options, receipt = inqueue.get()
result = task.apply_async(args,
kwargs,
publisher=publisher,
**options)
receipt.finished(result)
class ProducerPool(object):
Receipt = Receipt
def __init__(self, size=20):
self.size = size
self.inqueue = LightQueue()
self._running = None
self._producers = None
def apply_async(self, task, args, kwargs, callback=None, **options):
if self._running is None:
self._running = spawn_n(self._run)
receipt = self.Receipt(callback)
self.inqueue.put((task, args, kwargs, options, receipt))
return receipt
def _run(self):
self._producers = [spawn_n(self._producer)
for _ in xrange(self.size)]
def _producer(self):
connection = current_app.broker_connection()
publisher = current_app.amqp.TaskPublisher(connection)
inqueue = self.inqueue
while 1:
task, args, kwargs, options, receipt = inqueue.get()
result = task.apply_async(args, kwargs,
publisher=publisher,
**options)
receipt.finished(result)
class ProducerPool(object):
"""Usage::
>>> app = Celery(broker='amqp://')
>>> ProducerPool(app)
"""
Receipt = Receipt
def __init__(self, app, size=20):
self.app = app
self.size = size
self.inqueue = LightQueue()
self._running = None
self._producers = None
def apply_async(self, task, args, kwargs, callback=None, **options):
if self._running is None:
self._running = spawn_n(self._run)
receipt = self.Receipt(callback)
self.inqueue.put((task, args, kwargs, options, receipt))
return receipt
def _run(self):
self._producers = [spawn_n(self._producer) for _ in range(self.size)]
def _producer(self):
inqueue = self.inqueue
with self.app.producer_or_acquire() as producer:
while 1:
task, args, kwargs, options, receipt = inqueue.get()
result = task.apply_async(
args, kwargs, producer=producer, **options)
receipt.finished(result)
def get(self, block=True, timeout=None):
event = LightQueue(1)
if flask.has_request_context():
emit('{}#get'.format(self._uuid), callback=lambda x: event.put(x))
else:
sio = flask.current_app.extensions['socketio']
sio.emit('{}#get'.format(self._uuid),
callback=lambda x: event.put(x))
return event.get(timeout=1)
def get(self, timeout=10):
name = getter.__name__
# pylint: disable=protected-access
signal = '{uuid}#{event}'.format(uuid=self._uuid, event=name)
event = LightQueue(1)
if flask.has_request_context():
emit(signal, callback=lambda x: event.put(unpack(x)))
else:
sio = flask.current_app.extensions['socketio']
sio.emit(signal, callback=lambda x: event.put(unpack(x)))
data = event.get(timeout=timeout)
return getter(self, data)
def get(self, timeout=10): # pylint: disable=missing-docstring
name = getter.__name__
signal = '{uuid}{sep}{event}'.format(
uuid=self._uuid, # pylint: disable=protected-access
sep=SEPARATOR,
event=name
)
event = LightQueue(1)
if flask.has_request_context():
emit(signal, callback=lambda x: event.put(unpack(x)))
else:
sio = flask.current_app.extensions['socketio']
sio.emit(signal, callback=lambda x: event.put(unpack(x)))
data = event.get(timeout=timeout)
return getter(self, data)
class LocalMailbox(Mailbox):
__slots__ = ['_queue']
def __init__(self):
self._queue = LightQueue()
def put(self, message):
self._queue.put(message)
def get(self):
return self._queue.get(block=True)
def encode(self):
raise NotImplementedError
@staticmethod
def decode(params):
raise NotImplementedError
class ProducerPool(object):
"""Usage::
>>> app = Celery(broker='amqp://')
>>> ProducerPool(app)
"""
Receipt = Receipt
def __init__(self, app, size=20):
self.app = app
self.size = size
self.inqueue = LightQueue()
self._running = None
self._producers = None
def apply_async(self, task, args, kwargs, callback=None, **options):
if self._running is None:
self._running = spawn_n(self._run)
receipt = self.Receipt(callback)
self.inqueue.put((task, args, kwargs, options, receipt))
return receipt
def _run(self):
self._producers = [
spawn_n(self._producer) for _ in range(self.size)
]
def _producer(self):
inqueue = self.inqueue
with self.app.producer_or_acquire() as producer:
while 1:
task, args, kwargs, options, receipt = inqueue.get()
result = task.apply_async(args, kwargs,
producer=producer,
**options)
receipt.finished(result)
class VCenter(object):
# PropertyCollector discovers changes on vms and their hardware and produces
# (mac, switch, portKey, portGroupKey, connectable.connected, connectable.status)
# internally, it keeps internally vm and key for identifying updates
# Subsequently, the mac has to be identified with a port
#
def __init__(self, config=None, pool=None, agent=None):
self.pool = pool
self.agent = agent
self.config = config or CONF.ML2_VMWARE
self.connection = _create_session(self.config)
self.context = agent.context
self._monitor_process = VCenterMonitor(self,
self.config,
connection=self.connection,
pool=self.pool)
self.queue = Queue(None)
self.uuid_port_map = {}
self.mac_port_map = {}
self.uuid_dvs_map = {}
self.network_dvs_map = {}
for network, dvs in six.iteritems(
dvs_util.create_network_map_from_config(
self.config, connection=self.connection, pool=pool)):
self.network_dvs_map[network] = dvs
self.uuid_dvs_map[dvs.uuid] = dvs
def vcenter_port_changes(self, changed):
ports_by_mac = defaultdict(dict)
for port_desc in changed:
port = {
'port_desc': port_desc,
'port': {
'binding:vif_details': {
'dvs_port_key': port_desc.port_key,
'dvs_uuid': port_desc.dvs_uuid,
},
'mac_address': port_desc.mac_address
}
}
dvs = self.get_dvs_by_uuid(port_desc.dvs_uuid)
if not dvs:
continue
if port_desc.status != 'deleted':
dvs.ports_by_key[port_desc.port_key] = port
ports_by_mac[port_desc.mac_address] = port
else:
dvs.ports_by_key.pop(port_desc.port_key, None)
ports_by_mac.pop(port_desc.mac_address, None)
# self.read_dvs_ports(ports_by_mac) Skip that
macs = set(six.iterkeys(ports_by_mac))
port_list = []
for port_id, mac, status, admin_state_up, network_id, network_type, segmentation_id in self.get_ports_by_mac(
macs):
macs.discard(mac)
port_info = ports_by_mac[mac]
neutron_info = {
"port_id": port_id,
"id": port_id,
"device": port_id,
"mac_address": mac,
"admin_state_up": admin_state_up,
"status": status,
"network_id": network_id,
"network_type": network_type,
"segmentation_id": segmentation_id,
}
port_info["port"]["id"] = port_id
c_util.dict_merge(port_info, neutron_info)
self.uuid_port_map[port_id] = port_info
port_list.append(port_info)
if macs:
LOG.warning(
_LW("Could not find the following macs: {}").format(macs))
LOG.debug("Got port information from db for %d ports", len(port_list))
for port in port_list:
self.queue.put(port)
def start(self):
self._monitor_process.start()
@staticmethod
def update_port_desc(port, port_info):
# Validate connectionCookie, so we still have the same instance behind that portKey
port_desc = port['port_desc']
connection_cookie = _cast(getattr(port_info, 'connectionCookie', None))
if port_desc.connection_cookie != connection_cookie:
LOG.error("Cookie mismatch {} {} {} <> {}".format(
port_desc.mac_address, port_desc.port_key,
port_desc.connection_cookie, connection_cookie))
return False
for k, v in six.iteritems(_DVSPortDesc.from_dvs_port(port_info)):
setattr(port_desc, k, v)
return True
def ports_by_switch_and_key(self, ports):
ports_by_switch_and_key = defaultdict(dict)
for port in ports:
port_desc = port['port_desc']
dvs = self.get_dvs_by_uuid(port_desc.dvs_uuid)
if dvs:
ports_by_switch_and_key[dvs][port_desc.port_key] = port
return ports_by_switch_and_key
@c_util.stats.timed()
def bind_ports(self, ports, callback=None):
ports_by_switch_and_key = self.ports_by_switch_and_key(ports)
for dvs, ports_by_key in six.iteritems(ports_by_switch_and_key):
specs = []
for port in six.itervalues(ports_by_key):
if (port["network_type"] == "vlan" and not port["segmentation_id"] is None) \
or port["network_type"] == "flat":
spec = builder.neutron_to_port_config_spec(port)
if not CONF.AGENT.dry_run:
specs.append(spec)
else:
LOG.debug(spec)
dvs.queue_update_specs(specs, callback=callback)
def get_dvs_by_uuid(self, uuid):
return self.uuid_dvs_map.get(uuid, None)
def get_port_by_uuid(self, uuid):
return self.uuid_port_map.get(uuid, None)
def fetch_ports_by_mac(self, portgroup_key=None, mac_addr=None):
for dvs in six.itervalues(self.uuid_dvs_map):
port_keys = dvs._dvs.FetchDVPortKeys(
dvs._dvs, criteria=builder.port_criteria())
ports = dvs._dvs.FetchDVPorts(criteria=builder.port_criteria(
port_group_key=portgroup_key, port_key=port_keys))
for port in ports:
if hasattr(port, 'state'):
if hasattr(port.state, 'runtimeInfo'):
if mac_addr == port.state.runtimeInfo.macAddress:
return port
else:
continue
raise Exception('DVS port not found!')
def get_new_ports(self, block=False, timeout=1.0, max_ports=None):
ports_by_mac = defaultdict(dict)
try:
while max_ports is None or len(ports_by_mac) < max_ports:
new_port = self.queue.get(block=block, timeout=timeout)
port_desc = new_port['port_desc']
block = False # Only block on the first item
if port_desc.status == 'deleted':
ports_by_mac.pop(port_desc.mac_address, None)
port = self.mac_port_map.pop(port_desc.mac_address, None)
if port:
port_desc = port['port_desc']
self.uuid_port_map.pop(port['id'], None)
dvs = self.get_dvs_by_uuid(port_desc.dvs_uuid)
dvs.ports_by_key.pop(port_desc.port_key, None)
else:
port = self.mac_port_map.get(port_desc.mac_address, {})
port.update(dict(new_port))
ports_by_mac[port_desc.mac_address] = port
dvs = self.get_dvs_by_uuid(port_desc.dvs_uuid)
if dvs:
dvs.ports_by_key[port_desc.port_key] = port
except Empty:
pass
return ports_by_mac
def read_dvs_ports(self, ports_by_mac):
ports_by_switch_and_key = self.ports_by_switch_and_key(
six.itervalues(ports_by_mac))
# This loop can get very slow, if get_port_info_by_portkey gets port keys passed of instances, which are only
# partly connected, meaning: the instance is associated, but the link is not quite up yet
for dvs, ports_by_key in six.iteritems(ports_by_switch_and_key):
for port_info in dvs.get_port_info_by_portkey(
list(
six.iterkeys(ports_by_key))): # View is not sufficient
port = ports_by_key[port_info.key]
if not VCenter.update_port_desc(port, port_info):
port_desc = port['port_desc']
ports_by_mac.pop(port_desc.mac_address)
LOG.debug("Read all ports")
@enginefacade.reader
def get_ports_by_mac(self, mac_addresses):
if not mac_addresses:
return []
if not self.context:
self.context = neutron.context.get_admin_context()
session = self.context.session
with session.begin(subtransactions=True):
return session.query(models_v2.Port.id, models_v2.Port.mac_address, models_v2.Port.status, models_v2.Port.admin_state_up,
models_ml2.NetworkSegment.network_id,
models_ml2.NetworkSegment.network_type,
models_ml2.NetworkSegment.segmentation_id).\
join(models_ml2.PortBindingLevel, models_v2.Port.id == models_ml2.PortBindingLevel.port_id).\
join(models_ml2.NetworkSegment, models_ml2.PortBindingLevel.segment_id == models_ml2.NetworkSegment.id).\
filter(models_ml2.PortBindingLevel.host == self.agent.conf.host,
models_ml2.PortBindingLevel.driver == constants.DVS,
models_v2.Port.mac_address.in_(mac_addresses),
).all()
def stop(self):
self._monitor_process.stop()
try:
while True:
self.queue.get_nowait()
except Empty:
pass