class ProducerPool(object): Receipt = Receipt def __init__(self, size=20): self.size = size self.inqueue = LightQueue() self._running = None self._producers = None def apply_async(self, task, args, kwargs, callback=None, **options): if self._running is None: self._running = spawn_n(self._run) receipt = self.Receipt(callback) self.inqueue.put((task, args, kwargs, options, receipt)) return receipt def _run(self): self._producers = [spawn_n(self._producer) for _ in xrange(self.size)] def _producer(self): connection = current_app.broker_connection() publisher = current_app.amqp.TaskProducer(connection) inqueue = self.inqueue while 1: task, args, kwargs, options, receipt = inqueue.get() result = task.apply_async(args, kwargs, publisher=publisher, **options) receipt.finished(result)
class ProducerPool(object): Receipt = Receipt def __init__(self, size=20): self.size = size self.inqueue = LightQueue() self._running = None self._producers = None def apply_async(self, task, args, kwargs, callback=None, **options): if self._running is None: self._running = spawn_n(self._run) receipt = self.Receipt(callback) self.inqueue.put((task, args, kwargs, options, receipt)) return receipt def _run(self): self._producers = [spawn_n(self._producer) for _ in xrange(self.size)] def _producer(self): connection = current_app.broker_connection() publisher = current_app.amqp.TaskPublisher(connection) inqueue = self.inqueue while 1: task, args, kwargs, options, receipt = inqueue.get() result = task.apply_async(args, kwargs, publisher=publisher, **options) receipt.finished(result)
class ProducerPool(object): """Usage:: >>> app = Celery(broker='amqp://') >>> ProducerPool(app) """ Receipt = Receipt def __init__(self, app, size=20): self.app = app self.size = size self.inqueue = LightQueue() self._running = None self._producers = None def apply_async(self, task, args, kwargs, callback=None, **options): if self._running is None: self._running = spawn_n(self._run) receipt = self.Receipt(callback) self.inqueue.put((task, args, kwargs, options, receipt)) return receipt def _run(self): self._producers = [spawn_n(self._producer) for _ in range(self.size)] def _producer(self): inqueue = self.inqueue with self.app.producer_or_acquire() as producer: while 1: task, args, kwargs, options, receipt = inqueue.get() result = task.apply_async( args, kwargs, producer=producer, **options) receipt.finished(result)
def get(self, block=True, timeout=None): event = LightQueue(1) if flask.has_request_context(): emit('{}#get'.format(self._uuid), callback=lambda x: event.put(x)) else: sio = flask.current_app.extensions['socketio'] sio.emit('{}#get'.format(self._uuid), callback=lambda x: event.put(x)) return event.get(timeout=1)
def get(self, timeout=10): name = getter.__name__ # pylint: disable=protected-access signal = '{uuid}#{event}'.format(uuid=self._uuid, event=name) event = LightQueue(1) if flask.has_request_context(): emit(signal, callback=lambda x: event.put(unpack(x))) else: sio = flask.current_app.extensions['socketio'] sio.emit(signal, callback=lambda x: event.put(unpack(x))) data = event.get(timeout=timeout) return getter(self, data)
def get(self, timeout=10): # pylint: disable=missing-docstring name = getter.__name__ signal = '{uuid}{sep}{event}'.format( uuid=self._uuid, # pylint: disable=protected-access sep=SEPARATOR, event=name ) event = LightQueue(1) if flask.has_request_context(): emit(signal, callback=lambda x: event.put(unpack(x))) else: sio = flask.current_app.extensions['socketio'] sio.emit(signal, callback=lambda x: event.put(unpack(x))) data = event.get(timeout=timeout) return getter(self, data)
class LocalMailbox(Mailbox): __slots__ = ['_queue'] def __init__(self): self._queue = LightQueue() def put(self, message): self._queue.put(message) def get(self): return self._queue.get(block=True) def encode(self): raise NotImplementedError @staticmethod def decode(params): raise NotImplementedError
class ProducerPool(object): """Usage:: >>> app = Celery(broker='amqp://') >>> ProducerPool(app) """ Receipt = Receipt def __init__(self, app, size=20): self.app = app self.size = size self.inqueue = LightQueue() self._running = None self._producers = None def apply_async(self, task, args, kwargs, callback=None, **options): if self._running is None: self._running = spawn_n(self._run) receipt = self.Receipt(callback) self.inqueue.put((task, args, kwargs, options, receipt)) return receipt def _run(self): self._producers = [ spawn_n(self._producer) for _ in range(self.size) ] def _producer(self): inqueue = self.inqueue with self.app.producer_or_acquire() as producer: while 1: task, args, kwargs, options, receipt = inqueue.get() result = task.apply_async(args, kwargs, producer=producer, **options) receipt.finished(result)
class VCenter(object): # PropertyCollector discovers changes on vms and their hardware and produces # (mac, switch, portKey, portGroupKey, connectable.connected, connectable.status) # internally, it keeps internally vm and key for identifying updates # Subsequently, the mac has to be identified with a port # def __init__(self, config=None, pool=None, agent=None): self.pool = pool self.agent = agent self.config = config or CONF.ML2_VMWARE self.connection = _create_session(self.config) self.context = agent.context self._monitor_process = VCenterMonitor(self, self.config, connection=self.connection, pool=self.pool) self.queue = Queue(None) self.uuid_port_map = {} self.mac_port_map = {} self.uuid_dvs_map = {} self.network_dvs_map = {} for network, dvs in six.iteritems( dvs_util.create_network_map_from_config( self.config, connection=self.connection, pool=pool)): self.network_dvs_map[network] = dvs self.uuid_dvs_map[dvs.uuid] = dvs def vcenter_port_changes(self, changed): ports_by_mac = defaultdict(dict) for port_desc in changed: port = { 'port_desc': port_desc, 'port': { 'binding:vif_details': { 'dvs_port_key': port_desc.port_key, 'dvs_uuid': port_desc.dvs_uuid, }, 'mac_address': port_desc.mac_address } } dvs = self.get_dvs_by_uuid(port_desc.dvs_uuid) if not dvs: continue if port_desc.status != 'deleted': dvs.ports_by_key[port_desc.port_key] = port ports_by_mac[port_desc.mac_address] = port else: dvs.ports_by_key.pop(port_desc.port_key, None) ports_by_mac.pop(port_desc.mac_address, None) # self.read_dvs_ports(ports_by_mac) Skip that macs = set(six.iterkeys(ports_by_mac)) port_list = [] for port_id, mac, status, admin_state_up, network_id, network_type, segmentation_id in self.get_ports_by_mac( macs): macs.discard(mac) port_info = ports_by_mac[mac] neutron_info = { "port_id": port_id, "id": port_id, "device": port_id, "mac_address": mac, "admin_state_up": admin_state_up, "status": status, "network_id": network_id, "network_type": network_type, "segmentation_id": segmentation_id, } port_info["port"]["id"] = port_id c_util.dict_merge(port_info, neutron_info) self.uuid_port_map[port_id] = port_info port_list.append(port_info) if macs: LOG.warning( _LW("Could not find the following macs: {}").format(macs)) LOG.debug("Got port information from db for %d ports", len(port_list)) for port in port_list: self.queue.put(port) def start(self): self._monitor_process.start() @staticmethod def update_port_desc(port, port_info): # Validate connectionCookie, so we still have the same instance behind that portKey port_desc = port['port_desc'] connection_cookie = _cast(getattr(port_info, 'connectionCookie', None)) if port_desc.connection_cookie != connection_cookie: LOG.error("Cookie mismatch {} {} {} <> {}".format( port_desc.mac_address, port_desc.port_key, port_desc.connection_cookie, connection_cookie)) return False for k, v in six.iteritems(_DVSPortDesc.from_dvs_port(port_info)): setattr(port_desc, k, v) return True def ports_by_switch_and_key(self, ports): ports_by_switch_and_key = defaultdict(dict) for port in ports: port_desc = port['port_desc'] dvs = self.get_dvs_by_uuid(port_desc.dvs_uuid) if dvs: ports_by_switch_and_key[dvs][port_desc.port_key] = port return ports_by_switch_and_key @c_util.stats.timed() def bind_ports(self, ports, callback=None): ports_by_switch_and_key = self.ports_by_switch_and_key(ports) for dvs, ports_by_key in six.iteritems(ports_by_switch_and_key): specs = [] for port in six.itervalues(ports_by_key): if (port["network_type"] == "vlan" and not port["segmentation_id"] is None) \ or port["network_type"] == "flat": spec = builder.neutron_to_port_config_spec(port) if not CONF.AGENT.dry_run: specs.append(spec) else: LOG.debug(spec) dvs.queue_update_specs(specs, callback=callback) def get_dvs_by_uuid(self, uuid): return self.uuid_dvs_map.get(uuid, None) def get_port_by_uuid(self, uuid): return self.uuid_port_map.get(uuid, None) def fetch_ports_by_mac(self, portgroup_key=None, mac_addr=None): for dvs in six.itervalues(self.uuid_dvs_map): port_keys = dvs._dvs.FetchDVPortKeys( dvs._dvs, criteria=builder.port_criteria()) ports = dvs._dvs.FetchDVPorts(criteria=builder.port_criteria( port_group_key=portgroup_key, port_key=port_keys)) for port in ports: if hasattr(port, 'state'): if hasattr(port.state, 'runtimeInfo'): if mac_addr == port.state.runtimeInfo.macAddress: return port else: continue raise Exception('DVS port not found!') def get_new_ports(self, block=False, timeout=1.0, max_ports=None): ports_by_mac = defaultdict(dict) try: while max_ports is None or len(ports_by_mac) < max_ports: new_port = self.queue.get(block=block, timeout=timeout) port_desc = new_port['port_desc'] block = False # Only block on the first item if port_desc.status == 'deleted': ports_by_mac.pop(port_desc.mac_address, None) port = self.mac_port_map.pop(port_desc.mac_address, None) if port: port_desc = port['port_desc'] self.uuid_port_map.pop(port['id'], None) dvs = self.get_dvs_by_uuid(port_desc.dvs_uuid) dvs.ports_by_key.pop(port_desc.port_key, None) else: port = self.mac_port_map.get(port_desc.mac_address, {}) port.update(dict(new_port)) ports_by_mac[port_desc.mac_address] = port dvs = self.get_dvs_by_uuid(port_desc.dvs_uuid) if dvs: dvs.ports_by_key[port_desc.port_key] = port except Empty: pass return ports_by_mac def read_dvs_ports(self, ports_by_mac): ports_by_switch_and_key = self.ports_by_switch_and_key( six.itervalues(ports_by_mac)) # This loop can get very slow, if get_port_info_by_portkey gets port keys passed of instances, which are only # partly connected, meaning: the instance is associated, but the link is not quite up yet for dvs, ports_by_key in six.iteritems(ports_by_switch_and_key): for port_info in dvs.get_port_info_by_portkey( list( six.iterkeys(ports_by_key))): # View is not sufficient port = ports_by_key[port_info.key] if not VCenter.update_port_desc(port, port_info): port_desc = port['port_desc'] ports_by_mac.pop(port_desc.mac_address) LOG.debug("Read all ports") @enginefacade.reader def get_ports_by_mac(self, mac_addresses): if not mac_addresses: return [] if not self.context: self.context = neutron.context.get_admin_context() session = self.context.session with session.begin(subtransactions=True): return session.query(models_v2.Port.id, models_v2.Port.mac_address, models_v2.Port.status, models_v2.Port.admin_state_up, models_ml2.NetworkSegment.network_id, models_ml2.NetworkSegment.network_type, models_ml2.NetworkSegment.segmentation_id).\ join(models_ml2.PortBindingLevel, models_v2.Port.id == models_ml2.PortBindingLevel.port_id).\ join(models_ml2.NetworkSegment, models_ml2.PortBindingLevel.segment_id == models_ml2.NetworkSegment.id).\ filter(models_ml2.PortBindingLevel.host == self.agent.conf.host, models_ml2.PortBindingLevel.driver == constants.DVS, models_v2.Port.mac_address.in_(mac_addresses), ).all() def stop(self): self._monitor_process.stop() try: while True: self.queue.get_nowait() except Empty: pass