def start(address): global _running if _running: raise RuntimeError('trying to start reporter while running') logging.info("Starting hawkular reporter") concurrent.thread(_run, name='hawkular', args=(address,)).start() _running = True
def start(address): global _running if _running: raise RuntimeError('trying to start reporter while running') logging.info("Starting hawkular reporter") concurrent.thread(_run, name='hawkular', args=(address, )).start() _running = True
def __init__(self, irs, log, scheduler): """ Initialize the (single) clientIF instance :param irs: a Dispatcher object to be used as this object's irs. :type irs: :class:`storage.dispatcher.Dispatcher` :param log: a log object to be used for this object's logging. :type log: :class:`logging.Logger` """ self.vmContainerLock = threading.Lock() self._networkSemaphore = threading.Semaphore() self._shutdownSemaphore = threading.Semaphore() self.irs = irs if self.irs: self._contEIOVmsCB = partial(clientIF.contEIOVms, proxy(self)) self.irs.registerDomainStateChangeCallback(self._contEIOVmsCB) self.log = log self._recovery = True self.channelListener = Listener(self.log) self._generationID = str(uuid.uuid4()) self.mom = None self.bindings = {} self._broker_client = None self._subscriptions = defaultdict(list) self._scheduler = scheduler if _glusterEnabled: self.gluster = gapi.GlusterApi(self, log) else: self.gluster = None try: self.vmContainer = {} self._hostStats = sampling.HostStatsThread( sampling.host_samples) self._hostStats.start() self.lastRemoteAccess = 0 self._enabled = True self._netConfigDirty = False self._prepareMOM() secret.clear() concurrent.thread(self._recoverThread, name='clientIFinit').start() self.channelListener.settimeout( config.getint('vars', 'guest_agent_timeout')) self.channelListener.start() self.threadLocal = threading.local() self.threadLocal.client = '' host = config.get('addresses', 'management_ip') port = config.getint('addresses', 'management_port') self._createAcceptor(host, port) self._prepareXMLRPCBinding() self._prepareJSONRPCBinding() self._connectToBroker() except: self.log.error('failed to init clientIF, ' 'shutting down storage dispatcher') if self.irs: self.irs.prepareForShutdown() raise
def __init__(self, irs, log, scheduler): """ Initialize the (single) clientIF instance :param irs: a Dispatcher object to be used as this object's irs. :type irs: :class:`storage.dispatcher.Dispatcher` :param log: a log object to be used for this object's logging. :type log: :class:`logging.Logger` """ self.vmContainerLock = threading.Lock() self._networkSemaphore = threading.Semaphore() self._shutdownSemaphore = threading.Semaphore() self.irs = irs if self.irs: self._contEIOVmsCB = partial(clientIF.contEIOVms, proxy(self)) self.irs.registerDomainStateChangeCallback(self._contEIOVmsCB) self.log = log self._recovery = True self.channelListener = Listener(self.log) self._generationID = str(uuid.uuid4()) self.mom = None self.bindings = {} self._broker_client = None self._subscriptions = defaultdict(list) self._scheduler = scheduler if _glusterEnabled: self.gluster = gapi.GlusterApi(self, log) else: self.gluster = None try: self.vmContainer = {} self._hostStats = sampling.HostStatsThread(sampling.host_samples) self._hostStats.start() self.lastRemoteAccess = 0 self._enabled = True self._netConfigDirty = False self._prepareMOM() secret.clear() concurrent.thread(self._recoverThread, name='clientIFinit').start() self.channelListener.settimeout( config.getint('vars', 'guest_agent_timeout')) self.channelListener.start() self.threadLocal = threading.local() self.threadLocal.client = '' host = config.get('addresses', 'management_ip') port = config.getint('addresses', 'management_port') self._createAcceptor(host, port) self._prepareXMLRPCBinding() self._prepareJSONRPCBinding() self._connectToBroker() except: self.log.error('failed to init clientIF, ' 'shutting down storage dispatcher') if self.irs: self.irs.prepareForShutdown() raise
def __del__(self): def finalize(log, owner, taskDir): log.warn("Task was autocleaned") owner.releaseAll() if taskDir is not None: getProcPool().fileUtils.cleanupdir(taskDir) if not self.state.isDone(): taskDir = None if (self.cleanPolicy == TaskCleanType.auto and self.store is not None): taskDir = os.path.join(self.store, self.id) concurrent.thread(finalize, args=(self.log, self.resOwner, taskDir)).start()
def __init__(self, sdUUID, hostId, interval, changeEvent, checker): self.thread = concurrent.thread(self._run, logger=log.name, name="monitor/" + sdUUID[:7]) self.stopEvent = threading.Event() self.domain = None self.sdUUID = sdUUID self.hostId = hostId self.interval = interval self.changeEvent = changeEvent self.checker = checker self.lock = threading.Lock() self.monitoringPath = None # For backward compatibility, we must present a fake status before # collecting the first sample. The fake status is marked as # actual=False so engine can handle it correctly. self.status = Status(PathStatus(actual=False), DomainStatus(actual=False)) self.isIsoDomain = None self.isoPrefix = None self.lastRefresh = time.time() # Use float to allow short refresh internal during tests. self.refreshTime = \ config.getfloat("irs", "repo_stats_cache_refresh_timeout") self.wasShutdown = False # Used for synchronizing during the tests self.cycleCallback = None
def start(self, blocking): if blocking: return self._dhclient() else: t = concurrent.thread(self._dhclient, name='dhclient/%s' % self.iface) t.start()
def test_non_daemon_thread(self): t = concurrent.thread(lambda: None, daemon=False) t.start() try: self.assertFalse(t.daemon) finally: t.join()
def test_default_daemon_thread(self): t = concurrent.thread(lambda: None) t.start() try: self.assertTrue(t.daemon) finally: t.join()
def __init__(self, inbox, outbox, hostID, queue, monitorInterval): # Save arguments tpSize = config.getint('irs', 'thread_pool_size') / 2 waitTimeout = 3 maxTasks = config.getint('irs', 'max_tasks') self.tp = ThreadPool(tpSize, waitTimeout, maxTasks) self._stop = False self._flush = False self._queue = queue self._activeMessages = {} self._monitorInterval = monitorInterval self._hostID = int(hostID) self._used_slots_array = [0] * MESSAGES_PER_MAILBOX self._outgoingMail = EMPTYMAILBOX self._incomingMail = EMPTYMAILBOX # TODO: add support for multiple paths (multiple mailboxes) self._spmStorageDir = config.get('irs', 'repository') self._inCmd = [ constants.EXT_DD, 'if=' + str(inbox), 'iflag=direct,fullblock', 'bs=' + str(BLOCK_SIZE), 'count=' + str(BLOCKS_PER_MAILBOX), 'skip=' + str(self._hostID * BLOCKS_PER_MAILBOX) ] self._outCmd = [ constants.EXT_DD, 'of=' + str(outbox), 'iflag=fullblock', 'oflag=direct', 'conv=notrunc', 'bs=' + str(BLOCK_SIZE), 'seek=' + str(self._hostID * BLOCKS_PER_MAILBOX) ] self._init = False self._initMailbox() # Read initial mailbox state self._msgCounter = 0 self._sendMail() # Clear outgoing mailbox self._thread = concurrent.thread(self.run, name="mailbox/hsm", logger=self.log.name) self._thread.start()
def __init__(self, sdUUID, hostId, interval, changeEvent, checker): self.thread = concurrent.thread(self._run, logger=log.name) self.stopEvent = threading.Event() self.domain = None self.sdUUID = sdUUID self.hostId = hostId self.interval = interval self.changeEvent = changeEvent self.checker = checker self.lock = threading.Lock() self.monitoringPath = None # For backward compatibility, we must present a fake status before # collecting the first sample. The fake status is marked as # actual=False so engine can handle it correctly. self.status = Status(PathStatus(actual=False), DomainStatus(actual=False)) self.isIsoDomain = None self.isoPrefix = None self.lastRefresh = time.time() # Use float to allow short refresh internal during tests. self.refreshTime = \ config.getfloat("irs", "repo_stats_cache_refresh_timeout") self.wasShutdown = False # Used for synchronizing during the tests self.cycleCallback = None
def test_echo(self, concurrency): msg = b"ping" sockets = [] try: for i in range(concurrency): sock1, sock2 = socket.socketpair() self.loop.create_dispatcher(Echo, sock2) sockets.append(sock1) t = concurrent.thread(self.loop.run_forever) t.start() try: start = time.time() for sock in sockets: osutils.uninterruptible(sock.send, msg) for sock in sockets: data = osutils.uninterruptible(sock.recv, len(msg)) self.assertEqual(data, msg) elapsed = time.time() - start print("%7d echos: %f seconds" % (concurrency, elapsed)) finally: self.loop.call_soon_threadsafe(self.loop.stop) t.join() finally: for sock in sockets: sock.close()
def __init__(self, pool, maxHostID, monitorInterval=2): self._messageTypes = {} # Save arguments self._stop = False self._stopped = False self._poolID = str(pool.spUUID) self._spmStorageDir = pool.storage_repository tpSize = config.getint('irs', 'thread_pool_size') / 2 waitTimeout = 3 maxTasks = config.getint('irs', 'max_tasks') self.tp = ThreadPool(tpSize, waitTimeout, maxTasks) # *** IMPORTANT NOTE: The SPM's inbox is the HSMs' outbox and vice # versa *** # self._inbox = os.path.join(self._spmStorageDir, self._poolID, "mastersd", sd.DOMAIN_META_DATA, "inbox") if not os.path.exists(self._inbox): self.log.error("SPM_MailMonitor create failed - inbox %s does not " "exist" % repr(self._inbox)) raise RuntimeError("SPM_MailMonitor create failed - inbox %s does " "not exist" % repr(self._inbox)) self._outbox = os.path.join(self._spmStorageDir, self._poolID, "mastersd", sd.DOMAIN_META_DATA, "outbox") if not os.path.exists(self._outbox): self.log.error("SPM_MailMonitor create failed - outbox %s does " "not exist" % repr(self._outbox)) raise RuntimeError("SPM_MailMonitor create failed - outbox %s " "does not exist" % repr(self._outbox)) self._numHosts = int(maxHostID) self._outMailLen = MAILBOX_SIZE * self._numHosts self._monitorInterval = monitorInterval # TODO: add support for multiple paths (multiple mailboxes) self._outgoingMail = self._outMailLen * "\0" self._incomingMail = self._outgoingMail self._inCmd = ['dd', 'if=' + str(self._inbox), 'iflag=direct,fullblock', 'count=1' ] self._outCmd = ['dd', 'of=' + str(self._outbox), 'oflag=direct', 'iflag=fullblock', 'conv=notrunc', 'count=1' ] self._outLock = threading.Lock() self._inLock = threading.Lock() # Clear outgoing mail self.log.debug("SPM_MailMonitor - clearing outgoing mail, command is: " "%s", self._outCmd) cmd = self._outCmd + ['bs=' + str(self._outMailLen)] (rc, out, err) = _mboxExecCmd(cmd, data=self._outgoingMail) if rc: self.log.warning("SPM_MailMonitor couldn't clear outgoing mail, " "dd failed") t = concurrent.thread(self.run, name="mailbox.SPMMonitor", logger=self.log.name) t.start() self.log.debug('SPM_MailMonitor created for pool %s' % self._poolID)
def __init__(self, pool, maxHostID, monitorInterval=2): self._messageTypes = {} # Save arguments self._stop = False self._stopped = False self._poolID = str(pool.spUUID) self._spmStorageDir = pool.storage_repository tpSize = config.getint('irs', 'thread_pool_size') / 2 waitTimeout = 3 maxTasks = config.getint('irs', 'max_tasks') self.tp = ThreadPool("mailbox-spm", tpSize, waitTimeout, maxTasks) # *** IMPORTANT NOTE: The SPM's inbox is the HSMs' outbox and vice # versa *** # self._inbox = os.path.join(self._spmStorageDir, self._poolID, "mastersd", sd.DOMAIN_META_DATA, "inbox") if not os.path.exists(self._inbox): self.log.error("SPM_MailMonitor create failed - inbox %s does not " "exist" % repr(self._inbox)) raise RuntimeError("SPM_MailMonitor create failed - inbox %s does " "not exist" % repr(self._inbox)) self._outbox = os.path.join(self._spmStorageDir, self._poolID, "mastersd", sd.DOMAIN_META_DATA, "outbox") if not os.path.exists(self._outbox): self.log.error("SPM_MailMonitor create failed - outbox %s does " "not exist" % repr(self._outbox)) raise RuntimeError("SPM_MailMonitor create failed - outbox %s " "does not exist" % repr(self._outbox)) self._numHosts = int(maxHostID) self._outMailLen = MAILBOX_SIZE * self._numHosts self._monitorInterval = monitorInterval # TODO: add support for multiple paths (multiple mailboxes) self._outgoingMail = self._outMailLen * "\0" self._incomingMail = self._outgoingMail self._inCmd = ['dd', 'if=' + str(self._inbox), 'iflag=direct,fullblock', 'count=1' ] self._outCmd = ['dd', 'of=' + str(self._outbox), 'oflag=direct', 'iflag=fullblock', 'conv=notrunc', 'count=1' ] self._outLock = threading.Lock() self._inLock = threading.Lock() # Clear outgoing mail self.log.debug("SPM_MailMonitor - clearing outgoing mail, command is: " "%s", self._outCmd) cmd = self._outCmd + ['bs=' + str(self._outMailLen)] (rc, out, err) = _mboxExecCmd(cmd, data=self._outgoingMail) if rc: self.log.warning("SPM_MailMonitor couldn't clear outgoing mail, " "dd failed") t = concurrent.thread(self.run, name="mailbox-spm", logger=self.log.name) t.start() self.log.debug('SPM_MailMonitor created for pool %s' % self._poolID)
def __del__(self): if self._isValid and self.autoRelease: def release(log, namespace, name): log.warn("Resource reference was not properly released. " "Autoreleasing.") # In Python, objects are refcounted and are deleted immediately # when the last reference is freed. This means the __del__ # method can be called inside of any context. The # releaseResource method we use tries to acquire locks. So we # might try to acquire the lock in a locked context and reach a # deadlock. This is why I need to use a timer. It will defer # the operation and use a different context. ResourceManager.getInstance().releaseResource(namespace, name) concurrent.thread(release, args=(self._log, self.namespace, self.name)).start() self._isValid = False
def __init__(self, vm, dst='', dstparams='', mode=MODE_REMOTE, method=METHOD_ONLINE, tunneled=False, dstqemu='', abortOnError=False, consoleAddress=None, compressed=False, autoConverge=False, **kwargs): self.log = vm.log self._vm = vm self._dst = dst self._mode = mode if method != METHOD_ONLINE: self.log.warning( 'migration method %s is deprecated, forced to "online"', method) self._dstparams = dstparams self._enableGuestEvents = kwargs.get('enableGuestEvents', False) self._machineParams = {} # TODO: utils.tobool shouldn't be used in this constructor, the # conversions should be handled properly in the API layer self._tunneled = utils.tobool(tunneled) self._abortOnError = utils.tobool(abortOnError) self._consoleAddress = consoleAddress self._dstqemu = dstqemu self._downtime = kwargs.get('downtime') or \ config.get('vars', 'migration_downtime') self._maxBandwidth = int( kwargs.get('maxBandwidth') or config.getint('vars', 'migration_max_bandwidth')) self._autoConverge = utils.tobool(autoConverge) self._compressed = utils.tobool(compressed) self._incomingLimit = kwargs.get('incomingLimit') self._outgoingLimit = kwargs.get('outgoingLimit') self.status = { 'status': { 'code': 0, 'message': 'Migration in progress' } } self._progress = 0 self._thread = concurrent.thread(self.run, name='migsrc/' + self._vm.id[:8]) self._preparingMigrationEvt = True self._migrationCanceledEvt = threading.Event() self._monitorThread = None self._destServer = None self._convergence_schedule = {'init': [], 'stalling': []} self._use_convergence_schedule = False if 'convergenceSchedule' in kwargs: self._convergence_schedule = kwargs.get('convergenceSchedule') self._use_convergence_schedule = True self.log.debug('convergence schedule set to: %s', str(self._convergence_schedule))
def __init__(self, poolID, maxHostID, inbox, outbox, monitorInterval=2): """ Note: inbox paramerter here should point to the HSM's outbox mailbox file, and vice versa. """ self._messageTypes = {} # Save arguments self._stop = False self._stopped = False self._poolID = poolID tpSize = config.getint('irs', 'thread_pool_size') / 2 waitTimeout = wait_timeout(monitorInterval) maxTasks = config.getint('irs', 'max_tasks') self.tp = ThreadPool("mailbox-spm", tpSize, waitTimeout, maxTasks) self._inbox = inbox if not os.path.exists(self._inbox): self.log.error("SPM_MailMonitor create failed - inbox %s does not " "exist" % repr(self._inbox)) raise RuntimeError("SPM_MailMonitor create failed - inbox %s does " "not exist" % repr(self._inbox)) self._outbox = outbox if not os.path.exists(self._outbox): self.log.error("SPM_MailMonitor create failed - outbox %s does " "not exist" % repr(self._outbox)) raise RuntimeError("SPM_MailMonitor create failed - outbox %s " "does not exist" % repr(self._outbox)) self._numHosts = int(maxHostID) self._outMailLen = MAILBOX_SIZE * self._numHosts self._monitorInterval = monitorInterval # TODO: add support for multiple paths (multiple mailboxes) self._outgoingMail = self._outMailLen * "\0" self._incomingMail = self._outgoingMail self._inCmd = ['dd', 'if=' + str(self._inbox), 'iflag=direct,fullblock', 'count=1' ] self._outCmd = ['dd', 'of=' + str(self._outbox), 'oflag=direct', 'iflag=fullblock', 'conv=notrunc', 'count=1' ] self._outLock = threading.Lock() self._inLock = threading.Lock() # Clear outgoing mail self.log.debug("SPM_MailMonitor - clearing outgoing mail, command is: " "%s", self._outCmd) cmd = self._outCmd + ['bs=' + str(self._outMailLen)] (rc, out, err) = _mboxExecCmd(cmd, data=self._outgoingMail) if rc: self.log.warning("SPM_MailMonitor couldn't clear outgoing mail, " "dd failed") self._thread = concurrent.thread( self.run, name="mailbox-spm", log=self.log) self._thread.start() self.log.debug('SPM_MailMonitor created for pool %s' % self._poolID)
def handle_request(self): sock, addr = self.queue.get() if sock is self._STOP: return self.log.info("Starting request handler for %s:%d", addr[0], addr[1]) t = concurrent.thread(self._process_requests, args=(sock, addr), log=self.log) t.start()
def __init__(self, vm, dst='', dstparams='', mode=MODE_REMOTE, method=METHOD_ONLINE, tunneled=False, dstqemu='', abortOnError=False, consoleAddress=None, compressed=False, autoConverge=False, recovery=False, **kwargs): self.log = vm.log self._vm = vm self._dst = dst self._mode = mode if method != METHOD_ONLINE: self.log.warning( 'migration method %s is deprecated, forced to "online"', method) self._dstparams = dstparams self._enableGuestEvents = kwargs.get('enableGuestEvents', False) self._machineParams = {} # TODO: conv.tobool shouldn't be used in this constructor, the # conversions should be handled properly in the API layer self._tunneled = conv.tobool(tunneled) self._abortOnError = conv.tobool(abortOnError) self._consoleAddress = consoleAddress self._dstqemu = dstqemu self._downtime = kwargs.get('downtime') or \ config.get('vars', 'migration_downtime') self._maxBandwidth = int( kwargs.get('maxBandwidth') or config.getint('vars', 'migration_max_bandwidth') ) self._autoConverge = conv.tobool(autoConverge) self._compressed = conv.tobool(compressed) self._incomingLimit = kwargs.get('incomingLimit') self._outgoingLimit = kwargs.get('outgoingLimit') self.status = { 'status': { 'code': 0, 'message': 'Migration in progress'}} # we need to guard against concurrent updates only self._lock = threading.Lock() self._progress = 0 self._thread = concurrent.thread( self.run, name='migsrc/' + self._vm.id[:8]) self._preparingMigrationEvt = True self._migrationCanceledEvt = threading.Event() self._monitorThread = None self._destServer = None self._convergence_schedule = { 'init': [], 'stalling': [] } self._use_convergence_schedule = False if 'convergenceSchedule' in kwargs: self._convergence_schedule = kwargs.get('convergenceSchedule') self._use_convergence_schedule = True self.log.debug('convergence schedule set to: %s', str(self._convergence_schedule)) self._started = False self._recovery = recovery
def progress(op, estimated_size): done = threading.Event() th = concurrent.thread(volume_progress, args=(op, done, estimated_size)) th.start() try: yield th finally: done.set() th.join()
def _emit(self, *args, **kwargs): self._log.debug("Emitting event") with self._syncRoot: for funcId, (funcRef, oneshot) in self._registrar.items(): func = funcRef() if func is None or oneshot: del self._registrar[funcId] if func is None: continue try: self._log.debug("Calling registered method `%s`", logUtils.funcName(func)) if self._sync: func(*args, **kwargs) else: concurrent.thread(func, args=args, kwargs=kwargs).start() except: self._log.warn("Could not run registered method because " "of an exception", exc_info=True) self._log.debug("Event emitted")
def test_run_callable_in_thread(self): self.thread = threading.current_thread() def run(): self.thread = threading.current_thread() t = concurrent.thread(run) t.start() t.join() self.assertEqual(t, self.thread)
def test_pass_args(self): self.args = () def run(*args): self.args = args t = concurrent.thread(run, args=(1, 2, 3)) t.start() t.join() self.assertEqual((1, 2, 3), self.args)
def test_pass_kwargs(self): self.kwargs = () def run(**kwargs): self.kwargs = kwargs kwargs = {'a': 1, 'b': 2} t = concurrent.thread(run, kwargs=kwargs) t.start() t.join() self.assertEqual(kwargs, self.kwargs)
def __init__(self, vm, startTime, conv_schedule, use_conv_schedule): super(MonitorThread, self).__init__() self._stop = threading.Event() self._vm = vm self._startTime = startTime self.daemon = True self.progress = None self._conv_schedule = conv_schedule self._use_conv_schedule = use_conv_schedule self.downtime_thread = _FakeThreadInterface() self._thread = concurrent.thread(self.run)
def __init__(self, vm, startTime, conv_schedule, use_conv_schedule): super(MonitorThread, self).__init__() self._stop = threading.Event() self._vm = vm self._startTime = startTime self.daemon = True self.progress = None self._conv_schedule = conv_schedule self._use_conv_schedule = use_conv_schedule self.downtime_thread = _FakeThreadInterface() self._thread = concurrent.thread( self.run, name='migmon/' + self._vm.id[:8])
def test_close(self): reactor = Reactor() thread = concurrent.thread(reactor.process_requests, name="test ractor") thread.start() s1, s2 = socket.socketpair() with closing(s2): disp = reactor.create_dispatcher(s1, impl=TestingImpl()) reactor.stop() thread.join(timeout=1) self.assertTrue(disp.closing) self.assertFalse(reactor._wakeupEvent.closing)
def __init__(self, log): self.log = log self._quit = False self._epoll = select.epoll() self._channels = {} self._unconnected = {} self._update_lock = threading.Lock() self._add_channels = {} self._del_channels = [] self._timeout = None self._thread = concurrent.thread( self.run, name='vmchannels' )
def test_close(self): reactor = Reactor() thread = concurrent.thread(reactor.process_requests, name='test ractor') thread.start() s1, s2 = socket.socketpair() with closing(s2): disp = reactor.create_dispatcher(s1, impl=TestingImpl()) reactor.stop() thread.join(timeout=1) self.assertTrue(disp.closing) self.assertFalse(reactor._wakeupEvent.closing)
def itmap(func, iterable, maxthreads=UNLIMITED_THREADS): """ Make an iterator that computes the function using arguments from the iterable. It works similar to tmap by running each operation in a different thread, this causes the results not to return in any particular order so it's good if you don't care about the order of the results. maxthreads stands for maximum threads that we can initiate simultaneosly. If we reached to max threads the function waits for thread to finish before initiate the next one. """ if maxthreads < 1 and maxthreads != UNLIMITED_THREADS: raise ValueError("Wrong input to function itmap: %s", maxthreads) respQueue = queue.Queue() def wrapper(value): try: respQueue.put(func(value)) except Exception as e: respQueue.put(e) threadsCreated = 0 threadsCount = 0 for arg in iterable: if maxthreads != UNLIMITED_THREADS: if maxthreads == 0: # This not supposed to happened. If it does, it's a bug. # maxthreads should get to 0 only after threadsCount is # greater than 1 if threadsCount < 1: raise RuntimeError("No thread initiated") else: yield respQueue.get() # if yield returns one thread stopped, so we can run # another thread in queue maxthreads += 1 threadsCount -= 1 name = "itmap/%d" % threadsCreated t = concurrent.thread(wrapper, args=(arg, ), name=name) t.start() threadsCreated += 1 threadsCount += 1 maxthreads -= 1 # waiting for rest threads to end for i in range(threadsCount): yield respQueue.get()
def itmap(func, iterable, maxthreads=UNLIMITED_THREADS): """ Make an iterator that computes the function using arguments from the iterable. It works similar to tmap by running each operation in a different thread, this causes the results not to return in any particular order so it's good if you don't care about the order of the results. maxthreads stands for maximum threads that we can initiate simultaneosly. If we reached to max threads the function waits for thread to finish before initiate the next one. """ if maxthreads < 1 and maxthreads != UNLIMITED_THREADS: raise ValueError("Wrong input to function itmap: %s", maxthreads) respQueue = queue.Queue() def wrapper(value): try: respQueue.put(func(value)) except Exception as e: respQueue.put(e) threadsCreated = 0 threadsCount = 0 for arg in iterable: if maxthreads != UNLIMITED_THREADS: if maxthreads == 0: # This not supposed to happened. If it does, it's a bug. # maxthreads should get to 0 only after threadsCount is # greater than 1 if threadsCount < 1: raise RuntimeError("No thread initiated") else: yield respQueue.get() # if yield returns one thread stopped, so we can run # another thread in queue maxthreads += 1 threadsCount -= 1 name = "itmap/%d" % threadsCreated t = concurrent.thread(wrapper, args=(arg,), name=name) t.start() threadsCreated += 1 threadsCount += 1 maxthreads -= 1 # waiting for rest threads to end for i in range(threadsCount): yield respQueue.get()
def test_pass_args_and_kwargs(self): self.args = () self.kwargs = {} def run(*args, **kwargs): self.args = args self.kwargs = kwargs args = (1, 2) kwargs = {'a': 3, 'b': 4} t = concurrent.thread(run, args=args, kwargs=kwargs) t.start() t.join() self.assertEqual(args, self.args) self.assertEqual(kwargs, self.kwargs)
def __init__(self, domainMonitor, sdUUID, hostId, interval): self.thread = concurrent.thread(self._run, logger=self.log.name) self.domainMonitor = domainMonitor self.stopEvent = threading.Event() self.domain = None self.sdUUID = sdUUID self.hostId = hostId self.interval = interval self.nextStatus = Status(actual=False) self.status = FrozenStatus(self.nextStatus) self.isIsoDomain = None self.isoPrefix = None self.lastRefresh = time.time() self.refreshTime = \ config.getint("irs", "repo_stats_cache_refresh_timeout")
def __init__(self, groups=frozenset(), timeout=None, silent_timeout=False): self._time_start = None self._timeout = timeout self._silent_timeout = silent_timeout if groups: unknown_groups = frozenset(groups).difference(frozenset(_GROUPS)) if unknown_groups: raise AttributeError('Invalid groups: %s' % (unknown_groups,)) self._groups = groups else: self._groups = _GROUPS.keys() self._queue = queue.Queue() self._scan_thread = concurrent.thread(self._scan) self._scanning_started = threading.Event() self._scanning_stopped = threading.Event()
def __init__(self, groups=frozenset(), timeout=None, silent_timeout=False): self._time_start = None self._timeout = timeout self._silent_timeout = silent_timeout if groups: unknown_groups = frozenset(groups).difference(frozenset(_GROUPS)) if unknown_groups: raise AttributeError('Invalid groups: %s' % (unknown_groups, )) self._groups = groups else: self._groups = _GROUPS.keys() self._queue = queue.Queue() self._scan_thread = concurrent.thread(self._scan) self._scanning_started = threading.Event() self._scanning_stopped = threading.Event()
def __init__(self, vm, downtime, steps): self._vm = vm self._downtime = downtime self._steps = steps self._stop = threading.Event() delay_per_gib = config.getint('vars', 'migration_downtime_delay') memSize = int(vm.conf['memSize']) self._wait = min(delay_per_gib * memSize / (_MiB_IN_GiB * self._steps), self._WAIT_STEP_LIMIT) # do not materialize, keep as generator expression self._downtimes = exponential_downtime(self._downtime, self._steps) # we need the first value to support set_initial_downtime self._initial_downtime = next(self._downtimes) self._thread = concurrent.thread(self.run)
def _ifup(iface, cgroup=dhclient.DHCLIENT_CGROUP): if not iface.blockingdhcp and (iface.ipv4.bootproto == "dhcp" or iface.ipv6.dhcpv6): # wait for dhcp in another thread, so vdsm won't get stuck (BZ#498940) t = concurrent.thread(_exec_ifup, name="ifup/%s" % iface, args=(iface, cgroup)) t.start() else: if not iface.master and (iface.ipv4 or iface.ipv6): if iface.ipv4: wait_for_ip = waitfor.waitfor_ipv4_addr elif iface.ipv6: wait_for_ip = waitfor.waitfor_ipv6_addr with wait_for_ip(iface.name): _exec_ifup(iface, cgroup) else: with waitfor.waitfor_linkup(iface.name): _exec_ifup(iface, cgroup)
def test_call_soon_threadsafe(self, calls): self.count = 0 def callback(): self.count += 1 start = time.time() t = concurrent.thread(self.loop.run_forever) t.start() try: for i in range(calls): self.loop.call_soon_threadsafe(callback) finally: self.loop.call_soon_threadsafe(self.loop.stop) t.join() elapsed = time.time() - start print("%7d calls: %f seconds" % (calls, elapsed)) self.assertEqual(calls, self.count)
def _start_profiling(): global cherrypy global dowser global _thread logging.debug("Starting memory profiling") import cherrypy import dowser # this nonsense makes pyflakes happy cherrypy dowser with _lock: if is_running(): raise UsageError('Memory profiler is already running') _thread = concurrent.thread(_memory_viewer, name='memprofile') _thread.start()
def __init__(self, vm, downtime, steps): self._vm = vm self._downtime = downtime self._steps = steps self._stop = threading.Event() delay_per_gib = config.getint('vars', 'migration_downtime_delay') memSize = vm.mem_size_mb() self._wait = min( delay_per_gib * memSize / (_MiB_IN_GiB * self._steps), self._WAIT_STEP_LIMIT) # do not materialize, keep as generator expression self._downtimes = exponential_downtime(self._downtime, self._steps) # we need the first value to support set_initial_downtime self._initial_downtime = next(self._downtimes) self._thread = concurrent.thread( self.run, name='migdwn/' + self._vm.id[:8])
def test_abort_running(self): op = operation.Command(["sleep", "5"]) aborted = threading.Event() def run(): try: op.run() except exception.ActionStopped: aborted.set() t = concurrent.thread(run) t.start() try: # TODO: add way to wait until operation is stated? time.sleep(0.5) op.abort() finally: t.join() self.assertTrue(aborted.is_set())
def _ifup(iface, cgroup=dhclient.DHCLIENT_CGROUP): if not iface.blockingdhcp and (iface.ipv4.bootproto == 'dhcp' or iface.ipv6.dhcpv6): # wait for dhcp in another thread, so vdsm won't get stuck (BZ#498940) t = concurrent.thread(_exec_ifup, name='ifup/%s' % iface, args=(iface, cgroup)) t.start() else: if not iface.master and (iface.ipv4 or iface.ipv6): if iface.ipv4: wait_for_ip = waitfor.waitfor_ipv4_addr elif iface.ipv6: wait_for_ip = waitfor.waitfor_ipv6_addr with wait_for_ip(iface.name): _exec_ifup(iface, cgroup) else: with waitfor.waitfor_linkup(iface.name): _exec_ifup(iface, cgroup)
def __init__(self): self._lock = threading.Lock() self._loop = asyncevent.EventLoop() self._thread = concurrent.thread(self._loop.run_forever, name="check/loop") self._checkers = {}
def startMonitoring(self): t = concurrent.thread(self._monitorConnections, logger=self._log.name) self._stopEvent.clear() t.start()
def start(): thread = concurrent.thread(reactor.process_requests, name='Client %s:%s' % (host, port)) thread.start()