def release(self): """Releases any past acquired locks (partial or otherwise).""" height = len(self._lock_stacks) if not height: # Raise the same error type as the threading.Lock raises so that # it matches the behavior of the built-in class (it's odd though # that the threading.RLock raises a runtime error on this same # method instead...) raise threading.ThreadError('Release attempted on unlocked lock') # Cleans off one level of the stack (this is done so that if there # are multiple __enter__() and __exit__() pairs active that this will # only remove one level (the last one), and not all levels... leftover = self._lock_stacks[-1] while leftover: lock = self._locks[leftover - 1] try: lock.release() except (threading.ThreadError, RuntimeError) as e: # Ensure that we adjust the lock stack under failure so that # if release is attempted again that we do not try to release # the locks we already released... self._lock_stacks[-1] = leftover raise threading.ThreadError( "Unable to release lock %s/%s due to '%s'" % (leftover, len(self._locks), e)) else: leftover -= 1 # At the end only clear it off, so that under partial failure we don't # lose any locks... self._lock_stacks.pop()
def acquire_lock(self, blocking=True): if self.lock.acquire(blocking): return True elif blocking: raise threading.ThreadError('Cannot acquire lock.') else: return False
def add_threaded_module(self, module: Module): if module.threaded: self.threaded_modules.append(module) else: msg = f"Module {module} is not registered as threaded, but is attempting to run threaded" self.logger.error(msg) raise threading.ThreadError(msg)
def start_multi_threading(thread_list, max_threads=20, all_threads_have_to_be_success=True): if len(thread_list) == 0: return None for count_i, thread in enumerate(thread_list): thread.daemon = True thread.start() logging.info( """Thread {thread_id} started""".format(thread_id=thread.name)) while len([ thread_i for thread_i in thread_list if thread_i.is_alive() ]) >= max_threads: time.sleep(0.1) logging.debug('You have run too many threads! Have a rest!!') for thread in thread_list: thread.join() if all_threads_have_to_be_success and \ 'exitcode' in dir(thread) and \ thread.exitcode == 1: raise threading.ThreadError(thread.name + ' failed') else: logging.info(thread.name + " Is Done")
def acquire(self): basedir = os.path.dirname(self.fname) if not os.path.exists(basedir): fileutils.ensure_tree(basedir) LOG.info(_LI('Created lock path: %s'), basedir) # Open in append mode so we don't overwrite any potential contents of # the target file. This eliminates the possibility of an attacker # creating a symlink to an important file in our lock_path. self.lockfile = open(self.fname, 'a') while True: try: # Using non-blocking locks since green threads are not # patched to deal with blocking locking calls. # Also upon reading the MSDN docs for locking(), it seems # to have a laughable 10 attempts "blocking" mechanism. self.trylock() LOG.debug('Got file lock "%s"', self.fname) return True except IOError as e: if e.errno in (errno.EACCES, errno.EAGAIN): # external locks synchronise things like iptables # updates - give it some time to prevent busy spinning time.sleep(0.01) else: raise threading.ThreadError( _("Unable to acquire lock on" " `%(filename)s` due to" " %(exception)s") % { 'filename': self.fname, 'exception': e, })
def thread_raise(thread, exctype): ''' Raises the exception exctype in the thread. Adapted from http://tomerfiliba.com/recipes/Thread2/ which explains: "The exception will be raised only when executing python bytecode. If your thread calls a native/built-in blocking function, the exception will be raised only when execution returns to the python code." ''' import ctypes, inspect, threading if not thread.is_alive(): raise threading.ThreadError('thread %s is not running' % thread) if not inspect.isclass(exctype): raise TypeError( 'cannot raise %s, only exception types can be raised (not ' 'instances)' % exc_type) res = ctypes.pythonapi.PyThreadState_SetAsyncExc( ctypes.c_long(thread.ident), ctypes.py_object(exctype)) if res == 0: raise ValueError('invalid thread id? thread.ident=%s' % thread.ident) elif res != 1: # if it returns a number greater than one, you're in trouble, # and you should call it again with exc=NULL to revert the effect ctypes.pythonapi.PyThreadState_SetAsyncExc(thread.ident, 0) raise SystemError('PyThreadState_SetAsyncExc failed')
def progress(self, file, dest_folder): size = os.path.getsize(file) new_file = os.path.join(dest_folder, os.path.basename(file)) counter = 0 got_store = 0 while True: if not os.path.exists(new_file): counter += 1 time.sleep(0.1) if counter > 100: logging.error("Can't create file %s" % new_file) return continue got = os.path.getsize(new_file) definite = got - got_store got_store = got fraction = (got + 0.0) / size self.pr_bar.set_fraction(fraction) self.pr_bar.set_text("%.0f%%" % (100 * fraction)) fraction = (definite + 0.0 ) / self.total_size + self.total_pr_bar.get_fraction() self.total_pr_bar.set_fraction(fraction) self.total_pr_bar.set_text("%.0f%%" % (100 * fraction)) time.sleep(0.1) if self.exit: raise threading.ThreadError("the thread is stopped") if got == size: break
def acquire(self): basedir = os.path.dirname(self.fname) if not os.path.exists(basedir): fileutils.ensure_tree(basedir) LOG.info(_LI('Created lock path: %s'), basedir) self.lockfile = open(self.fname, 'w') while True: try: # Using non-blocking locks since green threads are not # patched to deal with blocking locking calls. # Also upon reading the MSDN docs for locking(), it seems # to have a laughable 10 attempts "blocking" mechanism. self.trylock() LOG.debug('Got file lock "%s"', self.fname) return True except IOError as e: if e.errno in (errno.EACCES, errno.EAGAIN): # external locks synchronise things like iptables # updates - give it some time to prevent busy spinning time.sleep(0.01) else: raise threading.ThreadError( _("Unable to acquire lock on" " `%(filename)s` due to" " %(exception)s") % { 'filename': self.fname, 'exception': e, })
def test__release_resource_lock_unlocked(self): resource_id = '0ae77286-c0d6-11e5-9181-525400137dfc' fake_lock = mock.Mock(release=mock.Mock()) fake_lock.release.side_effect = threading.ThreadError() self.w._resource_locks = {resource_id: fake_lock} fake_sm = mock.Mock(resource_id=resource_id) # just ensure we dont raise self.w._release_resource_lock(fake_sm)
def test_fatal_error(self): for e in (MemoryError(), threading.ThreadError(), OSError(errno.ENOMEM, "")): with mock.patch('os.kill') as kill_mock: act = BogusActor.start(e).tell_proxy() act.doStuff() act.actor_ref.stop(block=True) self.assertTrue(kill_mock.called)
def test_thread_start_error(self): with mock.patch.object(threading.Thread, "start", side_effect=threading.ThreadError("nonewthread")) as m: c = tcp.TCPClient(("127.0.0.1", self.port)) c.connect() assert not c.rfile.read(1) assert m.called assert "nonewthread" in self.q.get_nowait() self.test_echo()
def automated(self): try: self._thread.start() except threading.ThreadError: raise threading.ThreadError() except KeyError: print('Fatal error during thread.') sys.exit(1)
def __enter__(self): gotten = self.acquire() if not gotten: # This shouldn't happen, but just incase... raise threading.ThreadError("Unable to acquire a file lock" " on `%s` (when used as a" " context manager)" % self.path) return self
def recv(self, block=True): '''Receive stream from server.\nDo not call it directly, it should be called by parse() or recvline().''' if self.recvlock.acquire(): try: if not self.sock: e = socket.error( '[errno %d] Socket operation on non-socket' % errno.ENOTSOCK) e.errno = errno.ENOTSOCK raise e try: received = b'' if block: received = self.sock.recv(self.buffer_length) else: oldtimeout = self.sock.gettimeout() self.sock.settimeout(0) try: if isinstance(self.sock, ssl.SSLSocket): received = self.sock.recv(self.buffer_length) else: received = self.sock.recv( self.buffer_length, socket.MSG_DONTWAIT) except ssl.SSLWantReadError: select.select([self.sock], [], []) received = self.sock.recv(self.buffer_length) except ssl.SSLWantWriteError: select.select([], [self.sock], []) received = self.sock.recv(self.buffer_length) finally: self.sock.settimeout(oldtimeout) del oldtimeout if received: self.recvbuf += received else: self.quit('Connection reset by peer.', wait=False) return True except socket.timeout as e: try: self.quit('Operation timed out.', wait=False) finally: self.sock = None raise except socket.error as e: if e.errno in (errno.EAGAIN, errno.EWOULDBLOCK): return False else: try: self.quit('Network error.', wait=False) finally: self.sock = None raise finally: self.recvlock.release() elif block: raise threading.ThreadError('Cannot acquire lock.') else: return False
def _get_my_tid(self): if not self.isAlive(): raise threading.ThreadError("the thread is not active") if hasattr(self, "_thread_id"): return self._thread_id for tid, tobj in threading._active.items(): if tobj is self: self._thread_id = tid return tid
def test_fatal_error(self): for e in (MemoryError(), threading.ThreadError(), OSError(errno.ENOMEM, "")): kill_mock = mock.Mock('os.kill') bgact = BogusActor.start(e, killfunc=kill_mock) act_thread = bgact.proxy().get_thread().get() act = bgact.tell_proxy() act.doStuff() act.actor_ref.stop(block=True) act_thread.join() self.assertTrue(kill_mock.called)
def release(self): self.lock.acquire() if self.maintainer is None: self.lock.release() raise threading.ThreadError() self.running = False self.maintainer.join() if self.lock_lost_callback is not None: self.lock_lost_callback() self.lock.release()
def killServices(self, *args, **kwargs): ''' attempt a nice shutdown of the thread request pool. ''' for threadx in self.monitors: threadx.commitSuicide() for threadx in self.monitors: threadx.join(self.kill_timeout) self.forceThreadCheck() if len(self.monitors): raise threading.ThreadError('Our children are misbehaving!')
def _get_my_tid(self): """determines this (self's) thread id CAREFUL : this function is executed in the context of the caller thread, to get the identity of the thread represented by this instance. """ if not self.isAlive(): raise threading.ThreadError("the thread is not active") return self.ident
def _get_my_tid(self): if not self.isAlive(): raise threading.ThreadError( "Attempted to fetch tid for an inactive thread") if hasattr(self, "_thread_id"): return self._thread_id for tid, tobj in threading._active.items(): if tobj is self: self._thread_id = tid return tid
def join_with_timeout(self, timeout): self.all_tasks_done.acquire() try: endtime = time.time() + timeout while self.unfinished_tasks: remaining = endtime - time.time() if remaining <= 0.0: raise threading.ThreadError('NotFinished') time.sleep(.05) self.all_tasks_done.wait(remaining) finally: self.all_tasks_done.release()
def retry_on_exception(e): # TODO(harlowja): once/if https://github.com/rholder/retrying/pull/20 # gets merged we should just switch to using that to avoid having to # catch and inspect all execeptions (and there types...) if isinstance(e, IOError) and e.errno in (errno.EACCES, errno.EAGAIN): return True raise threading.ThreadError(("Unable to acquire lock on" " `%(filename)s` due to" " %(exception)s") % { 'filename': filename, 'exception': e, })
def _get_my_tid(self): """determines this (self's) thread id""" if not self.isAlive(): raise threading.ThreadError('the thread is not active') if hasattr(self, '_thread_id'): return self._thread_id for tid, tobj in threading._active.items(): if tobj is self: self._thread_id = tid return tid raise AssertionError("could not determine the thread's id")
def __get_my_tid(self): """ determines this (self's) thread id """ if not self.isAlive(): raise threading.ThreadError("the thread is not active") # do we have it cached? if hasattr(self, "_thread_id"): return self._thread_id raise AssertionError("could not determine the thread's id")
def _async_raise(self, exctype): if not self.isAlive(): raise threading.ThreadError("the thread is not active") tid = ctypes.c_long(self.ident) res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype)) if res == 0: raise ValueError("invalid thread id") elif res != 1: # """if it returns a number greater than one, you're in trouble, # and you should call it again with exc=NULL to revert the effect""" ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, 0) raise SystemError("PyThreadState_SetAsyncExc failed")
def _upload_part(retries_left=num_retries): try: with contextlib.closing(StringIO(part_data)) as f: f.seek(0) cb = lambda c, t: progress_cb(part_no, c, t) if progress_cb else None upload_func(f, part_no, cb=cb, num_cb=100) except Exception as exc: retries_left -= 1 if retries_left > 0: return _upload_part(retries_left=retries_left) else: return threading.ThreadError(repr(threading.current_thread()) + ' ' + repr(exc))
def get_id(self) -> int: '''Return the thread id.''' if not self.isAlive(): raise threading.ThreadError("The thread is not active.") if hasattr(self, '_thread_id'): return self._thread_id for ID, thread in threading._active.items(): if thread is self: return ID raise AssertionError("Could not determine the thread's id.")
def _get_my_tid(self): """determines this (self's) thread id""" if not self.isAlive(): raise threading.ThreadError("the thread is not active") # do we have it cached? if hasattr(self, "_thread_id"): return self._thread_id # no, look for it in the _active dict for tid, tobj in threading._active.items(): if tobj is self: self._thread_id = tid return tid raise AssertionError("could not determine the thread's id")
def wrapper(self, *args, **kwargs): attr_value = getattr(self, attr_name) if isinstance(attr_value, (tuple, list)): with _utils.LockStack(logger=logger) as stack: for i, lock in enumerate(attr_value): if not stack.acquire_lock(lock): raise threading.ThreadError("Unable to acquire" " lock %s" % (i + 1)) return f(self, *args, **kwargs) else: lock = attr_value with lock: return f(self, *args, **kwargs)
def _get_own_tid(self): if not self.is_alive(): raise threading.ThreadError("Thread is not active") if hasattr(self, "_thread_id"): return self._thread_id for tid, tobj in threading._active.items(): if tobj is self: self._thread_id = tid return tid raise AssertionError("Thread id not found!")