def acquire(self): basedir = os.path.dirname(self.fname) if not os.path.exists(basedir): fileutils.ensure_tree(basedir) LOG.info(_LI('Created lock path: %s'), basedir) self.lockfile = open(self.fname, 'w') while True: try: # Using non-blocking locks since green threads are not # patched to deal with blocking locking calls. # Also upon reading the MSDN docs for locking(), it seems # to have a laughable 10 attempts "blocking" mechanism. self.trylock() LOG.debug('Got file lock "%s"', self.fname) return True except IOError as e: if e.errno in (errno.EACCES, errno.EAGAIN): # external locks synchronise things like iptables # updates - give it some time to prevent busy spinning time.sleep(0.01) else: raise threading.ThreadError( _("Unable to acquire lock on" " `%(filename)s` due to" " %(exception)s") % { 'filename': self.fname, 'exception': e })
def acquire(self): basedir = os.path.dirname(self.fname) if not os.path.exists(basedir): fileutils.ensure_tree(basedir) LOG.info(_LI('Created lock path: %s'), basedir) self.lockfile = open(self.fname, 'w') while True: try: # Using non-blocking locks since green threads are not # patched to deal with blocking locking calls. # Also upon reading the MSDN docs for locking(), it seems # to have a laughable 10 attempts "blocking" mechanism. self.trylock() LOG.debug('Got file lock "%s"', self.fname) return True except IOError as e: if e.errno in (errno.EACCES, errno.EAGAIN): # external locks synchronise things like iptables # updates - give it some time to prevent busy spinning time.sleep(0.01) else: raise threading.ThreadError(_("Unable to acquire lock on" " `%(filename)s` due to" " %(exception)s") % { 'filename': self.fname, 'exception': e, })
def inner(*args, **kwargs): # NOTE(soren): If we ever go natively threaded, this will be racy. # See http://stackoverflow.com/questions/5390569/dyn # amically-allocating-and-destroying-mutexes sem = _semaphores.get(name, semaphore.Semaphore()) if name not in _semaphores: # this check is not racy - we're already holding ref locally # so GC won't remove the item and there was no IO switch # (only valid in greenthreads) _semaphores[name] = sem with sem: LOG.debug(_('Got semaphore "%(lock)s" for method ' '"%(method)s"...'), {'lock': name, 'method': f.__name__}) if external and not CONF.disable_process_locking: LOG.debug(_('Attempting to grab file lock "%(lock)s" for ' 'method "%(method)s"...'), {'lock': name, 'method': f.__name__}) cleanup_dir = False # We need a copy of lock_path because it is non-local local_lock_path = lock_path if not local_lock_path: local_lock_path = CONF.lock_path if not local_lock_path: cleanup_dir = True local_lock_path = tempfile.mkdtemp() if not os.path.exists(local_lock_path): cleanup_dir = True fileutils.ensure_tree(local_lock_path) # NOTE(mikal): the lock name cannot contain directory # separators safe_name = name.replace(os.sep, '_') lock_file_name = '%s%s' % (lock_file_prefix, safe_name) lock_file_path = os.path.join(local_lock_path, lock_file_name) try: lock = InterProcessLock(lock_file_path) with lock: LOG.debug(_('Got file lock "%(lock)s" at %(path)s ' 'for method "%(method)s"...'), {'lock': name, 'path': lock_file_path, 'method': f.__name__}) retval = f(*args, **kwargs) finally: # NOTE(vish): This removes the tempdir if we needed # to create one. This is used to cleanup # the locks left behind by unit tests. if cleanup_dir: shutil.rmtree(local_lock_path) else: retval = f(*args, **kwargs) return retval
def test_ensure_tree(self): tmpdir = tempfile.mkdtemp() try: testdir = '%s/foo/bar/baz' % (tmpdir,) fileutils.ensure_tree(testdir) self.assertTrue(os.path.isdir(testdir)) finally: if os.path.exists(tmpdir): shutil.rmtree(tmpdir)
def test_ensure_tree(self): tmpdir = tempfile.mkdtemp() try: testdir = "%s/foo/bar/baz" % (tmpdir,) fileutils.ensure_tree(testdir) self.assertTrue(os.path.isdir(testdir)) finally: if os.path.exists(tmpdir): shutil.rmtree(tmpdir)
def test_ensure_tree(self): tmpdir = tempfile.mkdtemp() try: testdir = '%s/foo/bar/baz' % (tmpdir, ) fileutils.ensure_tree(testdir, TEST_PERMISSIONS) self.assertTrue(os.path.isdir(testdir)) self.assertEqual( os.stat(testdir).st_mode, TEST_PERMISSIONS | stat.S_IFDIR) finally: if os.path.exists(tmpdir): shutil.rmtree(tmpdir)
def test_ensure_tree(self): tmpdir = tempfile.mkdtemp() try: testdir = '%s/foo/bar/baz' % (tmpdir,) fileutils.ensure_tree(testdir, TEST_PERMISSIONS) self.assertTrue(os.path.isdir(testdir)) self.assertEqual(os.stat(testdir).st_mode, TEST_PERMISSIONS | stat.S_IFDIR) finally: if os.path.exists(tmpdir): shutil.rmtree(tmpdir)
def external_lock(name, lock_file_prefix=None, lock_path=None): with internal_lock(name): LOG.debug(_('Attempting to grab file lock "%(lock)s"'), {'lock': name}) # We need a copy of lock_path because it is non-local local_lock_path = lock_path or CONF.lock_path if not local_lock_path: raise cfg.RequiredOptError('lock_path') if not os.path.exists(local_lock_path): fileutils.ensure_tree(local_lock_path) LOG.info(_('Created lock path: %s'), local_lock_path) def add_prefix(name, prefix): if not prefix: return name sep = '' if prefix.endswith('-') else '-' return '%s%s%s' % (prefix, sep, name) # NOTE(mikal): the lock name cannot contain directory # separators lock_file_name = add_prefix(name.replace(os.sep, '_'), lock_file_prefix) lock_file_path = os.path.join(local_lock_path, lock_file_name) try: lock = InterProcessLock(lock_file_path) with lock as lock: LOG.debug(_('Got file lock "%(lock)s" at %(path)s'), {'lock': name, 'path': lock_file_path}) yield lock finally: LOG.debug(_('Released file lock "%(lock)s" at %(path)s'), {'lock': name, 'path': lock_file_path})
def lock(name, lock_file_prefix=None, external=False, lock_path=None): """Context based lock This function yields a `threading.Semaphore` instance (if we don't use eventlet.monkey_patch(), else `semaphore.Semaphore`) unless external is True, in which case, it'll yield an InterProcessLock instance. :param lock_file_prefix: The lock_file_prefix argument is used to provide lock files on disk with a meaningful prefix. :param external: The external keyword argument denotes whether this lock should work across multiple processes. This means that if two different workers both run a a method decorated with @synchronized('mylock', external=True), only one of them will execute at a time. :param lock_path: The lock_path keyword argument is used to specify a special location for external lock files to live. If nothing is set, then CONF.lock_path is used as a default. """ # NOTE(soren): If we ever go natively threaded, this will be racy. # See http://stackoverflow.com/questions/5390569/dyn # amically-allocating-and-destroying-mutexes sem = _semaphores.get(name, threading.Semaphore()) if name not in _semaphores: # this check is not racy - we're already holding ref locally # so GC won't remove the item and there was no IO switch # (only valid in greenthreads) _semaphores[name] = sem with sem: LOG.debug(_('Got semaphore "%(lock)s"'), {'lock': name}) # NOTE(mikal): I know this looks odd if not hasattr(local.strong_store, 'locks_held'): local.strong_store.locks_held = [] local.strong_store.locks_held.append(name) try: if external and not CONF.disable_process_locking: LOG.debug(_('Attempting to grab file lock "%(lock)s"'), {'lock': name}) # We need a copy of lock_path because it is non-local local_lock_path = lock_path or CONF.lock_path if not local_lock_path: raise cfg.RequiredOptError('lock_path') if not os.path.exists(local_lock_path): fileutils.ensure_tree(local_lock_path) LOG.info(_('Created lock path: %s'), local_lock_path) def add_prefix(name, prefix): if not prefix: return name sep = '' if prefix.endswith('-') else '-' return '%s%s%s' % (prefix, sep, name) # NOTE(mikal): the lock name cannot contain directory # separators lock_file_name = add_prefix(name.replace(os.sep, '_'), lock_file_prefix) lock_file_path = os.path.join(local_lock_path, lock_file_name) try: lock = InterProcessLock(lock_file_path) with lock as lock: LOG.debug(_('Got file lock "%(lock)s" at %(path)s'), {'lock': name, 'path': lock_file_path}) yield lock finally: LOG.debug(_('Released file lock "%(lock)s" at %(path)s'), {'lock': name, 'path': lock_file_path}) else: yield sem finally: local.strong_store.locks_held.remove(name)
def lock(name, lock_file_prefix=None, external=False, lock_path=None): """Context based lock This function yields a `threading.Semaphore` instance (if we don't use eventlet.monkey_patch(), else `semaphore.Semaphore`) unless external is True, in which case, it'll yield an InterProcessLock instance. :param lock_file_prefix: The lock_file_prefix argument is used to provide lock files on disk with a meaningful prefix. :param external: The external keyword argument denotes whether this lock should work across multiple processes. This means that if two different workers both run a a method decorated with @synchronized('mylock', external=True), only one of them will execute at a time. :param lock_path: The lock_path keyword argument is used to specify a special location for external lock files to live. If nothing is set, then CONF.lock_path is used as a default. """ # NOTE(soren): If we ever go natively threaded, this will be racy. # See http://stackoverflow.com/questions/5390569/dyn # amically-allocating-and-destroying-mutexes sem = _semaphores.get(name, threading.Semaphore()) if name not in _semaphores: # this check is not racy - we're already holding ref locally # so GC won't remove the item and there was no IO switch # (only valid in greenthreads) _semaphores[name] = sem with sem: LOG.debug(_('Got semaphore "%(lock)s"'), {'lock': name}) # NOTE(mikal): I know this looks odd if not hasattr(local.strong_store, 'locks_held'): local.strong_store.locks_held = [] local.strong_store.locks_held.append(name) try: if external and not CONF.disable_process_locking: LOG.debug(_('Attempting to grab file lock "%(lock)s"'), {'lock': name}) # We need a copy of lock_path because it is non-local local_lock_path = lock_path or CONF.lock_path if not local_lock_path: raise cfg.RequiredOptError('lock_path') if not os.path.exists(local_lock_path): fileutils.ensure_tree(local_lock_path) LOG.info(_('Created lock path: %s'), local_lock_path) def add_prefix(name, prefix): if not prefix: return name sep = '' if prefix.endswith('-') else '-' return '%s%s%s' % (prefix, sep, name) # NOTE(mikal): the lock name cannot contain directory # separators lock_file_name = add_prefix(name.replace(os.sep, '_'), lock_file_prefix) lock_file_path = os.path.join(local_lock_path, lock_file_name) try: lock = InterProcessLock(lock_file_path) with lock as lock: LOG.debug(_('Got file lock "%(lock)s" at %(path)s'), { 'lock': name, 'path': lock_file_path }) yield lock finally: LOG.debug(_('Released file lock "%(lock)s" at %(path)s'), { 'lock': name, 'path': lock_file_path }) else: yield sem finally: local.strong_store.locks_held.remove(name)
def inner(*args, **kwargs): # NOTE(soren): If we ever go natively threaded, this will be racy. # See http://stackoverflow.com/questions/5390569/dyn # amically-allocating-and-destroying-mutexes sem = _semaphores.get(name, semaphore.Semaphore()) if name not in _semaphores: # this check is not racy - we're already holding ref locally # so GC won't remove the item and there was no IO switch # (only valid in greenthreads) _semaphores[name] = sem with sem: LOG.debug( _('Got semaphore "%(lock)s" for method ' '"%(method)s"...'), { 'lock': name, 'method': f.__name__ }) if external and not CONF.disable_process_locking: LOG.debug( _('Attempting to grab file lock "%(lock)s" for ' 'method "%(method)s"...'), { 'lock': name, 'method': f.__name__ }) cleanup_dir = False # We need a copy of lock_path because it is non-local local_lock_path = lock_path if not local_lock_path: local_lock_path = CONF.lock_path if not local_lock_path: cleanup_dir = True local_lock_path = tempfile.mkdtemp() if not os.path.exists(local_lock_path): cleanup_dir = True fileutils.ensure_tree(local_lock_path) # NOTE(mikal): the lock name cannot contain directory # separators safe_name = name.replace(os.sep, '_') lock_file_name = '%s%s' % (lock_file_prefix, safe_name) lock_file_path = os.path.join(local_lock_path, lock_file_name) try: lock = InterProcessLock(lock_file_path) with lock: LOG.debug( _('Got file lock "%(lock)s" at %(path)s ' 'for method "%(method)s"...'), { 'lock': name, 'path': lock_file_path, 'method': f.__name__ }) retval = f(*args, **kwargs) finally: LOG.debug( _('Released file lock "%(lock)s" at %(path)s' ' for method "%(method)s"...'), { 'lock': name, 'path': lock_file_path, 'method': f.__name__ }) # NOTE(vish): This removes the tempdir if we needed # to create one. This is used to cleanup # the locks left behind by unit tests. if cleanup_dir: shutil.rmtree(local_lock_path) else: retval = f(*args, **kwargs) return retval