def copyPackage(title):
"""
Copy package directory to db path using a file lock to avoid potential
concurrency race conditions.
@param title: string to use in log entry
@type title: C{str}
"""
dbpath = FilePath(TimezoneCache.getDBPath())
pkgpath = TimezoneCache.FilteredFilePath(TimezoneCache._getPackageDBPath())
lockfile = FilesystemLock(dbpath.path + ".lock")
result = lockfile.lock()
try:
if result and not dbpath.exists():
log.info(
"{title} timezones from {pkg} to {to}",
title=title,
pkg=pkgpath.path,
to=dbpath.path
)
# Copy over the entire package
pkgpath.copyFilteredDirectoryTo(dbpath)
finally:
if result:
lockfile.unlock()
def copyPackage(title):
"""
Copy package directory to db path using a file lock to avoid potential
concurrency race conditions.
@param title: string to use in log entry
@type title: C{str}
"""
dbpath = FilePath(TimezoneCache.getDBPath())
pkgpath = TimezoneCache.FilteredFilePath(TimezoneCache._getPackageDBPath())
lockfile = FilesystemLock(dbpath.path + ".lock")
result = lockfile.lock()
try:
if result and not dbpath.exists():
log.info(
"{title} timezones from {pkg} to {to}",
title=title,
pkg=pkgpath.path,
to=dbpath.path
)
# Copy over the entire package
pkgpath.copyFilteredDirectoryTo(dbpath)
finally:
if result:
lockfile.unlock()
def is_locked(self):
"""Is this lock already taken?
Use this for informational purposes only.
The way this works is as follows:
1. Create a new `FilesystemLock` with the same path.
2. Use the global process lock to ensure no other processes are also
trying to access the lock.
3. Attempt to lock the file-system lock:
3a. Upon success, we know that the lock must have been unlocked;
return ``True``.
3b. Upon failure, no action is required because the lock failed. We
know that the lock must have been locked; return ``False``.
"""
fslock = FilesystemLock(self._fslock.name)
with self.PROCESS_LOCK:
if fslock.lock():
fslock.unlock()
return False
else:
return True
class ScriptLock():
config = None
flock = None
def __init__(self, config):
'''
Parse machine-local configuration file.
'''
self.config = config
def __enter__(self):
'''
Be a context manager.
'''
fname = self.config.get("Execution", "pidfile")
self.flock = FilesystemLock(fname)
logger.debug("Obtaining script lock")
self.flock.lock()
def __exit__(self, exc_type, exc_value, traceback):
'''
Be a context manager.
'''
logger.debug("Releasing script lock")
self.flock.unlock()
def do_restore():
lock = FilesystemLock("/var/run/jolicloud_restore_utility.lock")
if lock.lock():
if os.environ.get('DISPLAY', False):
JolicloudRestoreUtilityGtk()
else:
JolicloudRestoreUtilityText()
reactor.run()
lock.unlock()
if __name__ == '__main__':
parser = OptionParser(__doc__)
parser.add_option('-l',
'--lockfile',
dest='lockfile',
default=path.join(os.getcwd(), ".release-runner.lock"))
parser.add_option('-c',
'--config',
dest='config',
help='Configuration file')
options = parser.parse_args()[0]
if not options.config:
parser.error('Need to pass a config')
lockfile = options.lockfile
log.debug("Using lock file %s", lockfile)
lock = FilesystemLock(lockfile)
if not lock.lock():
raise Exception("Cannot acquire lock: %s" % lockfile)
log.debug("Lock acquired: %s", lockfile)
if not lock.clean:
log.warning("Previous run did not properly exit")
try:
main(options)
finally:
log.debug("Releasing lock: %s", lockfile)
lock.unlock()
log.error('Sendchange failed for %s: ' % release, exc_info=True)
if rc != 0:
sys.exit(rc)
if __name__ == '__main__':
parser = OptionParser(__doc__)
parser.add_option('-l', '--lockfile', dest='lockfile',
default=path.join(os.getcwd(), ".release-runner.lock"))
parser.add_option('-c', '--config', dest='config',
help='Configuration file')
options = parser.parse_args()[0]
if not options.config:
parser.error('Need to pass a config')
lockfile = options.lockfile
log.debug("Using lock file %s", lockfile)
lock = FilesystemLock(lockfile)
if not lock.lock():
raise Exception("Cannot acquire lock: %s" % lockfile)
log.debug("Lock acquired: %s", lockfile)
if not lock.clean:
log.warning("Previous run did not properly exit")
try:
main(options)
finally:
log.debug("Releasing lock: %s", lockfile)
lock.unlock()
class SystemLock:
"""A file-system lock.
It is also not reentrant, deliberately so, for good reason: if you use
this to guard against concurrent writing to a file, say, then opening it
twice in any circumstance is bad news.
This behaviour comes about by:
* Taking a process-global lock before performing any file-system
operations.
* Using :class:`twisted.python.lockfile.FilesystemLock` under the hood.
This does not permit double-locking of a file by the name process (or by
any other process, naturally).
There are options too:
* `SystemLock` uses the given path as its lock file. This is the most
general lock.
* `FileLock` adds a suffix of ".lock" to the given path and uses that as
its lock file. Use this when updating a file in a writable directory,
for example.
* `RunLock` puts its lock file in ``/run/lock`` with a distinctive name
based on the given path. Use this when updating a file in a non-writable
directory, for example.
* `NamedLock` also puts its lock file in ``/run/lock`` but with a name
based on the given _name_. `NamedLock`'s lock file are named in such a
way that they will never conflict with a `RunLock`'s. Use this to
synchronise between processes on a single host, for example, where
synchronisation does not naturally revolve around access to a specific
file.
"""
class NotAvailable(Exception):
"""Something has prevented acquisition of this lock.
For example, the lock has already been acquired.
"""
# File-system locks typically claim a lock for the sake of a *process*
# rather than a *thread within a process*. We use a process-global lock to
# serialise all file-system lock operations so that only one thread can
# claim a file-system lock at a time.
PROCESS_LOCK = threading.Lock()
def __init__(self, path):
super(SystemLock, self).__init__()
self._fslock = FilesystemLock(path)
def __enter__(self):
self.acquire()
def __exit__(self, *exc_info):
self.release()
def acquire(self):
"""Acquire the lock.
:raise NotAvailable: When the lock has already been acquired.
"""
with self.PROCESS_LOCK:
if not self._fslock.lock():
raise self.NotAvailable(self._fslock.name)
def release(self):
"""Release the lock."""
with self.PROCESS_LOCK:
self._fslock.unlock()
@contextmanager
def wait(self, timeout=86400):
"""Wait for the lock to become available.
:param timeout: The number of seconds to wait. By default it will wait
up to 1 day.
"""
interval = max(0.1, min(1.0, float(timeout) / 10.0))
for _, _, wait in retries(timeout, interval, reactor):
with self.PROCESS_LOCK:
if self._fslock.lock():
break
if wait > 0:
sleep(wait)
else:
raise self.NotAvailable(self._fslock.name)
try:
yield
finally:
with self.PROCESS_LOCK:
self._fslock.unlock()
@property
def path(self):
return self._fslock.name
def is_locked(self):
"""Is this lock already taken?
Use this for informational purposes only.
The way this works is as follows:
1. Create a new `FilesystemLock` with the same path.
2. Use the global process lock to ensure no other processes are also
trying to access the lock.
3. Attempt to lock the file-system lock:
3a. Upon success, we know that the lock must have been unlocked;
return ``True``.
3b. Upon failure, no action is required because the lock failed. We
know that the lock must have been locked; return ``False``.
"""
fslock = FilesystemLock(self._fslock.name)
with self.PROCESS_LOCK:
if fslock.lock():
fslock.unlock()
return False
else:
return True
class LockResource(object):
"""
Handle requests for locking documents.
This class uses Twisted's Filesystem lock to manage a lock in the shared
database.
"""
url_pattern = '/%s/lock/{uuid}' % SHARED_DB_NAME
"""
"""
TIMEOUT = 300 # XXX is 5 minutes reasonable?
"""
The timeout after which the lock expires.
"""
# used for lock doc storage
TIMESTAMP_KEY = '_timestamp'
LOCK_TOKEN_KEY = '_token'
FILESYSTEM_LOCK_TRIES = 5
FILESYSTEM_LOCK_SLEEP_SECONDS = 1
def __init__(self, uuid, state, responder):
"""
Initialize the lock resource. Parameters to this constructor are
automatically passed by u1db.
:param uuid: The user unique id.
:type uuid: str
:param state: The backend database state.
:type state: u1db.remote.ServerState
:param responder: The infrastructure to send responses to client.
:type responder: u1db.remote.HTTPResponder
"""
self._shared_db = state.open_database(SHARED_DB_NAME)
self._lock_doc_id = '%s%s' % (SHARED_DB_LOCK_DOC_ID_PREFIX, uuid)
self._lock = FilesystemLock(
os.path.join(
tempfile.gettempdir(),
hashlib.sha512(self._lock_doc_id).hexdigest()))
self._state = state
self._responder = responder
@http_app.http_method(content=str)
def put(self, content=None):
"""
Handle a PUT request to the lock document.
A lock is a document in the shared db with doc_id equal to
'lock-<uuid>' and the timestamp of its creation as content. This
method obtains a threaded-lock and creates a lock document if it does
not exist or if it has expired.
It returns '201 Created' and a pair containing a token to unlock and
the lock timeout, or '403 AlreadyLockedError' and the remaining amount
of seconds the lock will still be valid.
:param content: The content of the PUT request. It is only here
because PUT requests with empty content are considered
invalid requests by u1db.
:type content: str
"""
# obtain filesystem lock
if not self._try_obtain_filesystem_lock():
self._responder.send_response_json(
LockTimedOutError.status, # error: request timeout
error=LockTimedOutError.wire_description)
return
created_lock = False
now = time.time()
token = hashlib.sha256(os.urandom(10)).hexdigest() # for releasing
lock_doc = self._shared_db.get_doc(self._lock_doc_id)
remaining = self._remaining(lock_doc, now)
# if there's no lock, create one
if lock_doc is None:
lock_doc = self._shared_db.create_doc(
{
self.TIMESTAMP_KEY: now,
self.LOCK_TOKEN_KEY: token,
},
doc_id=self._lock_doc_id)
created_lock = True
else:
if remaining == 0:
# lock expired, create new one
lock_doc.content = {
self.TIMESTAMP_KEY: now,
self.LOCK_TOKEN_KEY: token,
}
self._shared_db.put_doc(lock_doc)
created_lock = True
self._try_release_filesystem_lock()
# send response to client
if created_lock is True:
self._responder.send_response_json(
201, timeout=self.TIMEOUT, token=token) # success: created
else:
self._responder.send_response_json(
AlreadyLockedError.status, # error: forbidden
error=AlreadyLockedError.wire_description, remaining=remaining)
@http_app.http_method(token=str)
def delete(self, token=None):
"""
Delete the lock if the C{token} is valid.
Delete the lock document in case C{token} is equal to the token stored
in the lock document.
:param token: The token returned when locking.
:type token: str
:raise NotLockedError: Raised in case the lock is not locked.
:raise InvalidTokenError: Raised in case the token is invalid for
unlocking.
"""
lock_doc = self._shared_db.get_doc(self._lock_doc_id)
if lock_doc is None or self._remaining(lock_doc, time.time()) == 0:
self._responder.send_response_json(
NotLockedError.status, # error: not found
error=NotLockedError.wire_description)
elif token != lock_doc.content[self.LOCK_TOKEN_KEY]:
self._responder.send_response_json(
InvalidTokenError.status, # error: unauthorized
error=InvalidTokenError.wire_description)
else:
self._shared_db.delete_doc(lock_doc)
# respond success: should use 204 but u1db does not support it.
self._responder.send_response_json(200)
def _remaining(self, lock_doc, now):
"""
Return the number of seconds the lock contained in C{lock_doc} is
still valid, when compared to C{now}.
:param lock_doc: The document containing the lock.
:type lock_doc: u1db.Document
:param now: The time to which to compare the lock timestamp.
:type now: float
:return: The amount of seconds the lock is still valid.
:rtype: float
"""
if lock_doc is not None:
lock_timestamp = lock_doc.content[self.TIMESTAMP_KEY]
remaining = lock_timestamp + self.TIMEOUT - now
return remaining if remaining > 0 else 0.0
return 0.0
def _try_obtain_filesystem_lock(self):
"""
Try to obtain the file system lock.
@return: Whether the lock was succesfully obtained.
@rtype: bool
"""
tries = self.FILESYSTEM_LOCK_TRIES
while tries > 0:
try:
return self._lock.lock()
except OSError as e:
tries -= 1
if tries == 0:
raise CouldNotObtainLockError(e.message)
time.sleep(self.FILESYSTEM_LOCK_SLEEP_SECONDS)
return False
def _try_release_filesystem_lock(self):
"""
Release the filesystem lock.
"""
try:
self._lock.unlock()
return True
except OSError as e:
if e.errno == errno.ENOENT:
return True
return False
class LockResource(object):
"""
Handle requests for locking documents.
This class uses Twisted's Filesystem lock to manage a lock in the shared
database.
"""
url_pattern = '/%s/lock/{uuid}' % SHARED_DB_NAME
"""
"""
TIMEOUT = 300 # XXX is 5 minutes reasonable?
"""
The timeout after which the lock expires.
"""
# used for lock doc storage
TIMESTAMP_KEY = '_timestamp'
LOCK_TOKEN_KEY = '_token'
FILESYSTEM_LOCK_TRIES = 5
FILESYSTEM_LOCK_SLEEP_SECONDS = 1
def __init__(self, uuid, state, responder):
"""
Initialize the lock resource. Parameters to this constructor are
automatically passed by u1db.
:param uuid: The user unique id.
:type uuid: str
:param state: The backend database state.
:type state: u1db.remote.ServerState
:param responder: The infrastructure to send responses to client.
:type responder: u1db.remote.HTTPResponder
"""
self._shared_db = state.open_database(SHARED_DB_NAME)
self._lock_doc_id = '%s%s' % (SHARED_DB_LOCK_DOC_ID_PREFIX, uuid)
self._lock = FilesystemLock(
os.path.join(tempfile.gettempdir(),
hashlib.sha512(self._lock_doc_id).hexdigest()))
self._state = state
self._responder = responder
@http_app.http_method(content=str)
def put(self, content=None):
"""
Handle a PUT request to the lock document.
A lock is a document in the shared db with doc_id equal to
'lock-<uuid>' and the timestamp of its creation as content. This
method obtains a threaded-lock and creates a lock document if it does
not exist or if it has expired.
It returns '201 Created' and a pair containing a token to unlock and
the lock timeout, or '403 AlreadyLockedError' and the remaining amount
of seconds the lock will still be valid.
:param content: The content of the PUT request. It is only here
because PUT requests with empty content are considered
invalid requests by u1db.
:type content: str
"""
# obtain filesystem lock
if not self._try_obtain_filesystem_lock():
self._responder.send_response_json(
LockTimedOutError.status, # error: request timeout
error=LockTimedOutError.wire_description)
return
created_lock = False
now = time.time()
token = hashlib.sha256(os.urandom(10)).hexdigest() # for releasing
lock_doc = self._shared_db.get_doc(self._lock_doc_id)
remaining = self._remaining(lock_doc, now)
# if there's no lock, create one
if lock_doc is None:
lock_doc = self._shared_db.create_doc(
{
self.TIMESTAMP_KEY: now,
self.LOCK_TOKEN_KEY: token,
},
doc_id=self._lock_doc_id)
created_lock = True
else:
if remaining == 0:
# lock expired, create new one
lock_doc.content = {
self.TIMESTAMP_KEY: now,
self.LOCK_TOKEN_KEY: token,
}
self._shared_db.put_doc(lock_doc)
created_lock = True
self._try_release_filesystem_lock()
# send response to client
if created_lock is True:
self._responder.send_response_json(201,
timeout=self.TIMEOUT,
token=token) # success: created
else:
self._responder.send_response_json(
AlreadyLockedError.status, # error: forbidden
error=AlreadyLockedError.wire_description,
remaining=remaining)
@http_app.http_method(token=str)
def delete(self, token=None):
"""
Delete the lock if the C{token} is valid.
Delete the lock document in case C{token} is equal to the token stored
in the lock document.
:param token: The token returned when locking.
:type token: str
:raise NotLockedError: Raised in case the lock is not locked.
:raise InvalidTokenError: Raised in case the token is invalid for
unlocking.
"""
lock_doc = self._shared_db.get_doc(self._lock_doc_id)
if lock_doc is None or self._remaining(lock_doc, time.time()) == 0:
self._responder.send_response_json(
NotLockedError.status, # error: not found
error=NotLockedError.wire_description)
elif token != lock_doc.content[self.LOCK_TOKEN_KEY]:
self._responder.send_response_json(
InvalidTokenError.status, # error: unauthorized
error=InvalidTokenError.wire_description)
else:
self._shared_db.delete_doc(lock_doc)
# respond success: should use 204 but u1db does not support it.
self._responder.send_response_json(200)
def _remaining(self, lock_doc, now):
"""
Return the number of seconds the lock contained in C{lock_doc} is
still valid, when compared to C{now}.
:param lock_doc: The document containing the lock.
:type lock_doc: u1db.Document
:param now: The time to which to compare the lock timestamp.
:type now: float
:return: The amount of seconds the lock is still valid.
:rtype: float
"""
if lock_doc is not None:
lock_timestamp = lock_doc.content[self.TIMESTAMP_KEY]
remaining = lock_timestamp + self.TIMEOUT - now
return remaining if remaining > 0 else 0.0
return 0.0
def _try_obtain_filesystem_lock(self):
"""
Try to obtain the file system lock.
@return: Whether the lock was succesfully obtained.
@rtype: bool
"""
tries = self.FILESYSTEM_LOCK_TRIES
while tries > 0:
try:
return self._lock.lock()
except OSError as e:
tries -= 1
if tries == 0:
raise CouldNotObtainLockError(e.message)
time.sleep(self.FILESYSTEM_LOCK_SLEEP_SECONDS)
return False
def _try_release_filesystem_lock(self):
"""
Release the filesystem lock.
"""
try:
self._lock.unlock()
return True
except OSError as e:
if e.errno == errno.ENOENT:
return True
return False
