def global_lock_setup(self):
if self.afs:
self.lockfn = os.path.join(self.sdir, "global_lock")
lock_path = str(self.lockfn) + '.afs'
lock_file = os.path.join(lock_path, "lock_file")
try:
if not os.path.exists(lock_path):
os.makedirs(lock_path)
if not os.path.isfile(lock_file):
with open(lock_file, "w"):
pass
except Exception as err:
logger.debug("Global Lock Setup Error: %s" % err)
else:
try:
self.lockfn = os.path.join(self.sdir, "global_lock")
if not os.path.isfile(self.lockfn):
with open(self.lockfn, "w"):
# create file (does not interfere with existing sessions)
pass
except IOError as x:
raise RepositoryError(
self.repo,
"Could not create lock file '%s': %s" % (self.lockfn, x))
except OSError as x:
raise RepositoryError(
self.repo,
"Could not open lock file '%s': %s" % (self.lockfn, x))
def test_all_exceptions(gpi):
"""Create all exceptions and make sure they behave correctly"""
import Ganga.Core.exceptions
test_str = "My Test Error"
def exception_test(err_name):
"""Run tests on the given exception"""
err_type = getattr(Ganga.Core.exceptions, err_name)
err_obj = err_type(test_str)
assert test_str in str(err_obj)
err_list = [
"GangaException", "GangaFileError", "PluginError",
"ApplicationConfigurationError", "ApplicationPrepareError",
"IncompleteJobSubmissionError", "IncompleteKillError",
"JobManagerError", "GangaAttributeError", "GangaValueError",
"GangaIOError", "SplitterError", "ProtectedAttributeError",
"ReadOnlyObjectError", "TypeMismatchError", "SchemaError",
"SchemaVersionError", "CredentialsError", "CredentialRenewalError",
"InvalidCredentialError", "ExpiredCredentialError"
]
for e in err_list:
exception_test(e)
# check the BackendError
from Ganga.Core.exceptions import BackendError
err = BackendError("TestBackend", test_str)
assert "TestBackend" in str(err)
assert test_str in str(err)
# check the InaccessibleObjectError
from Ganga.Core.exceptions import InaccessibleObjectError, JobManagerError
from Ganga.Core.GangaRepository import getRegistry
err = InaccessibleObjectError(
getRegistry('jobs').repository, 0,
JobManagerError("My JobManagerError"))
assert "jobs" in str(err)
assert "#0" in str(err)
assert "My JobManagerError" in str(err)
# check the RepositoryError
from Ganga.Core.exceptions import RepositoryError
from Ganga.Core.GangaRepository import getRegistry
RepositoryError(getRegistry('jobs').repository, test_str)
# Construct another to check the except clause in the exception is called
RepositoryError(getRegistry('jobs').repository, test_str)
def session_read(self, fn):
""" Reads a session file and returns a set of IDs locked by that session.
The global lock MUST be held for this function to work, although on NFS additional
locking is done
Raises RepositoryError if severe access problems occur (corruption otherwise!) """
try:
# This can fail (thats OK, file deleted in the meantime)
fd = None
try:
fd = os.open(fn, os.O_RDWR)
os.lseek(fd, 0, 0)
if not self.afs: # additional locking for NFS
fcntl.lockf(fd, fcntl.LOCK_SH)
try:
# 00)) # read up to 1 MB (that is more than enough...)
return pickle.loads(os.read(fd, 1048576))
except Exception as x:
logger.warning(
"corrupt or inaccessible session file '%s' - ignoring it (Exception %s %s)."
% (fn, getName(x), x))
finally:
if fd is not None:
if not self.afs: # additional locking for NFS
fcntl.lockf(fd, fcntl.LOCK_UN)
os.close(fd)
except OSError as x:
if x.errno != errno.ENOENT:
raise RepositoryError(
self.repo,
"Error on session file access '%s': %s" % (fn, x))
return set()
def afs_lock_require(self):
try:
lock_path = str(self.lockfn) + '.afs'
lock_file = os.path.join(lock_path, "lock_file")
def clean_path():
oldtime = os.stat(lock_file).st_ctime
nowtime = time.time()
if abs(int(nowtime) - oldtime) > 10:
#logger.debug( "cleaning global lock" )
os.system("fs setacl %s %s rlidwka" %
(quote(lock_path), getpass.getuser()))
while True:
try:
if os.path.isfile(lock_file):
clean_path()
os.unlink(lock_file)
break
except Exception as err:
logger.debug("Global Lock aquire Exception: %s" % err)
time.sleep(0.1)
os.system("fs setacl %s %s rliwka" %
(quote(lock_path), getpass.getuser()))
while not os.path.isfile(lock_file):
with open(lock_file, "w"):
pass
#logger.debug("global capture")
except IOError as x:
raise RepositoryError(self.repo,
"IOError on AFS global lock: %s" % (x, ))
def numberRepos(self):
try:
assert (len(self.fns) == len(self.repos))
except AssertionError:
raise RepositoryError(
"Number of repos is inconsistent with lock files")
return len(self.fns)
def make_new_ids(self, n):
""" Locks the next n available ids and returns them as a list
Raise RepositoryError on fatal error"""
# Actualize count
try:
newcount = self.cnt_read()
except ValueError:
logger.warning(
"Corrupt job counter (possibly due to crash of another session)! Trying to recover..."
)
newcount = self.count
except OSError:
raise RepositoryError(
self.repo,
"Job counter deleted! External modification to repository!")
if not newcount >= self.count:
#raise RepositoryError(self.repo, "Counter value decreased - logic error!")
logger.warning(
"Internal counter increased - probably the count file was deleted."
)
newcount = self.count
# someone used force_ids (for example old repository imports)
if self.locked and max(self.locked) >= newcount:
newcount = max(self.locked) + 1
ids = range(newcount, newcount + n)
self.locked.update(ids)
self.count = newcount + n
self.cnt_write()
self.session_write()
return list(ids)
def lock_ids(self, ids):
#logger.debug( "locking: %s" % ids)
ids = set(ids)
try:
sessions = [
sn for sn in os.listdir(self.sdir)
if sn.endswith(self.name + ".locks")
]
except OSError as x:
raise RepositoryError(
self.repo,
"Could not list session directory '%s'!" % (self.sdir))
slocked = set()
for session in sessions:
sf = os.path.join(self.sdir, session)
if sf == self.fn:
continue
slocked.update(self.session_read(sf))
#logger.debug( "locked: %s" % slocked)
ids.difference_update(slocked)
self.locked.update(ids)
#logger.debug( "stored_lock: %s" % self.locked)
self.session_write()
#logger.debug( "list: %s" % list(ids))
return list(ids)
def _add(self, obj, force_index=None):
""" Add an object to the registry and assigns an ID to it.
use force_index to set the index (for example for metadata). This overwrites existing objects!
Raises RepositoryError
Args:
_obj (GangaObject): This is th object which is to be added to this Registy/Repo
force_index (int): This is the index which we will give the object, None = auto-assign
"""
logger.debug("_add")
if self.hasStarted() is not True:
raise RepositoryError("Cannot add objects to a disconnected repository!")
if force_index is None:
ids = self.repository.add([obj])
else:
if len(self.repository.lock([force_index])) == 0:
raise RegistryLockError("Could not lock '%s' id #%i for a new object!" % (self.name, force_index))
ids = self.repository.add([obj], [force_index])
obj._setRegistry(self)
obj._registry_locked = True
self.repository.flush(ids)
return ids[0]
def mkdir(self, dn):
"""Make sure the given directory exists"""
try:
os.makedirs(dn)
except OSError as x:
if x.errno != errno.EEXIST:
raise RepositoryError(self.repo,
"OSError on directory create: %s" % x)
def cnt_read(self):
""" Tries to read the counter file.
Raises ValueError (invalid contents)
Raises OSError (no access/does not exist)
Raises RepositoryError (fatal)
"""
try:
if self.last_count_access is not None:
last_count_time = self.last_count_access.time
last_count_val = self.last_count_access.val
last_time = os.stat(self.cntfn).st_ctime
if last_time == last_count_time:
return last_count_val # If the file hasn't changed since last check, return the cached value
_output = None
fd = None
try:
fd = os.open(self.cntfn, os.O_RDWR)
if not self.afs: # additional locking for NFS
fcntl.lockf(fd, fcntl.LOCK_SH)
# 100 bytes should be enough for any ID. Can raise ValueErrorr
_output = int(os.read(fd, 100).split("\n")[0])
finally:
if fd is not None:
if not self.afs: # additional locking for NFS
fcntl.lockf(fd, fcntl.LOCK_UN)
os.close(fd)
if _output != None:
self.last_count_access = SessionLockManager.LastCountAccess(
os.stat(self.cntfn).st_ctime, _output)
return _output
except OSError as x:
if x.errno != errno.ENOENT:
raise RepositoryError(
self.repo,
"OSError on count file '%s' read: %s" % (self.cntfn, x))
else:
# This can be a recoverable error, depending on where it occurs
raise
except IOError as x:
raise RepositoryError(
self.repo,
"Locking error on count file '%s' write: %s" % (self.cntfn, x))
def afs_lock_release(self):
try:
lock_path = str(self.lockfn) + '.afs'
os.system("fs setacl %s %s rlidwka" %
(quote(lock_path), getpass.getuser()))
#logger.debug("global release")
except IOError as x:
raise RepositoryError(self.repo,
"IOError on AFS global lock: %s" % (x, ))
def removeRepo(self, fn, repo):
#logger.debug("Removing fn: %s" % fn )
self.fns.remove(fn)
#logger.debug("Removing fn: %s" % fn )
self.repos.remove(repo)
try:
assert (len(self.fns) == len(self.repos))
except AssertionError:
raise RepositoryError(
"Number of repos is inconsistent after removing repo!")
def cnt_write(self):
""" Writes the counter to the counter file.
The global lock MUST be held for this function to work correctly
Raises OSError if count file is inaccessible """
finished = False
try:
# If this fails, we want to shutdown the repository (corruption
# possible)
fd = None
try:
fd = os.open(self.cntfn, os.O_RDWR)
if not self.afs:
fcntl.lockf(fd, fcntl.LOCK_EX)
os.write(fd, str(self.count) + "\n")
if not self.afs:
fcntl.lockf(fd, fcntl.LOCK_UN)
finally:
if fd is not None:
os.close(fd)
finished = True
except OSError as x:
if x.errno != errno.ENOENT:
raise RepositoryError(
self.repo,
"OSError on count file '%s' write: %s" % (self.cntfn, x))
else:
raise RepositoryError(
self.repo,
"Count file '%s' not found! Repository was modified externally!"
% (self.cntfn))
except IOError as x:
raise RepositoryError(
self.repo,
"Locking error on count file '%s' write: %s" % (self.cntfn, x))
finally:
if finished is True:
self.last_count_access = SessionLockManager.LastCountAccess(
os.stat(self.cntfn).st_ctime, self.count)
def check(self):
with open(self.cntfn) as f:
newcount = int(f.readline())
try:
assert newcount >= self.count
except AssertionError:
raise RepositoryError(
"Count in lock file: %s is now inconsistent!" % self.cntfn)
sessions = os.listdir(self.sdir)
prevnames = set()
for session in sessions:
if not session.endswith(self.name + ".locks"):
continue
fd = None
try:
sf = self._path_helper(session)
if not self.afs:
fd = os.open(sf, os.O_RDWR)
fcntl.lockf(fd, fcntl.LOCK_SH) # ONLY NFS
with open(sf) as sf_file:
names = pickle.load(sf_file)
if not self.afs and fd is not None:
fcntl.lockf(fd, fcntl.LOCK_UN) # ONLY NFS
except Exception as x:
logger.warning("CHECKER: session file %s corrupted: %s %s" %
(session, getName(x), x))
continue
finally:
if fd is not None:
os.close(fd)
if not len(names & prevnames) == 0:
logger.error("Double-locked stuff: " + names & prevnames)
raise RepositoryError(
"Lock file has double-locked objects: %s" %
str(names & prevnames))
# prevnames.union_update(names) Should be alias to update but
# not in some versions of python
prevnames.update(names)
def session_write(self):
""" Writes the locked set to the session file.
The global lock MUST be held for this function to work, although on NFS additional
locking is done
Raises RepositoryError if session file is inaccessible """
#logger.debug("Openining Session File: %s " % self.fn )
try:
# If this fails, we want to shutdown the repository (corruption
# possible)
fd = None
try:
fd = os.open(self.fn, os.O_RDWR)
if not self.afs:
fcntl.lockf(fd, fcntl.LOCK_EX)
os.write(fd, pickle.dumps(self.locked))
if not self.afs:
fcntl.lockf(fd, fcntl.LOCK_UN)
os.fsync(fd)
finally:
if fd is not None:
os.close(fd)
except OSError as x:
if x.errno != errno.ENOENT:
raise RepositoryError(
self.repo,
"Error on session file access '%s': %s" % (self.fn, x))
else:
#logger.debug( "File NOT found %s" %self.fn )
raise RepositoryError(
self.repo,
"SessionWrite: Own session file not found! Possibly deleted by another ganga session.\n\
Possible reasons could be that this computer has a very high load, or that the system clocks on computers running Ganga are not synchronized.\n\
On computers with very high load and on network filesystems, try to avoid running concurrent ganga sessions for long.\n '%s' : %s"
% (self.fn, x))
except IOError as x:
raise RepositoryError(
self.repo,
"Error on session file locking '%s': %s" % (self.fn, x))
def setupGlobalLockRef(session_name, sdir, gfn, _on_afs):
global session_lock_refresher
if session_lock_refresher is None:
try:
# Just opening and closing a file to create it
os.close(os.open(gfn, os.O_EXCL | os.O_CREAT | os.O_WRONLY))
registerGlobalSessionFile(gfn)
except OSError as err:
logger.debug("Startup Lock Refresher Exception: %s" % err)
raise RepositoryError(
None, "Error on session file '%s' creation: %s" % (gfn, err))
session_lock_refresher = SessionLockRefresher(session_name, sdir, gfn,
None, _on_afs)
session_lock_refresher.start()
return session_lock_refresher
def updateLocksNow(self, index, failCount=0):
this_index_file = self.fns[index]
now = None
try:
oldnow = os.stat(this_index_file).st_ctime
os.system('touch "%s"' % this_index_file)
now = os.stat(
this_index_file).st_ctime # os.stat(self.fn).st_ctime
except OSError as x:
if x.errno != errno.ENOENT:
logger.debug(
"Session file timestamp could not be updated! Locks could be lost!"
)
if now is None and failCount < 4:
try:
logger.debug(
"Attempting to lock file again, unknown error:\n'%s'"
% x)
import time
time.sleep(0.5)
failcount = failCount + 1
now = self.updateLocksNow(index, failcount)
except Exception as err:
now = -999.
logger.debug(
"Received another type of exception, failing to update lockfile: %s"
% this_index_file)
else:
logger.warning("Failed to update lock file: %s 5 times." %
this_index_file)
logger.warning(
"This could be due to a filesystem problem, or multiple versions of ganga trying to access the same file"
)
now = -999.
else:
if self.repos[index] != None:
raise RepositoryError(
self.repos[index],
"[SessionFileUpdate] Run: Own session file not found! Possibly deleted by another ganga session.\n\
Possible reasons could be that this computer has a very high load, or that the system clocks on computers running Ganga are not synchronized.\n\
On computers with very high load and on network filesystems, try to avoid running concurrent ganga sessions for long.\n '%s' : %s"
% (this_index_file, x))
else:
from Ganga.Core.exceptions import GangaException
raise GangaException(
"Error Opening global .session file for this session: %s"
% this_index_file)
return now
def finish_startup(self):
# setup counter file if it does not exist, read it if it does
if not os.path.exists(self.cntfn):
fd = None
try:
fd = os.open(self.cntfn, os.O_EXCL | os.O_CREAT | os.O_WRONLY)
os.write(fd, "0")
except OSError as x:
if x.errno != errno.EEXIST:
raise RepositoryError(
self.repo, "OSError on count file create: %s" % x)
finally:
if fd is not None:
os.close(fd)
try:
self.count = max(self.count, self.cnt_read())
except ValueError, err:
logger.debug("Startup ValueError Exception: %s" % err)
logger.error("Corrupt count file '%s'! Trying to recover..." %
(self.cntfn))
def add(self, objs, force_ids=None):
"""
Add the object to the main dict
Args:
objs (list): Objects we want to store in memory
force_ids (list, None): IDs to assign to the objects, None for auto-assign
"""
try:
assert force_ids is None or len(force_ids) == len(objs)
except AssertionError:
raise RepositoryError(
"Inconsistent number of objects and ids, can't add to Repository"
)
ids = []
for i in range(len(objs)):
obj = objs[i]
if force_ids:
id = force_ids[i]
else:
id = self._next_id
self._internal_setitem__(id, obj)
ids.append(id)
self._next_id = max(self._next_id + 1, id + 1)
return ids
(self.cntfn))
except OSError as err:
logger.debug("Startup OSError Exception: %s" % err)
raise RepositoryError(
self.repo, "OSError on count file '%s' access!" % (self.cntfn))
self.cnt_write()
# Setup session file
fd = None
try:
fd = os.open(self.fn, os.O_EXCL | os.O_CREAT | os.O_WRONLY)
os.write(fd, pickle.dumps(set()))
registerGlobalSessionFile(self.fn)
except OSError as err:
logger.debug("Startup Session Exception: %s" % err)
raise RepositoryError(
self.repo,
"Error on session file '%s' creation: %s" % (self.fn, err))
finally:
if fd is not None:
os.close(fd)
setupGlobalLockRef(self.session_name, self.sdir, self.gfn, self.afs)
session_lock_refresher.addRepo(self.fn, self.repo)
self.session_write()
def updateNow(self):
session_lock_refresher.updateNow()
@synchronised
def shutdown(self):
class SessionLockManager(object):
""" Class with thread that keeps a global lock file that synchronizes
ID and counter access across Ganga sessions.
DEVELOPER WARNING: On NFS, files that are not locked with lockf (NOT flock) will
NOT be synchronized across clients, even if a global lock file is used!
Interface:
* startup() starts the session, automatically called on init
* shutdown() stops the thread, FREES ALL LOCKS
* make_new_ids(n) returns n new (locked) ids
* lock_ids(ids) returns the ids that were successfully locked
* release_ids(ids) returns the ids that were successfully released (now: all)
All access to an instance of this class MUST be synchronized!
Should ONLY raise RepositoryError (if possibly-corrupting errors are found)
"""
LastCountAccess = namedtuple('LastCountAccess', ['time', 'val'])
def mkdir(self, dn):
"""Make sure the given directory exists"""
try:
os.makedirs(dn)
except OSError as x:
if x.errno != errno.EEXIST:
raise RepositoryError(self.repo,
"OSError on directory create: %s" % x)
def __init__(self, repo, root, name, minimum_count=0):
self.repo = repo
self.mkdir(root)
realpath = os.path.realpath(root)
# Use the hostname (os.uname()[1]) and the current time in ms to construct the session filename.
# TODO: Perhaps put the username here?
if session_lock_refresher is None:
t = datetime.datetime.now()
this_date = t.strftime("%H.%M_%A_%d_%B_%Y")
session_name = ".".join([
os.uname()[1],
str(this_date), "PID",
str(os.getpid()), "session"
])
#session_name = ".".join([os.uname()[1], str(int(time.time()*1000)), str(os.getpid()), "session"])
else:
session_name = session_lock_refresher.session_name
self.sdir = os.path.join(realpath, "sessions")
self.gfn = os.path.join(self.sdir, session_name)
self.fn = os.path.join(self.sdir, session_name + "." + name + ".locks")
self.cntfn = os.path.join(realpath, name, "cnt")
self.afs = (realpath[:4] == "/afs")
self.locked = set()
self.count = minimum_count
self.session_name = session_name
self.name = name
self.realpath = realpath
#logger.debug( "Initializing SessionLockManager: " + self.fn )
self._lock = threading.RLock()
self.last_count_access = None
@synchronised
def startup(self):
self.last_count_access = None
# Ensure directories exist
self.mkdir(os.path.join(self.realpath, "sessions"))
self.mkdir(os.path.join(self.realpath, self.name))
# setup global lock
self.global_lock_setup()
self.finish_startup()
@global_disk_lock
def finish_startup(self):
# setup counter file if it does not exist, read it if it does
if not os.path.exists(self.cntfn):
fd = None
try:
fd = os.open(self.cntfn, os.O_EXCL | os.O_CREAT | os.O_WRONLY)
os.write(fd, "0")
except OSError as x:
if x.errno != errno.EEXIST:
raise RepositoryError(
self.repo, "OSError on count file create: %s" % x)
finally:
if fd is not None:
os.close(fd)
try:
self.count = max(self.count, self.cnt_read())
except ValueError, err:
logger.debug("Startup ValueError Exception: %s" % err)
logger.error("Corrupt count file '%s'! Trying to recover..." %
(self.cntfn))
except OSError as err:
logger.debug("Startup OSError Exception: %s" % err)
raise RepositoryError(
self.repo, "OSError on count file '%s' access!" % (self.cntfn))