def session_write(self):
""" Writes the locked set to the session file.
The global lock MUST be held for this function to work, although on NFS additional
locking is done
Raises RepositoryError if session file is inaccessible """
#logger.debug("Openining Session File: %s " % self.fn )
try:
# If this fails, we want to shutdown the repository (corruption
# possible)
fd = self.delayopen(self.fn)
if not self.afs:
fcntl.lockf(fd, fcntl.LOCK_EX)
self.delaywrite(fd, pickle.dumps(self.locked))
if not self.afs:
fcntl.lockf(fd, fcntl.LOCK_UN)
os.fsync(fd)
os.close(fd)
except OSError as x:
if x.errno != errno.ENOENT:
raise RepositoryError(
self.repo,
"Error on session file access '%s': %s" % (self.fn, x))
else:
#logger.debug( "File NOT found %s" %self.fn )
raise RepositoryError(
self.repo,
"SessionWrite: Own session file not found! Possibly deleted by another ganga session.\n\
Possible reasons could be that this computer has a very high load, or that the system clocks on computers running Ganga are not synchronized.\n\
On computers with very high load and on network filesystems, try to avoid running concurrent ganga sessions for long.\n '%s' : %s"
% (self.fn, x))
except IOError as x:
raise RepositoryError(
self.repo,
"Error on session file locking '%s': %s" % (self.fn, x))
def global_lock_setup(self):
if self.afs:
self.lockfn = os.path.join(self.sdir, "global_lock")
lock_path = str(self.lockfn) + '.afs'
lock_file = os.path.join(lock_path, "lock_file")
try:
if not os.path.exists(lock_path):
os.makedirs(lock_path)
if not os.path.isfile(lock_file):
lock_file_hand = open(lock_file, "w")
lock_file_hand.close()
except Exception as err:
logger.debug("Global Lock Setup Error: %s" % str(err))
else:
try:
self.lockfn = os.path.join(self.sdir, "global_lock")
if not os.path.isfile(self.lockfn):
lock = open(self.lockfn, "w")
# create file (does not interfere with existing sessions)
lock.close()
self.lockfd = self.delayopen_global(self.lockfn)
registerGlobalSessionFile(self.lockfn)
registerGlobalSessionFileHandler(self.lockfd)
except IOError as x:
raise RepositoryError(
self.repo,
"Could not create lock file '%s': %s" % (self.lockfn, x))
except OSError as x:
raise RepositoryError(
self.repo,
"Could not open lock file '%s': %s" % (self.lockfn, x))
def startup(self):
self.last_count_access = None
# Ensure directories exist
self.mkdir(os.path.join(self.realpath, "sessions"))
self.mkdir(os.path.join(self.realpath, self.name))
# setup global lock
self.global_lock_setup()
self.global_lock_acquire()
try:
# setup counter file if it does not exist, read it if it does
if not os.path.exists(self.cntfn):
try:
fd = self.delay_init_open(self.cntfn)
try:
os.write(fd, "0")
finally:
os.close(fd)
#registerGlobalSessionFile( self.cntfn )
except OSError as x:
if x.errno != errno.EEXIST:
raise RepositoryError(
self.repo, "OSError on count file create: %s" % x)
try:
self.count = max(self.count, self.cnt_read())
except ValueError, err:
logger.debug("Startup ValueError Exception: %s" % err)
logger.error("Corrupt count file '%s'! Trying to recover..." %
(self.cntfn))
except OSError as err:
logger.debug("Startup OSError Exception: %s" % err)
raise RepositoryError(
self.repo,
"OSError on count file '%s' access!" % (self.cntfn))
self.cnt_write()
# Setup session file
try:
fd = self.delay_init_open(self.fn)
try:
os.write(fd, pickle.dumps(set()))
finally:
os.close(fd)
registerGlobalSessionFile(self.fn)
except OSError as err:
logger.debug("Startup Session Exception: %s" % err)
raise RepositoryError(
self.repo,
"Error on session file '%s' creation: %s" % (self.fn, err))
setupGlobalLockRef(self.session_name, self.sdir, self.gfn,
self.afs)
session_lock_refresher.addRepo(self.fn, self.repo)
self.session_write()
def cnt_read(self):
""" Tries to read the counter file.
Raises ValueError (invalid contents)
Raises OSError (no access/does not exist)
Raises RepositoryError (fatal)
"""
try:
global last_count_access
if len(last_count_access) == 2:
last_count_time = last_count_access[0]
last_count_val = last_count_access[1]
last_time = os.stat(self.cntfn).st_ctime
if last_time == last_count_time:
last_count_val
_output = None
fd = os.open(self.cntfn, os.O_RDONLY)
try:
if not self.afs: # additional locking for NFS
fcntl.lockf(fd, fcntl.LOCK_SH)
# 100 bytes should be enough for any ID. Can raise ValueErrorr
_output = int(os.read(fd, 100).split("\n")[0])
finally:
if not self.afs: # additional locking for NFS
fcntl.lockf(fd, fcntl.LOCK_UN)
os.close(fd)
if _output != None:
if len(last_count_access) != 2:
last_count_access = []
last_count_access.append(os.stat(self.cntfn).st_ctime)
last_count_access.append(_output)
else:
last_count_access[0] = os.stat(self.cntfn).st_ctime
last_count_access[1] = _output
return _output
except OSError as x:
if x.errno != errno.ENOENT:
raise RepositoryError(
self.repo,
"OSError on count file '%s' read: %s" % (self.cntfn, x))
else:
# This can be a recoverable error, depending on where it occurs
raise
except IOError as x:
raise RepositoryError(
self.repo,
"Locking error on count file '%s' write: %s" % (self.cntfn, x))
def flush(self, ids):
logger.debug("Flushing: %s" % ids)
#import traceback
#traceback.print_stack()
for this_id in ids:
try:
logger.debug("safe_flush")
self._safe_flush_xml(this_id)
self._cache_load_timestamp[this_id] = time.time()
self._cached_cls[this_id] = getName(self.objects[this_id])
self._cached_cat[this_id] = self.objects[this_id]._category
self._cached_obj[this_id] = self.objects[
this_id].getNodeIndexCache()
try:
self.index_write(this_id)
except:
logger.debug("Index write failed")
pass
if this_id not in self._fully_loaded.keys():
self._fully_loaded[this_id] = self.objects[this_id]
except (OSError, IOError, XMLFileError) as x:
raise RepositoryError(
self, "Error of type: %s on flushing id '%s': %s" %
(type(x), str(this_id), str(x)))
def __init__(self, repo, root, name, minimum_count=0):
# Used for self runtime thread safety
self.sync_lock = threading.RLock()
# Required to pretend there are some locked jobs from other sessions
self.locked = set()
self.repo = repo
self.mkdir(root)
realpath = os.path.realpath(root)
# Location of the count file for this repo
self.cntfn = os.path.join(realpath, name, "cnt")
# Required for lock files
sessions_folder = os.path.join(realpath, "sessions")
if not os.path.exists(sessions_folder):
os.mkdir(sessions_folder)
# Location of fixed lock for this repo
self.global_lock = os.path.join(sessions_folder,
'%s_fixed_lock' % name)
self.count = minimum_count
if os.path.exists(self.global_lock):
msg = "\n\nCannot start this registry: %s due to a lock file already existing: '%s'\n" % (
name, self.global_lock)
msg += "If you are trying to run only 1 ganga session please remove this file and re-launch ganga\n"
raise RepositoryError(repo, msg)
with open(self.global_lock, 'w'):
pass
def _add(self, obj, force_index=None):
""" Add an object to the registry and assigns an ID to it.
use force_index to set the index (for example for metadata). This overwrites existing objects!
Raises RepositoryError
Args:
_obj (GangaObject): This is th object which is to be added to this Registy/Repo
force_index (int): This is the index which we will give the object, None = auto-assign
"""
logger.debug("_add")
if self.hasStarted() is not True:
raise RepositoryError("Cannot add objects to a disconnected repository!")
if force_index is None:
ids = self.repository.add([obj])
else:
if len(self.repository.lock([force_index])) == 0:
raise RegistryLockError("Could not lock '%s' id #%i for a new object!" % (self.name, force_index))
ids = self.repository.add([obj], [force_index])
obj._setRegistry(self)
obj._registry_locked = True
self.repository.flush(ids)
return ids[0]
def session_read(self, fn):
""" Reads a session file and returns a set of IDs locked by that session.
The global lock MUST be held for this function to work, although on NFS additional
locking is done
Raises RepositoryError if severe access problems occur (corruption otherwise!) """
try:
# This can fail (thats OK, file deleted in the meantime)
fd = self.delay_session_open(fn)
os.lseek(fd, 0, 0)
try:
if not self.afs: # additional locking for NFS
fcntl.lockf(fd, fcntl.LOCK_SH)
try:
# 00)) # read up to 1 MB (that is more than enough...)
return pickle.loads(os.read(fd, 1048576))
except Exception as x:
logger.warning(
"corrupt or inaccessible session file '%s' - ignoring it (Exception %s %s)."
% (fn, getName(x), str(x)))
finally:
if not self.afs: # additional locking for NFS
fcntl.lockf(fd, fcntl.LOCK_UN)
os.close(fd)
except OSError as x:
if x.errno != errno.ENOENT:
raise RepositoryError(
self.repo,
"Error on session file access '%s': %s" % (fn, x))
return set()
def add(self, objs, force_ids=None):
ids = []
def _readonly():
return True
for o in objs:
obj = copy.deepcopy(o)
fn = os.path.join(self.filebase,
'%s.%s' % (obj.name, self.file_ext))
try:
if self.pickle_files:
obj._registry = None
pickle.dump(obj, open(fn, 'wb'))
else:
if not stripped_export(obj, fn):
raise RepositoryError(
self,
'Failure in stripped_export method, returned False'
)
except:
logger.error("Unable to write to file '%s'" % fn)
raise
else:
obj.id = self._next_id
obj._registry = self.registry
obj._registry_id = self._next_id
setattr(obj, '_readonly', _readonly)
self.objects[self._next_id] = obj
ids.append(self._next_id)
self._next_id += 1
return ids
def startup(self):
""" Starts a repository and reads in a directory structure.
Raise RepositoryError"""
self._load_timestamp = {}
# New Master index to speed up loading of many, MANY files
self._cache_load_timestamp = {}
self._cached_cat = {}
self._cached_cls = {}
self._cached_obj = {}
self._master_index_timestamp = 0
self.known_bad_ids = []
if "XML" in self.registry.type:
self.to_file = xml_to_file
self.from_file = xml_from_file
elif "Pickle" in self.registry.type:
self.to_file = pickle_to_file
self.from_file = pickle_from_file
else:
raise RepositoryError(
self.repo, "Unknown Repository type: %s" % self.registry.type)
self.sessionlock = SessionLockManager(self, self.lockroot,
self.registry.name)
self.sessionlock.startup()
# Load the list of files, this time be verbose and print out a summary
# of errors
self.update_index(verbose=True, firstRun=True)
logger.debug("GangaRepositoryLocal Finished Startup")
def lock_ids(self, ids):
self.safe_LockCheck()
#logger.debug( "locking: %s" % str(ids) )
ids = set(ids)
self.global_lock_acquire()
try:
try:
sessions = [
sn for sn in os.listdir(self.sdir)
if sn.endswith(self.name + ".locks")
]
except OSError as x:
raise RepositoryError(
self.repo,
"Could not list session directory '%s'!" % (self.sdir))
slocked = set()
for session in sessions:
sf = os.path.join(self.sdir, session)
if sf == self.fn:
continue
slocked.update(self.session_read(sf))
#logger.debug( "locked: %s" % str(slocked) )
ids.difference_update(slocked)
self.locked.update(ids)
#logger.debug( "stored_lock: %s" % str(self.locked) )
self.session_write()
#logger.debug( "list: %s" % str(list(ids)) )
return list(ids)
finally:
self.global_lock_release()
def make_new_ids(self, n):
""" Locks the next n available ids and returns them as a list
Raise RepositoryError on fatal error"""
self.global_lock_acquire()
try:
# Actualize count
try:
newcount = self.cnt_read()
except ValueError:
logger.warning(
"Corrupt job counter (possibly due to crash of another session)! Trying to recover..."
)
newcount = self.count
except OSError:
raise RepositoryError(
self.repo,
"Job counter deleted! External modification to repository!"
)
if not newcount >= self.count:
#raise RepositoryError(self.repo, "Counter value decreased - logic error!")
logger.warning(
"Internal counter increased - probably the count file was deleted."
)
newcount = self.count
# someone used force_ids (for example old repository imports)
if self.locked and max(self.locked) >= newcount:
newcount = max(self.locked) + 1
ids = range(newcount, newcount + n)
self.locked.update(ids)
self.count = newcount + n
self.cnt_write()
self.session_write()
return list(ids)
finally:
self.global_lock_release()
def mkdir(self, dn):
"""Make sure the given directory exists"""
try:
os.makedirs(dn)
except OSError as x:
if x.errno != errno.EEXIST:
raise RepositoryError(self.repo,
"OSError on directory create: %s" % x)
def add(self, objs, force_ids=None):
""" Add the given objects to the repository, forcing the IDs if told to.
Raise RepositoryError"""
logger.debug("add")
if force_ids not in [
None, []
]: # assume the ids are already locked by Registry
if not len(objs) == len(force_ids):
raise RepositoryError(
self,
"Internal Error: add with different number of objects and force_ids!"
)
ids = force_ids
else:
ids = self.sessionlock.make_new_ids(len(objs))
logger.debug("made ids")
for i in range(0, len(objs)):
fn = self.get_fn(ids[i])
try:
os.makedirs(os.path.dirname(fn))
except OSError as e:
if e.errno != errno.EEXIST:
raise RepositoryError(self,
"OSError on mkdir: %s" % (str(e)))
self._internal_setitem__(ids[i], objs[i])
# Set subjobs dirty - they will not be flushed if they are not.
if self.sub_split and self.sub_split in objs[i].getNodeData():
try:
sj_len = len(objs[i].getNodeAttribute(self.sub_split))
if sj_len > 0:
for j in range(sj_len):
objs[i].getNodeAttribute(
self.sub_split)[j]._dirty = True
except AttributeError as err:
logger.debug("RepoXML add Exception: %s" % str(err))
logger.debug("Added")
return ids
def cnt_read(self):
""" Tries to read the counter file.
Raises ValueError (invalid contents)
Raises OSError (no access/does not exist)
Raises RepositoryError (fatal)
"""
try:
if self.last_count_access is not None:
last_count_time = self.last_count_access.time
last_count_val = self.last_count_access.val
last_time = os.stat(self.cntfn).st_ctime
if last_time == last_count_time:
return last_count_val # If the file hasn't changed since last check, return the cached value
_output = None
fd = os.open(self.cntfn, os.O_RDONLY)
try:
if not self.afs: # additional locking for NFS
fcntl.lockf(fd, fcntl.LOCK_SH)
# 100 bytes should be enough for any ID. Can raise ValueErrorr
_output = int(os.read(fd, 100).split("\n")[0])
finally:
if not self.afs: # additional locking for NFS
fcntl.lockf(fd, fcntl.LOCK_UN)
os.close(fd)
if _output != None:
self.last_count_access = SessionLockManager.LastCountAccess(
os.stat(self.cntfn).st_ctime, _output)
return _output
except OSError as x:
if x.errno != errno.ENOENT:
raise RepositoryError(
self.repo,
"OSError on count file '%s' read: %s" % (self.cntfn, x))
else:
# This can be a recoverable error, depending on where it occurs
raise
except IOError as x:
raise RepositoryError(
self.repo,
"Locking error on count file '%s' write: %s" % (self.cntfn, x))
def get_index_listing(self):
"""Get dictionary of possible objects in the Repository: True means index is present,
False if not present
Raise RepositoryError"""
try:
obj_chunks = [
d for d in os.listdir(self.root)
if d.endswith("xxx") and d[:-3].isdigit()
]
except OSError as err:
logger.debug("get_index_listing Exception: %s" % str(err))
raise RepositoryError(
self, "Could not list repository '%s'!" % (self.root))
objs = {} # True means index is present, False means index not present
for c in obj_chunks:
try:
listing = os.listdir(os.path.join(self.root, c))
except OSError as err:
logger.debug("get_index_listing Exception: %s")
raise RepositoryError(
self, "Could not list repository '%s'!" %
(os.path.join(self.root, c)))
objs.update(dict([(int(l), False) for l in listing
if l.isdigit()]))
for l in listing:
if l.endswith(".index") and l[:-6].isdigit():
this_id = int(l[:-6])
if this_id in objs:
objs[this_id] = True
else:
try:
rmrf(self.get_idxfn(this_id))
logger.warning(
"Deleted index file without data file: %s" %
self.get_idxfn(this_id))
except OSError as err:
logger.debug(
"get_index_listing delete Exception: %s" %
str(err))
return objs
def cnt_write(self):
""" Writes the counter to the counter file.
The global lock MUST be held for this function to work correctly
Raises OSError if count file is inaccessible """
finished = False
try:
# If this fails, we want to shutdown the repository (corruption
# possible)
fd = os.open(self.cntfn, os.O_WRONLY)
if not self.afs:
fcntl.lockf(fd, fcntl.LOCK_EX)
os.write(fd, str(self.count) + "\n")
if not self.afs:
fcntl.lockf(fd, fcntl.LOCK_UN)
os.close(fd)
finished = True
except OSError as x:
if x.errno != errno.ENOENT:
raise RepositoryError(
self.repo,
"OSError on count file '%s' write: %s" % (self.cntfn, x))
else:
raise RepositoryError(
self.repo,
"Count file '%s' not found! Repository was modified externally!"
% (self.cntfn))
except IOError as x:
raise RepositoryError(
self.repo,
"Locking error on count file '%s' write: %s" % (self.cntfn, x))
finally:
if finished is True:
global last_count_access
if len(last_count_access) != 2:
last_count_access = []
last_count_access.append(os.stat(self.cntfn).st_ctime)
last_count_access.append(self.count)
else:
last_count_access[0] = os.stat(self.cntfn).st_ctime
last_count_access[1] = self.count
def global_lock_release(self):
try:
if self.afs:
lock_path = str(self.lockfn) + '.afs'
os.system("fs setacl %s %s rlidwka" %
(lock_path, getpass.getuser()))
else:
self.delay_lock_mod(self.lockfd, fcntl.LOCK_UN)
#logger.debug("global release")
except IOError as x:
raise RepositoryError(
self.repo, "IOError on unlock ('%s'): %s" % (self.lockfn, x))
def _open_xml_file(self, fn, this_id, _copy_backup=False):
"""
This loads the XML for the job "this_id" in self.objects using the file "fn" and knowing whether we want the file or the backup by _copy_backup
"""
fobj = None
has_loaded_backup = False
try:
if not os.path.isfile(fn) and _copy_backup:
if os.path.isfile(fn + '~'):
logger.warning(
"XML File: %s missing, recovering from backup, recent changes may have been lost!"
% fn)
has_loaded_backup = True
try:
from shutil import copyfile
copyfile(fn + '~', fn)
except:
logger.warning(
"Error Recovering the backup file! loading of Job may Fail!"
)
fobj = open(fn, "r")
except IOError as x:
if x.errno == errno.ENOENT:
# remove index so we do not continue working with wrong information
try:
# remove internal representation
self._internal_del__(this_id)
rmrf(os.path.dirname(fn) + ".index")
except OSError as err:
logger.debug("load unlink Error: %s" % err)
pass
raise KeyError(this_id)
else:
raise RepositoryError(self, "IOError: %s" % x)
finally:
try:
if os.path.isdir(os.path.dirname(fn)):
ld = os.listdir(os.path.dirname(fn))
if len(ld) == 0:
os.rmdir(os.path.dirname(fn))
logger.warning(
"No job index or data found, removing empty directory: %s"
% os.path.dirname(fn))
except Exception as err:
logger.debug("load error %s" % err)
pass
return fobj, has_loaded_backup
def getGlobalLockRef(session_name, sdir, gfn, _on_afs):
global session_lock_refresher
if session_lock_refresher is None:
try:
os.close(SessionLockManager.delay_init_open(gfn))
registerGlobalSessionFile(gfn)
except OSError as err:
logger.debug("Startup Lock Refresher Exception: %s" % str(err))
raise RepositoryError(
None, "Error on session file '%s' creation: %s" % (gfn, err))
session_lock_refresher = SessionLockRefresher(session_name, sdir, gfn,
None, _on_afs)
session_lock_refresher.start()
return session_lock_refresher
def _reallyUpdateLocks(self, index, failCount=0):
this_index_file = self.fns[index]
now = None
try:
oldnow = self.delayread(this_index_file)
os.system('touch %s' % str(this_index_file))
now = self.delayread(this_index_file) # os.stat(self.fn).st_ctime
except OSError as x:
if x.errno != errno.ENOENT:
logger.debug(
"Session file timestamp could not be updated! Locks could be lost!"
)
if now is None and failCount < 4:
try:
logger.debug(
"Attempting to lock file again, unknown error:\n'%s'"
% str(x))
import time
time.sleep(0.5)
failcount = failCount + 1
now = self._reallyUpdateLocks(index, failcount)
except Exception as err:
now = -999.
logger.debug(
"Received another type of exception, failing to update lockfile: %s"
% str(this_index_file))
else:
logger.warning("Failed to update lock file: %s 5 times." %
str(this_index_file))
logger.warning(
"This could be due to a filesystem problem, or multiple versions of ganga trying to access the same file"
)
now = -999.
else:
if self.repos[index] != None:
raise RepositoryError(
self.repos[index],
"[SessionFileUpdate] Run: Own session file not found! Possibly deleted by another ganga session.\n\
Possible reasons could be that this computer has a very high load, or that the system clocks on computers running Ganga are not synchronized.\n\
On computers with very high load and on network filesystems, try to avoid running concurrent ganga sessions for long.\n '%s' : %s"
% (this_index_file, x))
else:
from Ganga.Core import GangaException
raise GangaException(
"Error Opening global .session file for this session: %s"
% this_index_file)
return now
def global_lock_acquire(self):
try:
if self.afs:
lock_path = str(self.lockfn) + '.afs'
lock_file = os.path.join(lock_path, "lock_file")
def clean_path():
oldtime = os.stat(lock_file).st_ctime
nowtime = time.time()
if abs(int(nowtime) - oldtime) > 10:
#logger.debug( "cleaning global lock" )
os.system("fs setacl %s %s rlidwka" %
(lock_path, getpass.getuser()))
while True:
try:
if os.path.isfile(lock_file):
clean_path()
os.unlink(lock_file)
break
except Exception as err:
logger.debug("Global Lock aquire Exception: %s" %
str(err))
time.sleep(0.01)
os.system("fs setacl %s %s rliwka" %
(lock_path, getpass.getuser()))
while not os.path.isfile(lock_file):
lock_file_hand = open(lock_file, "w")
lock_file_hand.close()
time.sleep(0.01)
else:
self.delay_lock_mod(self.lockfd, fcntl.LOCK_EX)
#logger.debug("global capture")
except IOError as x:
raise RepositoryError(
self.repo, "IOError on lock ('%s'): %s" % (self.lockfn, x))
def _safe_flush_xml(self, this_id):
fn = self.get_fn(this_id)
obj = self.objects[this_id]
from Ganga.Core.GangaRepository.VStreamer import EmptyGangaObject
if not isType(obj, EmptyGangaObject):
split_cache = None
has_children = (not self.sub_split is None) and (
self.sub_split in obj.getNodeData()) and obj.getNodeAttribute(
self.sub_split) and len(
obj.getNodeAttribute(self.sub_split)) > 0
if has_children:
logger.debug("has_children")
if hasattr(obj.getNodeAttribute(self.sub_split), 'flush'):
# I've been read from disk in the new SubJobXMLList format I know how to flush
obj.getNodeAttribute(self.sub_split).flush()
else:
# I have been constructed in this session, I don't know how to flush!
if hasattr(
obj.getNodeAttribute(self.sub_split)[0], "_dirty"):
split_cache = obj.getNodeAttribute(self.sub_split)
for i in range(len(split_cache)):
if not split_cache[i]._dirty:
continue
sfn = os.path.join(os.path.dirname(fn), str(i),
self.dataFileName)
if not os.path.exists(os.path.dirname(sfn)):
logger.debug("Constructing Folder: %s" %
str(os.path.dirname(sfn)))
os.makedirs(os.path.dirname(sfn))
else:
logger.debug("Using Folder: %s" %
str(os.path.dirname(sfn)))
safe_save(sfn, split_cache[i], self.to_file)
split_cache[i]._setFlushed()
from Ganga.Core.GangaRepository.SubJobXMLList import SubJobXMLList
# Now generate an index file to take advantage of future non-loading goodness
tempSubJList = SubJobXMLList(os.path.dirname(fn),
self.registry,
self.dataFileName,
False,
parent=obj)
## equivalent to for sj in job.subjobs
tempSubJList._setParent(obj)
job_dict = {}
for sj in obj.getNodeAttribute(self.sub_split):
job_dict[sj.id] = stripProxy(sj)
tempSubJList._reset_cachedJobs(job_dict)
tempSubJList.flush()
del tempSubJList
safe_save(fn, obj, self.to_file, self.sub_split)
# clean files not in subjobs anymore... (bug 64041)
for idn in os.listdir(os.path.dirname(fn)):
split_cache = obj.getNodeAttribute(self.sub_split)
if idn.isdigit() and int(idn) >= len(split_cache):
rmrf(os.path.join(os.path.dirname(fn), idn))
else:
logger.debug("not has_children")
safe_save(fn, obj, self.to_file, "")
# clean files leftover from sub_split
for idn in os.listdir(os.path.dirname(fn)):
if idn.isdigit():
rmrf(os.path.join(os.path.dirname(fn), idn))
self.index_write(this_id)
#obj.setNodeIndexCache(None)
obj._setFlushed()
else:
raise RepositoryError(
self, "Cannot flush an Empty object for ID: %s" % str(this_id))
if this_id not in self._fully_loaded.keys():
self._fully_loaded[this_id] = obj
def load(self, ids, load_backup=False):
"""
Load the following "ids" from disk
If we want to load the backup files for these ids then use _copy_backup
Correctly loaded objects are dirty, Objects loaded from backups for whatever reason are marked dirty
"""
#print("load: %s " % ids)
#import traceback
#traceback.print_stack()
#print("\n")
logger.debug("Loading Repo object(s): %s" % ids)
for this_id in ids:
if this_id in self.incomplete_objects:
raise RepositoryError(
self.repo,
"Trying to re-load a corrupt repository id: %s" % this_id)
fn = self.get_fn(this_id)
if load_backup:
has_loaded_backup = True
fn = fn + "~"
else:
has_loaded_backup = False
fobj = None
try:
fobj, has_loaded_backup2 = self._open_xml_file(
fn, this_id, _copy_backup=True)
if has_loaded_backup2:
has_loaded_backup = has_loaded_backup2
except Exception as err:
logger.debug("XML load: Failed to load XML file: %s" % fn)
logger.debug("Error was:\n%s" % err)
logger.error(
"Adding id: %s to Corrupt IDs will not attempt to re-load this session"
% this_id)
self.incomplete_objects.append(this_id)
raise
try:
self._actually_load_xml(fobj, fn, this_id, load_backup)
except RepositoryError as err:
logger.debug("Repo Exception: %s" % err)
logger.error(
"Adding id: %s to Corrupt IDs will not attempt to re-load this session"
% this_id)
self.incomplete_objects.append(this_id)
raise
except Exception as err:
should_continue = self._handle_load_exception(
err, fn, this_id, load_backup)
if should_continue is True:
has_loaded_backup = True
continue
else:
logger.error(
"Adding id: %s to Corrupt IDs will not attempt to re-load this session"
% this_id)
self.incomplete_objects.append(this_id)
raise
finally:
fobj.close()
if has_loaded_backup:
self.objects[this_id]._setDirty()
else:
self.objects[this_id]._setFlushed()
logger.debug("Finished 'load'-ing of: %s" % ids)
