if len(summary) > 0:
cnt = {}
examples = {}
for id,x in summary:
if id in self.known_bad_ids:
continue
cnt[x.__class__.__name__] = cnt.get(x.__class__.__name__,[]) + [str(id)]
examples[x.__class__.__name__] = str(x)
self.known_bad_ids.append(id)
global printed_explanation
for exc,ids in cnt.items():
if examples[exc].find('comments') > 0:
printed_explanation = True
from Ganga.Utility.repairJobRepository import repairJobRepository
for jobid in ids:
repairJobRepository(int(jobid))
else:
logger.error("Registry '%s': Failed to load %i jobs (IDs: %s) due to '%s' (first error: %s)" % (self.registry.name, len(ids), ",".join(ids), exc, examples[exc]))
if not printed_explanation:
logger.error("If you want to delete the incomplete objects, you can type 'for i in %s.incomplete_ids(): %s(i).remove()' (press 'Enter' twice)" % (self.registry.name, self.registry.name))
printed_explanation = True
logger.debug("updated index done")
return changed_ids
def add(self, objs, force_ids = None):
""" Add the given objects to the repository, forcing the IDs if told to.
Raise RepositoryError"""
if not force_ids is None: # assume the ids are already locked by Registry
if not len(objs) == len(force_ids):
raise RepositoryError(self, "Internal Error: add with different number of objects and force_ids!")
ids = force_ids
def update_index(self, id=None, verbose=False, firstRun=False):
""" Update the list of available objects
Raise RepositoryError"""
# First locate and load the index files
logger.debug("updating index...")
objs = self.get_index_listing()
changed_ids = []
deleted_ids = set(self.objects.keys())
summary = []
if firstRun:
self._read_master_cache()
for id, idx in objs.iteritems():
deleted_ids.discard(id)
# Make sure we do not overwrite older jobs if someone deleted the
# count file
if id > self.sessionlock.count:
self.sessionlock.count = id + 1
# Locked IDs can be ignored
if id in self.sessionlock.locked:
continue
# Now we treat unlocked IDs
try:
# if this succeeds, all is well and we are done
if self.index_load(id):
changed_ids.append(id)
continue
except IOError as err:
logger.debug("IOError: Failed to load index %i: %s" % (id, str(err)))
except OSError as err:
logger.debug("OSError: Failed to load index %i: %s" % (id, str(err)))
except PluginManagerError as err:
# Probably should be DEBUG
logger.debug("PluginManagerError: Failed to load index %i: %s" % (id, str(err)))
# This is a FATAL error - do not try to load the main file, it
# will fail as well
summary.append((id, err))
continue
# print id
# print self.objects
# this is bad - no or corrupted index but object not loaded yet!
# Try to load it!
if not id in self.objects:
try:
self.load([id])
changed_ids.append(id)
# Write out a new index if the file can be locked
if len(self.lock([id])) != 0:
self.index_write(id)
self.unlock([id])
#except KeyError as err:
# logger.debug("update Error: %s" % str(err))
# # deleted job
# if id in self.objects:
# self._internal_del__(id)
# changed_ids.append(id)
except Exception as x:
## WE DO NOT CARE what type of error occured here and it can be
## due to corruption so could be one of MANY exception types
## If the job is not accessible this should NOT cause the loading of ganga to fail!
## we can't reasonably write all possible exceptions here!
logger.debug("Failed to load id %i: %s %s" % (id, getName(x.orig), x.orig))
summary.append((id, x.orig))
# Check deleted files:
for id in deleted_ids:
self._internal_del__(id)
changed_ids.append(id)
if len(deleted_ids) > 0:
logger.warning("Registry '%s': Job %s externally deleted." % (
self.registry.name, ",".join(map(str, list(deleted_ids)))))
if len(summary) > 0:
cnt = {}
examples = {}
for id, x in summary:
if id in self.known_bad_ids:
continue
cnt[getName(x)] = cnt.get(getName(x), []) + [str(id)]
examples[getName(x)] = str(x)
self.known_bad_ids.append(id)
# add object to incomplete_objects
if not id in self.incomplete_objects:
self.incomplete_objects.append(id)
global printed_explanation
for exc, ids in cnt.items():
if examples[exc].find('comments') > 0:
printed_explanation = True
from Ganga.Utility.repairJobRepository import repairJobRepository
for jobid in ids:
repairJobRepository(int(jobid))
else:
logger.error("Registry '%s': Failed to load %i jobs (IDs: %s) due to '%s' (first error: %s)" % (
self.registry.name, len(ids), ",".join(ids), exc, examples[exc]))
if not printed_explanation:
logger.error("If you want to delete the incomplete objects, you can type 'for i in %s.incomplete_ids(): %s(i).remove()' (press 'Enter' twice)" % (
self.registry.name, self.registry.name))
logger.error(
"WARNING!!! This will result in corrupt jobs being completely deleted!!!")
printed_explanation = True
logger.debug("updated index done")
if len(changed_ids) != 0:
self._write_master_cache(shutdown=True)
return changed_ids
