def lookup(self, sharedir, unprepare=False):
"""
Report Job, Box and Task repository items which reference a given ShareDir object.
The optional parameter 'unprepare=True' can be set to call the unprepare method
on the returned objects.
"""
from Ganga.GPI import jobs, box, tasks
objectlist = []
for thing in jobs.select():
objectlist.append({thing: 'job'})
for thing in box.select():
objectlist.append({thing: 'box'})
for thing in tasks.select():
objectlist.append({thing: 'task'})
run_unp = None
master_index = 0
for item in objectlist:
try:
item.keys()[0].is_prepared.name
if item.keys()[0].is_prepared.name == sharedir:
logger.info('ShareDir %s is referenced by item #%s in %s repository', sharedir,
stripProxy(item.keys()[0])._registry_id, item.values()[0])
run_unp = item.keys()[0]
master_index += 1
except AttributeError as err:
logger.debug("Err: %s" % str(err))
try:
item.keys()[0].application.is_prepared.name
if item.keys()[0].application.is_prepared.name == sharedir:
logger.info('ShareDir %s is referenced by item #%s in %s repository', sharedir,
stripProxy(item.keys()[0])._registry_id, item.values()[0])
run_unp = item.keys()[0].application
master_index += 1
except AttributeError as err2:
logger.debug("Err2: %s" % str(err2))
try:
item.keys()[0].analysis.application.is_prepared.name
if item.keys()[0].analysis.application.is_prepared.name == sharedir:
logger.info('ShareDir %s is referenced by item #%s in %s repository', sharedir,
stripProxy(item.keys()[0])._registry_id, item.values()[0])
run_unp = item.keys()[0].analysis.application
master_index += 1
except AttributeError as err3:
logger.debug("Err3: %s" % str(err3))
pass
if run_unp is not None and unprepare is True:
logger.info('Unpreparing %s repository object #%s associated with ShareDir %s',
item.values()[0], stripProxy(item.keys()[0])._registry_id, sharedir)
# stripProxy(item.keys()[0]).unprepare()
run_unp.unprepare()
run_unp = None
if unprepare is not True:
logger.info('%s item(s) found referencing ShareDir %s', master_index, sharedir)
def rebuild(self, unprepare=True, rmdir=False):
"""Rebuild the shareref table.
Clears the shareref table and then rebuilds it by iterating over all Ganga Objects
in the Job, Box and Task repositories. If an object has a ShareDir associated with it,
that ShareDir is added into the shareref table (the reference counter being incremented
accordingly). If called with the optional parameter 'unprepare=False', objects whose
ShareDirs are not present on disk will not be unprepared. Note that the default behaviour
is unprepare=True, i.e. the job/application would be unprepared.
After all Job/Box/Task objects have been checked, the inverse operation is performed,
i.e., for each directory in the ShareDir repository, a check is done to ensure there
is a matching entry in the shareref table. If not, and the optional parameter
'rmdir=True' is set, then the (orphaned) ShareDir will removed from the filesystem.
Otherwise, it will be added to the shareref table with a reference count of zero;
this results in the directory being deleted upon Ganga exit.
"""
self._getWriteAccess()
from Ganga.GPI import jobs, box, tasks
# clear the shareref table
self.name = {}
lookup_input = []
from Ganga.GPIDev.Lib.File import getSharedPath
objectlist = []
for thing in jobs.select():
objectlist.append({thing: 'job'})
for thing in box.select():
objectlist.append({thing: 'box'})
for thing in tasks.select():
objectlist.append({thing: 'task'})
for item in objectlist:
shortname = None
try:
shortname = item.keys()[0].is_prepared.name
except AttributeError as err:
logger.debug("Err: %s" % str(err))
try:
shortname = item.keys()[0].application.is_prepared.name
except AttributeError as err2:
logger.debug("Err2: %s" % str(err2))
try:
shortname = item.keys()[0].analysis.application.is_prepared.name
except AttributeError as err3:
logger.debug("Err3: %s" % str(err3))
pass
try:
if shortname is not None and shortname is not True:
if os.path.basename(shortname) != shortname:
self.to_relative(item.keys()[0].is_prepared)
try:
numsubjobs = len(item.keys()[0].subjobs.ids())
except Exception as err:
logger.debug("Path Error: %s" % str(err))
Ganga.Utility.logging.log_unknown_exception()
numsubjobs = 0
self.helper(shortname, unp=unprepare, numsubjobs=numsubjobs)
except Exception as err:
logger.debug("-Error: %s" % str(err))
Ganga.Utility.logging.log_unknown_exception()
pass
# here we iterate over the lookup_input list and unprepare as
# necessary.
for item in lookup_input:
logger.info('Unpreparing objects referencing ShareDir %s' % item)
self.lookup(sharedir=item, unprepare=True)
# check to see that all sharedirs have an entry in the shareref. Otherwise, set their ref counter to 0
# so the user is made aware of them at shutdown
for this_dir in os.listdir(getSharedPath()):
if this_dir not in self.__getName().keys() and rmdir is False:
logger.debug("%s isn't referenced by a GangaObject in the Job or Box repository." % this_dir)
self.__getName()[this_dir] = 0
elif this_dir not in self.__getName().keys() and rmdir is True:
logger.debug("%s isn't referenced by a GangaObject in the Job or Box repository. Removing directory." % this_dir)
shutil.rmtree(os.path.join(getSharedPath(), this_dir))
self._setDirty()
self._releaseWriteAccess()
def rebuild(self, unprepare=True, rmdir=False):
"""Rebuild the shareref table.
Clears the shareref table and then rebuilds it by iterating over all Ganga Objects
in the Job, Box and Task repositories. If an object has a ShareDir associated with it,
that ShareDir is added into the shareref table (the reference counter being incremented
accordingly). If called with the optional parameter 'unprepare=False', objects whose
ShareDirs are not present on disk will not be unprepared. Note that the default behaviour
is unprepare=True, i.e. the job/application would be unprepared.
After all Job/Box/Task objects have been checked, the inverse operation is performed,
i.e., for each directory in the ShareDir repository, a check is done to ensure there
is a matching entry in the shareref table. If not, and the optional parameter
'rmdir=True' is set, then the (orphaned) ShareDir will removed from the filesystem.
Otherwise, it will be added to the shareref table with a reference count of zero;
this results in the directory being deleted upon Ganga exit.
"""
self._getWriteAccess()
from Ganga.GPI import jobs, box, tasks
#clear the shareref table
self.name={}
lookup_input=[]
def helper(object, unp=True, numsubjobs=0):
shareddir = os.path.join(ShareDir._root_shared_path,os.path.basename(object))
logger.debug('Adding %s to the shareref table.' % shareddir)
if self.name.has_key(os.path.basename(object)):
self.name[os.path.basename(object)] +=1
else:
self.name[os.path.basename(object)] = 1
if numsubjobs > 0:
self.name[os.path.basename(object)] += numsubjobs
if not os.path.isdir(shareddir) and os.path.basename(object) not in lookup_input:
logger.info('Shared directory %s not found on disk.' % shareddir)
if unp == True:
lookup_input.append(os.path.basename(object))
def to_relative(object):
logger.info('Absolute ShareDir().name attribute found in Job #%s', object.id)
logger.info('Converting to relative path and moving associated directory if it exists.')
try:
shutil.move(object.is_prepared.name,\
os.path.join(ShareDir._root_shared_path,os.path.basename(object.is_prepared.name)))
except:
logger.warn('Unable to move directory %s to %s', object.is_prepared.name,\
os.path.join(ShareDir._root_shared_path,os.path.basename(object.is_prepared.name)))
try:
object._impl.is_prepared.name = os.path.basename(object.is_prepared.name)
except:
logger.warn("Unable to convert object's is_prepared.name attribute to a relative path")
objectlist = []
for thing in jobs.select():
objectlist.append({thing:'job'})
for thing in box.select():
objectlist.append({thing:'box'})
for thing in tasks.select():
objectlist.append({thing:'task'})
for item in objectlist:
shortname = None
try:
shortname = item.keys()[0].is_prepared.name
except AttributeError:
try:
shortname = item.keys()[0].application.is_prepared.name
except AttributeError:
try:
shortname = item.keys()[0].analysis.application.is_prepared.name
except AttributeError:
pass
try:
if shortname is not None and shortname is not True:
if os.path.basename(shortname) != shortname:
to_relative(item.keys()[0].is_prepared)
try:
numsubjobs = len(item.keys()[0].subjobs.ids())
except:
numsubjobs = 0
helper(shortname, unp=unprepare, numsubjobs=numsubjobs)
except:
pass
#here we iterate over the lookup_input list and unprepare as necessary.
for item in lookup_input:
logger.info('Unpreparing objects referencing ShareDir %s' % item)
self.lookup(sharedir=item, unprepare=True)
#check to see that all sharedirs have an entry in the shareref. Otherwise, set their ref counter to 0
#so the user is made aware of them at shutdown
for dir in os.listdir(ShareDir._root_shared_path):
if not self.name.has_key(dir) and rmdir is False:
logger.debug("%s isn't referenced by a GangaObject in the Job or Box repository." % dir)
self.name[dir] = 0
elif not self.name.has_key(dir) and rmdir is True:
logger.debug("%s isn't referenced by a GangaObject in the Job or Box repository. Removing directory." % dir)
shutil.rmtree(os.path.join(ShareDir._root_shared_path,dir))
self._setDirty()
self._releaseWriteAccess()
def rebuild(self, unprepare=True, rmdir=False):
"""Rebuild the shareref table.
Clears the shareref table and then rebuilds it by iterating over all Ganga Objects
in the Job, Box and Task repositories. If an object has a ShareDir associated with it,
that ShareDir is added into the shareref table (the reference counter being incremented
accordingly). If called with the optional parameter 'unprepare=False', objects whose
ShareDirs are not present on disk will not be unprepared. Note that the default behaviour
is unprepare=True, i.e. the job/application would be unprepared.
After all Job/Box/Task objects have been checked, the inverse operation is performed,
i.e., for each directory in the ShareDir repository, a check is done to ensure there
is a matching entry in the shareref table. If not, and the optional parameter
'rmdir=True' is set, then the (orphaned) ShareDir will removed from the filesystem.
Otherwise, it will be added to the shareref table with a reference count of zero;
this results in the directory being deleted upon Ganga exit.
"""
self._getWriteAccess()
from Ganga.GPI import jobs, box, tasks
# clear the shareref table
self.name = {}
lookup_input = []
from Ganga.GPIDev.Lib.File import getSharedPath
objectlist = []
for thing in jobs.select():
objectlist.append({thing: 'job'})
for thing in box.select():
objectlist.append({thing: 'box'})
for thing in tasks.select():
objectlist.append({thing: 'task'})
for item in objectlist:
shortname = None
try:
shortname = item.keys()[0].is_prepared.name
except AttributeError as err:
logger.debug("Err: %s" % str(err))
try:
shortname = item.keys()[0].application.is_prepared.name
except AttributeError as err2:
logger.debug("Err2: %s" % str(err2))
try:
shortname = item.keys(
)[0].analysis.application.is_prepared.name
except AttributeError as err3:
logger.debug("Err3: %s" % str(err3))
pass
try:
if shortname is not None and shortname is not True:
if os.path.basename(shortname) != shortname:
self.to_relative(item.keys()[0].is_prepared)
try:
numsubjobs = len(item.keys()[0].subjobs.ids())
except Exception as err:
logger.debug("Path Error: %s" % str(err))
Ganga.Utility.logging.log_unknown_exception()
numsubjobs = 0
self.helper(shortname,
unp=unprepare,
numsubjobs=numsubjobs)
except Exception as err:
logger.debug("-Error: %s" % str(err))
Ganga.Utility.logging.log_unknown_exception()
pass
# here we iterate over the lookup_input list and unprepare as
# necessary.
for item in lookup_input:
logger.info('Unpreparing objects referencing ShareDir %s' % item)
self.lookup(sharedir=item, unprepare=True)
# check to see that all sharedirs have an entry in the shareref. Otherwise, set their ref counter to 0
# so the user is made aware of them at shutdown
for this_dir in os.listdir(getSharedPath()):
if this_dir not in self.__getName().keys() and rmdir is False:
logger.debug(
"%s isn't referenced by a GangaObject in the Job or Box repository."
% this_dir)
self.__getName()[this_dir] = 0
elif this_dir not in self.__getName().keys() and rmdir is True:
logger.debug(
"%s isn't referenced by a GangaObject in the Job or Box repository. Removing directory."
% this_dir)
shutil.rmtree(os.path.join(getSharedPath(), this_dir))
self._setDirty()
self._releaseWriteAccess()
def lookup(self, sharedir, unprepare=False):
"""
Report Job, Box and Task repository items which reference a given ShareDir object.
The optional parameter 'unprepare=True' can be set to call the unprepare method
on the returned objects.
"""
from Ganga.GPI import jobs, box, tasks
objectlist = []
for thing in jobs.select():
objectlist.append({thing: 'job'})
for thing in box.select():
objectlist.append({thing: 'box'})
for thing in tasks.select():
objectlist.append({thing: 'task'})
run_unp = None
master_index = 0
for item in objectlist:
try:
item.keys()[0].is_prepared.name
if item.keys()[0].is_prepared.name == sharedir:
logger.info(
'ShareDir %s is referenced by item #%s in %s repository'
% (sharedir, stripProxy(
item.keys()[0])._registry_id, item.values()[0]))
run_unp = item.keys()[0]
master_index += 1
except AttributeError as err:
logger.debug("Err: %s" % str(err))
try:
item.keys()[0].application.is_prepared.name
if item.keys()[0].application.is_prepared.name == sharedir:
logger.info(
'ShareDir %s is referenced by item #%s in %s repository'
%
(sharedir, stripProxy(item.keys()[0])._registry_id,
item.values()[0]))
run_unp = item.keys()[0].application
master_index += 1
except AttributeError as err2:
logger.debug("Err2: %s" % str(err2))
try:
item.keys()[0].analysis.application.is_prepared.name
if item.keys(
)[0].analysis.application.is_prepared.name == sharedir:
logger.info(
'ShareDir %s is referenced by item #%s in %s repository'
% (sharedir, stripProxy(
item.keys()[0])._registry_id,
item.values()[0]))
run_unp = item.keys()[0].analysis.application
master_index += 1
except AttributeError as err3:
logger.debug("Err3: %s" % str(err3))
pass
if run_unp is not None and unprepare is True:
logger.info(
'Unpreparing %s repository object #%s associated with ShareDir %s'
% (item.values()[0], stripProxy(
item.keys()[0])._registry_id, sharedir))
# stripProxy(item.keys()[0]).unprepare()
run_unp.unprepare()
run_unp = None
if unprepare is not True:
logger.info('%s item(s) found referencing ShareDir %s',
master_index, sharedir)