def to_relative(this_object):
from Ganga.GPIDev.Lib.File import getSharedPath
logger.info('Absolute ShareDir().name attribute found in Job #%s',
this_object.id)
logger.info(
'Converting to relative path and moving associated directory if it exists.'
)
try:
shutil.move(
this_object.is_prepared.name,
os.path.join(getSharedPath(),
os.path.basename(this_object.is_prepared.name)))
except OSError as err:
Ganga.Utility.logging.log_unknown_exception()
logger.warn(
'Unable to move directory %s to %s',
this_object.is_prepared.name,
os.path.join(getSharedPath(),
os.path.basename(this_object.is_prepared.name)))
try:
stripProxy(this_object).is_prepared.name = os.path.basename(
this_object.is_prepared.name)
except Exception as err:
logger.debug("rebuild Error: %s" % err)
Ganga.Utility.logging.log_unknown_exception()
logger.warn(
"Unable to convert object's is_prepared.name attribute to a relative path"
)
def add(self, input):
from Ganga.Core.GangaRepository import getRegistry
if not isType(input, list):
input = [input]
for item in input:
if isType(item, str):
if os.path.isfile(expandfilename(item)):
logger.info('Copying file %s to shared directory %s' %
(item, self.name))
shutil.copy2(expandfilename(item),
os.path.join(getSharedPath(), self.name))
shareref = getRegistry("prep").getShareRef()
shareref.increase(self.name)
shareref.decrease(self.name)
else:
logger.error('File %s not found' % expandfilename(item))
elif isType(item, File) and item.name is not '' and os.path.isfile(
expandfilename(item.name)):
logger.info('Copying file object %s to shared directory %s' %
(item.name, self.name))
shutil.copy2(expandfilename(item.name),
os.path.join(getSharedPath(), self.name))
shareref = getRegistry("prep").getShareRef()
shareref.increase(self.name)
shareref.decrease(self.name)
else:
logger.error('File %s not found' % expandfilename(item.name))
def _display(self, interactive=0):
"""Prints content of the shareref metadata in a well formatted way.
"""
if len(self.__getName().keys()) > 0:
from Ganga.GPIDev.Lib.File import getSharedPath
fstring = " %48s | %20s | %15s"
disp_string = fstring % (
"Shared directory", "Date created", "Reference count\n")
# print fstring % (" ", " "," ")
disp_string += fstring % ("------------------------------------------------",
"--------------------", "---------------\n")
zero_ref = False
unsorted = []
all_elements = copy.deepcopy(self.__getName())
for element in all_elements:
full_shareddir_path = os.path.join(getSharedPath(), os.path.basename(element))
if os.path.isdir(full_shareddir_path):
unsorted.append(shareref_data(os.path.basename(element), int(
os.path.getctime(full_shareddir_path)), self.__getName()[element]))
else:
unsorted.append(shareref_data(os.path.basename(element), "Directory not found", self.__getName()[element]))
decorated = sorted((name.date, i, name) for i, name in enumerate(unsorted))
sorted_refs = [name for date, i, name in decorated]
for line in sorted_refs:
if isinstance(line.date, int):
tmp_string = fstring % (os.path.basename(line.name),
time.strftime(
"%d %b %Y %H:%M:%S", time.localtime(line.date)),
line.counter)
else:
tmp_string = fstring % (os.path.basename(line.name),
line.date,
line.counter)
if (line.counter == 0) or (isinstance(line.date, str)):
from Ganga.Utility.ColourText import ANSIMarkup, NoMarkup, Foreground, Background, Effects
fg = Foreground()
fg = Background()
fx = Effects()
if interactive:
m = ANSIMarkup()
else:
m = NoMarkup()
disp_string += fg.red + tmp_string + fx.normal + '\n'
#disp_string += m(tmp_string,code=fg.red)
if (line.counter == 0):
zero_ref = True
else:
disp_string += tmp_string + '\n'
disp_string += "\nThe shared directory repository is rooted at " + \
getSharedPath() + "\n"
if zero_ref:
disp_string += "\nShared directories with a zero reference count will be removed when Ganga exits.\n"
else:
disp_string = "No objects stored in the shared directory."
return disp_string
def decrease(self, shareddir, remove=0):
"""Reduce the reference counter for a given shared directory by 1. If the current value
of the counter is 0, the shared object will be removed from the metadata, and the files within
the shared object directory deleted when Ganga exits. If the optional remove parameter is specified
the shared directory is removed from the table.
Args:
shareddir (ShareDir): This is the shared directory object to reduce the counter for
remove (int): Effectively used as a bool. Should the directory be removed when the count reaches 0
"""
self._getSessionLock()
from Ganga.GPIDev.Lib.File import getSharedPath
shareddirname = os.path.join(getSharedPath(), os.path.basename(shareddir.name))
basedir = os.path.basename(shareddirname)
# if remove==1, we force the shareref counter to 0
try:
if self.__getName()[basedir] > 0:
if remove == 1:
self.__getName()[basedir] = 0
else:
self.__getName()[basedir] -= 1
if self.__getName()[basedir] is 0:
# shutil.rmtree(shareddir, ignore_errors=True)
shareddir.remove()
logger.info("Removed: %s" % shareddir.name)
# if we try to decrease a shareref that doesn't exist, we just set the
# corresponding shareref to 0
except KeyError as err:
logger.debug("KeyError: %s" % err)
self.__getName()[basedir] = 0
self.cleanUpOrphans([basedir,])
self._setDirty()
self._releaseSessionLockAndFlush()
def increase(self, shareddir, force=False):
"""Increase the reference counter for a given shared directory by 1. If the directory
doesn't currently have a reference counter, one is initialised with a value of 1.
The shareddir must exist for a reference counter to be initialised (unless Force=True).
Sharedir should be given relative to the user's shared directory repository, which can
be discovered by calling 'shareref'.
"""
logger.debug("running increase() in prepregistry")
self._getWriteAccess()
from Ganga.GPIDev.Lib.File import getSharedPath
shareddir = os.path.join(getSharedPath(), os.path.basename(shareddir))
basedir = os.path.basename(shareddir)
if os.path.isdir(shareddir) and force is False:
if basedir not in self.__getName().keys():
logger.debug('%s is not stored in the shareref metadata object...adding.' % basedir)
self.__getName()[basedir] = 1
else:
self.__getName()[basedir] += 1
elif not os.path.isdir(shareddir) and force is True and basedir is not '':
if basedir not in self.__getName().keys():
logger.debug('%s is not stored in the shareref metadata object...adding.' % basedir)
self.__getName()[basedir] = 0
else:
self.__getName()[basedir] += 1
else:
logger.error('Directory %s does not exist' % shareddir)
self._setDirty()
self._releaseWriteAccess()
def increase(self, shareddir, force=False):
"""Increase the reference counter for a given shared directory by 1. If the directory
doesn't currently have a reference counter, one is initialised with a value of 1.
The shareddir must exist for a reference counter to be initialised (unless Force=True).
Sharedir should be given relative to the user's shared directory repository, which can
be discovered by calling 'shareref'.
"""
logger.debug("running increase() in prepregistry")
self._getSessionLock()
from Ganga.GPIDev.Lib.File import getSharedPath
shareddir = os.path.join(getSharedPath(), os.path.basename(shareddir))
basedir = os.path.basename(shareddir)
if os.path.isdir(shareddir) and force is False:
if basedir not in self.__getName():
logger.debug(
'%s is not stored in the shareref metadata object...adding.'
% basedir)
self.__getName()[basedir] = 1
else:
self.__getName()[basedir] += 1
elif not os.path.isdir(
shareddir) and force is True and basedir is not '':
if basedir not in self.__getName():
logger.debug(
'%s is not stored in the shareref metadata object...adding.'
% basedir)
self.__getName()[basedir] = 0
else:
self.__getName()[basedir] += 1
else:
logger.error('Directory %s does not exist' % shareddir)
self._setDirty()
self._releaseSessionLockAndFlush()
def decrease(self, shareddir, remove=0):
"""Reduce the reference counter for a given shared directory by 1. If the current value
of the counter is 0, the shared object will be removed from the metadata, and the files within
the shared object directory deleted when Ganga exits. If the optional remove parameter is specified
the shared directory is removed from the table.
"""
self._getSessionLock()
from Ganga.GPIDev.Lib.File import getSharedPath
shareddir = os.path.join(getSharedPath(), os.path.basename(shareddir))
basedir = os.path.basename(shareddir)
# if remove==1, we force the shareref counter to 0
try:
if self.__getName()[basedir] > 0:
if remove == 1:
self.__getName()[basedir] = 0
else:
self.__getName()[basedir] -= 1
# if we try to decrease a shareref that doesn't exist, we just set the
# corresponding shareref to 0
except KeyError as err:
logger.debug("KeyError: %s" % err)
self.__getName()[basedir] = 0
self._setDirty()
self._releaseSessionLockAndFlush()
def counterVal(self, shareddir):
"""Return the current counter value for the named shareddir"""
from Ganga.GPIDev.Lib.File import getSharedPath
shareddir = os.path.join(getSharedPath(),
os.path.basename(shareddir.name))
basedir = os.path.basename(shareddir.name)
return self.__getName()[basedir]
def decrease(self, shareddir, remove=0):
"""Reduce the reference counter for a given shared directory by 1. If the current value
of the counter is 0, the shared object will be removed from the metadata, and the files within
the shared object directory deleted when Ganga exits. If the optional remove parameter is specified
the shared directory is removed from the table.
"""
self._getWriteAccess()
from Ganga.GPIDev.Lib.File import getSharedPath
shareddir = os.path.join(getSharedPath(), os.path.basename(shareddir))
basedir = os.path.basename(shareddir)
# if remove==1, we force the shareref counter to 0
try:
if self.__getName()[basedir] > 0:
if remove == 1:
self.__getName()[basedir] = 0
else:
self.__getName()[basedir] -= 1
# if we try to decrease a shareref that doesn't exist, we just set the
# corresponding shareref to 0
except KeyError as err:
logger.debug("KeyError: %s" % str(err))
self.__getName()[basedir] = 0
self._setDirty()
self._releaseWriteAccess()
def to_relative(object):
logger.info(
'Absolute ShareDir().name attribute found in Job #%s', object.id)
logger.info(
'Converting to relative path and moving associated directory if it exists.')
try:
shutil.move(object.is_prepared.name,
os.path.join(getSharedPath(), os.path.basename(object.is_prepared.name)))
except:
Ganga.Utility.logging.log_unknown_exception()
logger.warn('Unable to move directory %s to %s', object.is_prepared.name,
os.path.join(getSharedPath(), os.path.basename(object.is_prepared.name)))
try:
stripProxy(object).is_prepared.name = os.path.basename(
object.is_prepared.name)
except:
Ganga.Utility.logging.log_unknown_exception()
logger.warn(
"Unable to convert object's is_prepared.name attribute to a relative path")
def __init__(self, name=None, subdir=os.curdir):
super(ShareDir, self).__init__()
self._setRegistry(None)
if not name is None:
self.name = name
else:
# continue generating directory names until we create a unique one
# (which will likely be on the first attempt).
while True:
name = 'conf-{0}'.format(uuid.uuid4())
if not os.path.isdir(os.path.join(getSharedPath(), name)):
os.makedirs(os.path.join(getSharedPath(), name))
if not os.path.isdir(os.path.join(getSharedPath(), name)):
logger.error("ERROR creating path: %s" %
os.path.join(getSharedPath(), name))
raise GangaException("ShareDir ERROR")
else:
break
self.name = str(name)
def __init__(self, name=None, subdir=os.curdir):
super(ShareDir, self).__init__()
self._setRegistry(None)
if not name is None:
self.name = name
else:
# continue generating directory names until we create a unique one
# (which will likely be on the first attempt).
while True:
name = 'conf-{0}'.format(uuid.uuid4())
if not os.path.isdir(os.path.join(getSharedPath(), name)):
os.makedirs(os.path.join(getSharedPath(), name))
if not os.path.isdir(os.path.join(getSharedPath(), name)):
logger.error("ERROR creating path: %s" %
os.path.join(getSharedPath(), name))
raise GangaException("ShareDir ERROR")
else:
break
self.name = str(name)
def add(self, input):
from Ganga.Core.GangaRepository import getRegistry
if not isType(input, list):
input = [input]
for item in input:
if isType(item, str):
if os.path.isfile(expandfilename(item)):
logger.info('Copying file %s to shared directory %s' % (item, self.name))
shutil.copy2(expandfilename(item), os.path.join(getSharedPath(), self.name))
shareref = getRegistry("prep").getShareRef()
shareref.increase(self.name)
shareref.decrease(self.name)
else:
logger.error('File %s not found' % expandfilename(item))
elif isType(item, File) and item.name is not '' and os.path.isfile(expandfilename(item.name)):
logger.info('Copying file object %s to shared directory %s' % (item.name, self.name))
shutil.copy2(expandfilename(item.name), os.path.join(getSharedPath(), self.name))
shareref = getRegistry("prep").getShareRef()
shareref.increase(self.name)
shareref.decrease(self.name)
else:
logger.error('File %s not found' % expandfilename(item.name))
def helper(this_object, unp=True, numsubjobs=0):
shareddir = os.path.join(getSharedPath(), os.path.basename(this_object))
logger.debug('Adding %s to the shareref table.' % shareddir)
if os.path.basename(this_object) in self.name:
self.name[os.path.basename(this_object)] += 1
else:
self.name[os.path.basename(this_object)] = 1
if numsubjobs > 0:
self.name[os.path.basename(this_object)] += numsubjobs
if not os.path.isdir(shareddir) and os.path.basename(this_object) not in lookup_input:
logger.info('Shared directory %s not found on disk.' % shareddir)
if unp == True:
lookup_input.append(os.path.basename(this_object))
def cleanUpOrphans(self, orphans=None):
"""
This cleans up the orphan share dir objects on shutdown
Args:
orphans (list): An optional list of the orphans to remove from the list
"""
if orphans:
to_remove = orphans
else:
to_remove = self.removal_list
for shareddir in to_remove:
if os.path.exists(os.path.join(getSharedPath(), shareddir)):
try:
shutil.rmtree(os.path.join(getSharedPath(), shareddir))
except:
logger.error("Failed to remove Orphaned Shared Dir: %s" % shareddir)
for shareddir in to_remove:
if shareddir in self.removal_list:
self.removal_list.remove(shareddir)
def _getName(self):
"""
A getter method for the 'name' schema attribute which will trigger the creation of a SharedDir on disk only when information about it is asked
"""
share_dir = os.path.join(getSharedPath(), self._name)
if not os.path.isdir(share_dir):
logger.debug("Actually creating: %s" % share_dir)
os.makedirs(share_dir)
if not os.path.isdir(share_dir):
logger.error("ERROR creating path: %s" % share_dir)
raise GangaException("ShareDir ERROR")
return self._name
def _getName(self):
"""
A getter method for the 'name' schema attribute which will trigger the creation of a SharedDir on disk only when information about it is asked
"""
share_dir = os.path.join(getSharedPath(), self._name)
if not os.path.isdir(share_dir):
logger.debug("Actually creating: %s" % share_dir)
os.makedirs(share_dir)
if not os.path.isdir(share_dir):
logger.error("ERROR creating path: %s" % share_dir)
raise GangaException("ShareDir ERROR")
return self._name
def helper(self, this_object, unp=True, numsubjobs=0):
from Ganga.GPIDev.Lib.File import getSharedPath
shareddir = os.path.join(getSharedPath(), os.path.basename(this_object))
logger.debug('Adding %s to the shareref table.' % shareddir)
if os.path.basename(this_object) in self.__getName().keys():
self.__getName()[os.path.basename(this_object)] += 1
else:
self.__getName()[os.path.basename(this_object)] = 1
if numsubjobs > 0:
self.__getName()[os.path.basename(this_object)] += numsubjobs
if not os.path.isdir(shareddir) and os.path.basename(this_object) not in lookup_input:
logger.info('Shared directory %s not found on disk.' % shareddir)
if unp == True:
lookup_input.append(os.path.basename(this_object))
def cleanUpOrphans(self, orphans=None):
"""
This cleans up the orphan share dir objects on shutdown
Args:
orphans (list): An optional list of the orphans to remove from the list
"""
if orphans:
to_remove = orphans
else:
to_remove = self.removal_list
for shareddir in to_remove:
if os.path.exists(os.path.join(getSharedPath(), shareddir)):
try:
shutil.rmtree(os.path.join(getSharedPath(), shareddir))
except:
logger.error("Failed to remove Orphaned Shared Dir: %s" %
shareddir)
for shareddir in to_remove:
if shareddir in self.removal_list:
self.removal_list.remove(shareddir)
def helper(self, this_object, unp=True, numsubjobs=0):
from Ganga.GPIDev.Lib.File import getSharedPath
shareddir = os.path.join(getSharedPath(),
os.path.basename(this_object))
logger.debug('Adding %s to the shareref table.' % shareddir)
if os.path.basename(this_object) in self.__getName():
self.__getName()[os.path.basename(this_object)] += 1
else:
self.__getName()[os.path.basename(this_object)] = 1
if numsubjobs > 0:
self.__getName()[os.path.basename(this_object)] += numsubjobs
if not os.path.isdir(shareddir) and os.path.basename(
this_object) not in lookup_input:
logger.info('Shared directory %s not found on disk.' % shareddir)
if unp == True:
lookup_input.append(os.path.basename(this_object))
def to_relative(this_object):
from Ganga.GPIDev.Lib.File import getSharedPath
logger.info('Absolute ShareDir().name attribute found in Job #%s', this_object.id)
logger.info('Converting to relative path and moving associated directory if it exists.')
try:
shutil.move(this_object.is_prepared.name, os.path.join(getSharedPath(), os.path.basename(this_object.is_prepared.name)))
except OSError as err:
Ganga.Utility.logging.log_unknown_exception()
logger.warn('Unable to move directory %s to %s', this_object.is_prepared.name, os.path.join(getSharedPath(), os.path.basename(this_object.is_prepared.name)))
try:
stripProxy(this_object).is_prepared.name = os.path.basename(this_object.is_prepared.name)
except Exception as err:
logger.debug("rebuild Error: %s" % str(err))
Ganga.Utility.logging.log_unknown_exception()
logger.warn("Unable to convert object's is_prepared.name attribute to a relative path")
def cleanUpShareDirs():
"""Function to be used to clean up erronious empty folders in the Shared directory"""
share_path = getSharedPath()
logger.info("Cleaning Shared folders in: %s" % share_path)
logger.info("This may take a few minutes if you're running this for the first time after 6.1.23, feel free to go grab a tea/coffee")
for item in os.listdir(share_path):
this_dir = os.path.join(share_path, item)
if os.path.isdir(this_dir):
# NB we do need to explicitly test the length of the returned value here
# Checking is __MUCH__ faster than trying and failing to remove folders with contents on AFS
if len(os.listdir(this_dir)) == 0:
try:
os.rmdir(this_dir)
except OSError:
logger.debug("Failed to remove: %s" % this_dir)
def ls(self):
"""
Print the contents of the ShareDir
"""
full_shareddir_path = os.path.join(getSharedPath(), self.name)
try:
os.path.isdir(full_shareddir_path)
cmd = "find '%s'" % (full_shareddir_path)
files = os.popen(cmd).read().strip().split('\n')
padding = '| '
for file in files:
level = file.count(os.sep)
level = level - 6
pieces = file.split(os.sep)
symbol = {0: '', 1: '/'}[os.path.isdir(file)]
logger.info(padding * level + pieces[-1] + symbol)
except IOError:
logger.warn('ShareDir %s not found on storage' %
full_shareddir_path)
def cleanUpShareDirs():
"""Function to be used to clean up erronious empty folders in the Shared directory"""
share_path = getSharedPath()
logger.info("Cleaning Shared folders in: %s" % share_path)
logger.info(
"This may take a few minutes if you're running this for the first time after 6.1.23, feel free to go grab a tea/coffee"
)
for item in os.listdir(share_path):
this_dir = os.path.join(share_path, item)
if os.path.isdir(this_dir):
# NB we do need to explicitly test the length of the returned value here
# Checking is __MUCH__ faster than trying and failing to remove folders with contents on AFS
if len(os.listdir(this_dir)) == 0:
try:
os.rmdir(this_dir)
except OSError:
logger.debug("Failed to remove: %s" % this_dir)
def ls(self):
"""
Print the contents of the ShareDir
"""
full_shareddir_path = os.path.join(getSharedPath(), self.name)
try:
os.path.isdir(full_shareddir_path)
cmd = "find '%s'" % (full_shareddir_path)
files = os.popen(cmd).read().strip().split('\n')
padding = '| '
for file in files:
level = file.count(os.sep)
level = level - 6
pieces = file.split(os.sep)
symbol = {0: '', 1: '/'}[os.path.isdir(file)]
logger.info(padding * level + pieces[-1] + symbol)
except IOError:
logger.warn('ShareDir %s not found on storage' %
full_shareddir_path)
def ls(self, shareddir, print_files=True):
"""
Print the contents of the given shared directory, which can be specified as relative to the shared
directories repository, or absolute.
"""
shareddir_root = shareddir
full_shareddir_path = os.path.join(getSharedPath(), os.path.basename(shareddir))
if os.path.isdir(full_shareddir_path):
cmd = "find '%s'" % (full_shareddir_path)
files = os.popen(cmd).read().strip().split('\n')
padding = '| '
for file in files:
level = file.count(os.sep)
level = level - 6
pieces = file.split(os.sep)
symbol = {0: '', 1: '/'}[os.path.isdir(file)]
if not print_files and symbol != '/':
continue
logger.info(padding * level + pieces[-1] + symbol)
def ls(self, shareddir, print_files=True):
"""
Print the contents of the given shared directory, which can be specified as relative to the shared
directories repository, or absolute.
"""
shareddir_root = shareddir
full_shareddir_path = os.path.join(getSharedPath(),
os.path.basename(shareddir))
if os.path.isdir(full_shareddir_path):
cmd = "find '%s'" % (full_shareddir_path)
files = os.popen(cmd).read().strip().split('\n')
padding = '| '
for file in files:
level = file.count(os.sep)
level = level - 6
pieces = file.split(os.sep)
symbol = {0: '', 1: '/'}[os.path.isdir(file)]
if not print_files and symbol != '/':
continue
logger.info(padding * level + pieces[-1] + symbol)
def decrease(self, shareddir, remove=0):
"""Reduce the reference counter for a given shared directory by 1. If the current value
of the counter is 0, the shared object will be removed from the metadata, and the files within
the shared object directory deleted when Ganga exits. If the optional remove parameter is specified
the shared directory is removed from the table.
Args:
shareddir (ShareDir): This is the shared directory object to reduce the counter for
remove (int): Effectively used as a bool. Should the directory be removed when the count reaches 0
"""
self._getSessionLock()
from Ganga.GPIDev.Lib.File import getSharedPath
shareddirname = os.path.join(getSharedPath(),
os.path.basename(shareddir.name))
basedir = os.path.basename(shareddirname)
# if remove==1, we force the shareref counter to 0
try:
if self.__getName()[basedir] > 0:
if remove == 1:
self.__getName()[basedir] = 0
else:
self.__getName()[basedir] -= 1
if self.__getName()[basedir] is 0:
# shutil.rmtree(shareddir, ignore_errors=True)
shareddir.remove()
logger.info("Removed: %s" % shareddir.name)
# if we try to decrease a shareref that doesn't exist, we just set the
# corresponding shareref to 0
except KeyError as err:
logger.debug("KeyError: %s" % err)
self.__getName()[basedir] = 0
self.cleanUpOrphans([
basedir,
])
self._setDirty()
self._releaseSessionLockAndFlush()
def counterVal(self, shareddir):
"""Return the current counter value for the named shareddir"""
from Ganga.GPIDev.Lib.File import getSharedPath
shareddir = os.path.join(getSharedPath(), os.path.basename(shareddir.name))
basedir = os.path.basename(shareddir.name)
return self.__getName()[basedir]
def rebuild(self, unprepare=True, rmdir=False):
"""Rebuild the shareref table.
Clears the shareref table and then rebuilds it by iterating over all Ganga Objects
in the Job, Box and Task repositories. If an object has a ShareDir associated with it,
that ShareDir is added into the shareref table (the reference counter being incremented
accordingly). If called with the optional parameter 'unprepare=False', objects whose
ShareDirs are not present on disk will not be unprepared. Note that the default behaviour
is unprepare=True, i.e. the job/application would be unprepared.
After all Job/Box/Task objects have been checked, the inverse operation is performed,
i.e., for each directory in the ShareDir repository, a check is done to ensure there
is a matching entry in the shareref table. If not, and the optional parameter
'rmdir=True' is set, then the (orphaned) ShareDir will removed from the filesystem.
Otherwise, it will be added to the shareref table with a reference count of zero;
this results in the directory being deleted upon Ganga exit.
"""
self._getWriteAccess()
from Ganga.GPI import jobs, box, tasks
# clear the shareref table
self.name = {}
lookup_input = []
def helper(object, unp=True, numsubjobs=0):
shareddir = os.path.join(getSharedPath(), os.path.basename(object))
logger.debug('Adding %s to the shareref table.' % shareddir)
if os.path.basename(object) in self.name:
self.name[os.path.basename(object)] += 1
else:
self.name[os.path.basename(object)] = 1
if numsubjobs > 0:
self.name[os.path.basename(object)] += numsubjobs
if not os.path.isdir(shareddir) and os.path.basename(object) not in lookup_input:
logger.info(
'Shared directory %s not found on disk.' % shareddir)
if unp == True:
lookup_input.append(os.path.basename(object))
def to_relative(object):
logger.info(
'Absolute ShareDir().name attribute found in Job #%s', object.id)
logger.info(
'Converting to relative path and moving associated directory if it exists.')
try:
shutil.move(object.is_prepared.name,
os.path.join(getSharedPath(), os.path.basename(object.is_prepared.name)))
except:
Ganga.Utility.logging.log_unknown_exception()
logger.warn('Unable to move directory %s to %s', object.is_prepared.name,
os.path.join(getSharedPath(), os.path.basename(object.is_prepared.name)))
try:
stripProxy(object).is_prepared.name = os.path.basename(
object.is_prepared.name)
except:
Ganga.Utility.logging.log_unknown_exception()
logger.warn(
"Unable to convert object's is_prepared.name attribute to a relative path")
objectlist = []
for thing in jobs.select():
objectlist.append({thing: 'job'})
for thing in box.select():
objectlist.append({thing: 'box'})
for thing in tasks.select():
objectlist.append({thing: 'task'})
for item in objectlist:
shortname = None
try:
shortname = item.keys()[0].is_prepared.name
except AttributeError:
try:
shortname = item.keys()[0].application.is_prepared.name
except AttributeError:
try:
shortname = item.keys()[
0].analysis.application.is_prepared.name
except AttributeError:
pass
try:
if shortname is not None and shortname is not True:
if os.path.basename(shortname) != shortname:
to_relative(item.keys()[0].is_prepared)
try:
numsubjobs = len(item.keys()[0].subjobs.ids())
except:
Ganga.Utility.logging.log_unknown_exception()
numsubjobs = 0
helper(shortname, unp=unprepare, numsubjobs=numsubjobs)
except:
Ganga.Utility.logging.log_unknown_exception()
pass
# here we iterate over the lookup_input list and unprepare as
# necessary.
for item in lookup_input:
logger.info('Unpreparing objects referencing ShareDir %s' % item)
self.lookup(sharedir=item, unprepare=True)
# check to see that all sharedirs have an entry in the shareref. Otherwise, set their ref counter to 0
# so the user is made aware of them at shutdown
for dir in os.listdir(getSharedDir()):
if dir not in self.name and rmdir is False:
logger.debug(
"%s isn't referenced by a GangaObject in the Job or Box repository." % dir)
self.name[dir] = 0
elif dir not in self.name and rmdir is True:
logger.debug(
"%s isn't referenced by a GangaObject in the Job or Box repository. Removing directory." % dir)
shutil.rmtree(os.path.join(getSharedPath(), dir))
self._setDirty()
self._releaseWriteAccess()
def rebuild(self, unprepare=True, rmdir=False):
"""Rebuild the shareref table.
Clears the shareref table and then rebuilds it by iterating over all Ganga Objects
in the Job, Box and Task repositories. If an object has a ShareDir associated with it,
that ShareDir is added into the shareref table (the reference counter being incremented
accordingly). If called with the optional parameter 'unprepare=False', objects whose
ShareDirs are not present on disk will not be unprepared. Note that the default behaviour
is unprepare=True, i.e. the job/application would be unprepared.
After all Job/Box/Task objects have been checked, the inverse operation is performed,
i.e., for each directory in the ShareDir repository, a check is done to ensure there
is a matching entry in the shareref table. If not, and the optional parameter
'rmdir=True' is set, then the (orphaned) ShareDir will removed from the filesystem.
Otherwise, it will be added to the shareref table with a reference count of zero;
this results in the directory being deleted upon Ganga exit.
"""
self._getWriteAccess()
from Ganga.GPI import jobs, box, tasks
# clear the shareref table
self.name = {}
lookup_input = []
from Ganga.GPIDev.Lib.File import getSharedPath
objectlist = []
for thing in jobs.select():
objectlist.append({thing: 'job'})
for thing in box.select():
objectlist.append({thing: 'box'})
for thing in tasks.select():
objectlist.append({thing: 'task'})
for item in objectlist:
shortname = None
try:
shortname = item.keys()[0].is_prepared.name
except AttributeError as err:
logger.debug("Err: %s" % str(err))
try:
shortname = item.keys()[0].application.is_prepared.name
except AttributeError as err2:
logger.debug("Err2: %s" % str(err2))
try:
shortname = item.keys()[0].analysis.application.is_prepared.name
except AttributeError as err3:
logger.debug("Err3: %s" % str(err3))
pass
try:
if shortname is not None and shortname is not True:
if os.path.basename(shortname) != shortname:
self.to_relative(item.keys()[0].is_prepared)
try:
numsubjobs = len(item.keys()[0].subjobs.ids())
except Exception as err:
logger.debug("Path Error: %s" % str(err))
Ganga.Utility.logging.log_unknown_exception()
numsubjobs = 0
self.helper(shortname, unp=unprepare, numsubjobs=numsubjobs)
except Exception as err:
logger.debug("-Error: %s" % str(err))
Ganga.Utility.logging.log_unknown_exception()
pass
# here we iterate over the lookup_input list and unprepare as
# necessary.
for item in lookup_input:
logger.info('Unpreparing objects referencing ShareDir %s' % item)
self.lookup(sharedir=item, unprepare=True)
# check to see that all sharedirs have an entry in the shareref. Otherwise, set their ref counter to 0
# so the user is made aware of them at shutdown
for this_dir in os.listdir(getSharedPath()):
if this_dir not in self.__getName().keys() and rmdir is False:
logger.debug("%s isn't referenced by a GangaObject in the Job or Box repository." % this_dir)
self.__getName()[this_dir] = 0
elif this_dir not in self.__getName().keys() and rmdir is True:
logger.debug("%s isn't referenced by a GangaObject in the Job or Box repository. Removing directory." % this_dir)
shutil.rmtree(os.path.join(getSharedPath(), this_dir))
self._setDirty()
self._releaseWriteAccess()
def _display(self, interactive=0):
"""Prints content of the shareref metadata in a well formatted way.
"""
if len(self.__getName()) > 0:
from Ganga.GPIDev.Lib.File import getSharedPath
fstring = " %48s | %20s | %15s"
disp_string = fstring % ("Shared directory", "Date created",
"Reference count\n")
# print fstring % (" ", " "," ")
disp_string += fstring % (
"------------------------------------------------",
"--------------------", "---------------\n")
zero_ref = False
unsorted = []
all_elements = copy.deepcopy(self.__getName())
for element in all_elements:
full_shareddir_path = os.path.join(getSharedPath(),
os.path.basename(element))
if os.path.isdir(full_shareddir_path):
unsorted.append(
shareref_data(
os.path.basename(element),
int(os.path.getctime(full_shareddir_path)),
self.__getName()[element]))
else:
unsorted.append(
shareref_data(os.path.basename(element),
"Directory not found",
self.__getName()[element]))
decorated = sorted(
(name.date, i, name) for i, name in enumerate(unsorted))
sorted_refs = [name for date, i, name in decorated]
for line in sorted_refs:
if isinstance(line.date, int):
tmp_string = fstring % (
os.path.basename(line.name),
time.strftime("%d %b %Y %H:%M:%S",
time.localtime(line.date)), line.counter)
else:
tmp_string = fstring % (os.path.basename(
line.name), line.date, line.counter)
if (line.counter == 0) or (isinstance(line.date, str)):
from Ganga.Utility.ColourText import ANSIMarkup, NoMarkup, Foreground, Background, Effects
fg = Foreground()
fg = Background()
fx = Effects()
if interactive:
m = ANSIMarkup()
else:
m = NoMarkup()
disp_string += fg.red + tmp_string + fx.normal + '\n'
#disp_string += m(tmp_string,code=fg.red)
if (line.counter == 0):
zero_ref = True
else:
disp_string += tmp_string + '\n'
disp_string += "\nThe shared directory repository is rooted at " + \
getSharedPath() + "\n"
if zero_ref:
disp_string += "\nShared directories with a zero reference count will be removed when Ganga exits.\n"
else:
disp_string = "No objects stored in the shared directory."
return disp_string
def closedown(self):
"""Cleans up the Shared Directory registry upon shutdown of the registry, ie. when exiting a Ganga session."""
#stripProxy(self)._getRegistry()._hasStarted = True
from Ganga.GPIDev.Lib.File import getSharedPath
delete_share_config = Ganga.Utility.Config.getConfig(
'Configuration')['deleteUnusedShareDir']
if delete_share_config == 'ask':
ask_delete = 'Ask'
default = 'none'
elif delete_share_config == 'never':
ask_delete = 'none'
elif delete_share_config == 'always':
ask_delete = 'all'
else:
ask_delete = 'Ask'
default = 'none'
# list of keys to be removed from the shareref table
cleanup_list = []
try:
## FIXME. this triggers maximum recusion depth bug on shutdown in some situations! rcurrie
all_dirs = copy.deepcopy(self.__getName().keys())
except:
all_dirs = {}
for shareddir in all_dirs:
full_shareddir_path = os.path.join(getSharedPath(), shareddir)
# for each sharedir in the shareref table that also exists in the
# filesystem
if self.__getName()[shareddir] == 0 and os.path.isdir(
full_shareddir_path):
if ask_delete == 'Ask':
ask_delete = self.yes_no('',
default=default,
shareddir=shareddir)
if ask_delete == 'yes':
shutil.rmtree(full_shareddir_path)
if shareddir not in cleanup_list:
cleanup_list.append(shareddir)
ask_delete = 'Ask'
default = 'yes'
logger.debug(
'Deleting Sharedir %s because it is not referenced by a persisted Ganga object',
shareddir)
if ask_delete == 'no':
ask_delete = 'Ask'
default = 'no'
logger.debug(
'Keeping ShareDir %s even though it is not referenced by a persisted Ganga object',
shareddir)
if ask_delete == 'all':
shutil.rmtree(full_shareddir_path)
if shareddir not in cleanup_list:
cleanup_list.append(shareddir)
logger.debug(
'Deleting Sharedir %s because it is not referenced by a persisted Ganga object',
shareddir)
if ask_delete == 'none':
default = 'none'
logger.debug(
'Keeping ShareDir %s even though it is not referenced by a persisted Ganga object',
shareddir)
## DISABLED BY RCURRIE
# if the sharedir in the table doesn't exist on the filesytem, and the reference counter is > 0,
# we need to unprepare any associated jobs
#logger.debug("Examining: %s" % full_shareddir_path)
#logger.debug("shareddir: %s" % shareddir)
#logger.debug("cleanup_list: %s" % cleanup_list)
if not os.path.isdir(
full_shareddir_path) and shareddir not in cleanup_list:
# logger.info('%s not found on disk. Removing entry from shareref table and unpreparing any associated Ganga objects.' % shareddir)
# self.lookup(sharedir=shareddir, unprepare=True)
cleanup_list.append(shareddir)
## DISABLED BY RCURRIE
self._getSessionLock()
for element in cleanup_list:
del self.name[element]
allnames = copy.deepcopy(self.__getName())
for element in allnames:
del self.name[element]
self._setDirty()
def path(self):
"""Get the full path of the ShareDir location"""
return os.path.join(getSharedPath(), self.name)
def path(self):
"""Get the full path of the ShareDir location"""
return os.path.join(getSharedPath(), self._getName())
def closedown(self):
"""Cleans up the Shared Directory registry upon shutdown of the registry, ie. when exiting a Ganga session."""
def yes_no(question, default='none'):
"""Check whether the user wants to delete sharedirs which are no longer referenced by any Ganga object"""
valid = {"yes": "yes", "y": "yes", "no": "no",
"n": "no", "none": "none", "all": "all"}
if default == 'none':
prompt = '(Yes/No/All/[NONE])'
elif default == 'yes':
prompt = '([YES]/No/All/None)'
elif default == 'no':
prompt = '(Yes/[NO]/All/None)'
else:
raise ValueError("Invalid default answer: '%s'" % default)
while True:
logger.info(
'%s no longer being referenced by any objects. Delete directory?' % shareddir)
logger.info(question + prompt)
answer = raw_input().lower()
if answer == '':
return default
elif answer in valid.keys():
return valid[answer]
else:
logger.warn(
"Please respond with 'Yes/y', 'No/n', 'All' or 'None'")
delete_share_config = Ganga.Utility.Config.getConfig('Configuration')['deleteUnusedShareDir']
if delete_share_config == 'ask':
ask_delete = 'Ask'
default = 'none'
elif delete_share_config == 'never':
ask_delete = 'none'
elif delete_share_config == 'always':
ask_delete = 'all'
else:
ask_delete = 'Ask'
default = 'none'
# list of keys to be removed from the shareref table
cleanup_list = []
for shareddir in self.name.keys():
full_shareddir_path = os.path.join(getSharedPath(), shareddir)
# for each sharedir in the shareref table that also exists in the
# filesystem
if self.name[shareddir] == 0 and os.path.isdir(full_shareddir_path):
if ask_delete == 'Ask':
ask_delete = yes_no('', default=default)
if ask_delete == 'yes':
shutil.rmtree(full_shareddir_path)
if shareddir not in cleanup_list:
cleanup_list.append(shareddir)
ask_delete = 'Ask'
default = 'yes'
logger.debug(
'Deleting Sharedir %s because it is not referenced by a persisted Ganga object', shareddir)
if ask_delete == 'no':
ask_delete = 'Ask'
default = 'no'
logger.debug(
'Keeping ShareDir %s even though it is not referenced by a persisted Ganga object', shareddir)
if ask_delete == 'all':
shutil.rmtree(full_shareddir_path)
if shareddir not in cleanup_list:
cleanup_list.append(shareddir)
logger.debug(
'Deleting Sharedir %s because it is not referenced by a persisted Ganga object', shareddir)
if ask_delete == 'none':
default = 'none'
logger.debug(
'Keeping ShareDir %s even though it is not referenced by a persisted Ganga object', shareddir)
# if the sharedir in the table doesn't exist on the filesytem, and the reference counter is > 0,
# we need to unprepare any associated jobs
logger.debug("Examining: %s" % full_shareddir_path)
logger.debug("shareddir: %s" % str(shareddir))
logger.debug("cleanup_list: %s" % str(cleanup_list))
if not os.path.isdir(full_shareddir_path) and shareddir not in cleanup_list:
logger.info(
'%s not found on disk. Removing entry from shareref table and unpreparing any associated Ganga objects.' % shareddir)
self.lookup(sharedir=shareddir, unprepare=True)
cleanup_list.append(shareddir)
self._getWriteAccess()
for element in cleanup_list:
del self.name[element]
self._setDirty()
self._releaseWriteAccess()
def rebuild(self, unprepare=True, rmdir=False):
"""Rebuild the shareref table.
Clears the shareref table and then rebuilds it by iterating over all Ganga Objects
in the Job, Box and Task repositories. If an object has a ShareDir associated with it,
that ShareDir is added into the shareref table (the reference counter being incremented
accordingly). If called with the optional parameter 'unprepare=False', objects whose
ShareDirs are not present on disk will not be unprepared. Note that the default behaviour
is unprepare=True, i.e. the job/application would be unprepared.
After all Job/Box/Task objects have been checked, the inverse operation is performed,
i.e., for each directory in the ShareDir repository, a check is done to ensure there
is a matching entry in the shareref table. If not, and the optional parameter
'rmdir=True' is set, then the (orphaned) ShareDir will removed from the filesystem.
Otherwise, it will be added to the shareref table with a reference count of zero;
this results in the directory being deleted upon Ganga exit.
"""
self._getSessionLock()
# clear the shareref table
self.name = {}
lookup_input = []
from Ganga.GPIDev.Lib.File import getSharedPath
from Ganga.Core.GangaRepository import getRegistryProxy
objectlist = []
for thing in getRegistryProxy('jobs').select():
objectlist.append({thing: 'job'})
for thing in getRegistryProxy('box').select():
objectlist.append({thing: 'box'})
for thing in getRegistryProxy('tasks').select():
objectlist.append({thing: 'task'})
for item in objectlist:
shortname = None
try:
shortname = item.keys()[0].is_prepared.name
except AttributeError as err:
logger.debug("Err: %s" % err)
try:
shortname = item.keys()[0].application.is_prepared.name
except AttributeError as err2:
logger.debug("Err2: %s" % err2)
try:
shortname = item.keys(
)[0].analysis.application.is_prepared.name
except AttributeError as err3:
logger.debug("Err3: %s" % err3)
pass
try:
if shortname is not None and shortname is not True:
if os.path.basename(shortname) != shortname:
self.to_relative(item.keys()[0].is_prepared)
try:
numsubjobs = len(item.keys()[0].subjobs.ids())
except Exception as err:
logger.debug("Path Error: %s" % err)
Ganga.Utility.logging.log_unknown_exception()
numsubjobs = 0
self.helper(shortname,
unp=unprepare,
numsubjobs=numsubjobs)
except Exception as err:
logger.debug("-Error: %s" % err)
Ganga.Utility.logging.log_unknown_exception()
pass
# here we iterate over the lookup_input list and unprepare as
# necessary.
for item in lookup_input:
logger.info('Unpreparing objects referencing ShareDir %s' % item)
self.lookup(sharedir=item, unprepare=True)
# check to see that all sharedirs have an entry in the shareref. Otherwise, set their ref counter to 0
# so the user is made aware of them at shutdown
for this_dir in os.listdir(getSharedPath()):
if this_dir not in self.__getName().keys() and rmdir is False:
logger.debug(
"%s isn't referenced by a GangaObject in the Job or Box repository."
% this_dir)
self.__getName()[this_dir] = 0
elif this_dir not in self.__getName() and rmdir is True:
logger.debug(
"%s isn't referenced by a GangaObject in the Job or Box repository. Removing directory."
% this_dir)
shutil.rmtree(os.path.join(getSharedPath(), this_dir))
self._setDirty()
self._releaseSessionLockAndFlush()
def closedown(self):
"""Cleans up the Shared Directory registry upon shutdown of the registry, ie. when exiting a Ganga session."""
#stripProxy(self)._getRegistry()._hasStarted = True
from Ganga.GPIDev.Lib.File import getSharedPath
delete_share_config = Ganga.Utility.Config.getConfig('Configuration')['deleteUnusedShareDir']
if delete_share_config == 'ask':
ask_delete = 'Ask'
default = 'none'
elif delete_share_config == 'never':
ask_delete = 'none'
elif delete_share_config == 'always':
ask_delete = 'all'
else:
ask_delete = 'Ask'
default = 'none'
# list of keys to be removed from the shareref table
cleanup_list = []
try:
## FIXME. this triggers maximum recusion depth bug on shutdown in some situations! rcurrie
all_dirs = copy.deepcopy(self.__getName().keys())
except:
all_dirs = {}
for shareddir in all_dirs:
full_shareddir_path = os.path.join(getSharedPath(), shareddir)
# for each sharedir in the shareref table that also exists in the
# filesystem
if self.__getName()[shareddir] == 0 and os.path.isdir(full_shareddir_path):
if ask_delete == 'Ask':
ask_delete = self.yes_no('', default=default, shareddir=shareddir)
if ask_delete == 'yes':
shutil.rmtree(full_shareddir_path)
if shareddir not in cleanup_list:
cleanup_list.append(shareddir)
ask_delete = 'Ask'
default = 'yes'
logger.debug('Deleting Sharedir %s because it is not referenced by a persisted Ganga object', shareddir)
if ask_delete == 'no':
ask_delete = 'Ask'
default = 'no'
logger.debug('Keeping ShareDir %s even though it is not referenced by a persisted Ganga object', shareddir)
if ask_delete == 'all':
shutil.rmtree(full_shareddir_path)
if shareddir not in cleanup_list:
cleanup_list.append(shareddir)
logger.debug('Deleting Sharedir %s because it is not referenced by a persisted Ganga object', shareddir)
if ask_delete == 'none':
default = 'none'
logger.debug('Keeping ShareDir %s even though it is not referenced by a persisted Ganga object', shareddir)
## DISABLED BY RCURRIE
# if the sharedir in the table doesn't exist on the filesytem, and the reference counter is > 0,
# we need to unprepare any associated jobs
#logger.debug("Examining: %s" % full_shareddir_path)
#logger.debug("shareddir: %s" % str(shareddir))
#logger.debug("cleanup_list: %s" % str(cleanup_list))
if not os.path.isdir(full_shareddir_path) and shareddir not in cleanup_list:
# logger.info('%s not found on disk. Removing entry from shareref table and unpreparing any associated Ganga objects.' % shareddir)
# self.lookup(sharedir=shareddir, unprepare=True)
cleanup_list.append(shareddir)
## DISABLED BY RCURRIE
self._getWriteAccess()
for element in cleanup_list:
del self.name[element]
allnames = copy.deepcopy(self.__getName())
for element in allnames:
del self.name[element]
self._setDirty()
