def lookup(self, sharedir, unprepare=False):
"""
Report Job, Box and Task repository items which reference a given ShareDir object.
The optional parameter 'unprepare=True' can be set to call the unprepare method
on the returned objects.
"""
from Ganga.Core.GangaRepository import getRegistryProxy
objectlist = []
for thing in getRegistryProxy('jobs').select():
objectlist.append({thing: 'job'})
for thing in getRegistryProxy('box').select():
objectlist.append({thing: 'box'})
for thing in getRegistryProxy('tasks').select():
objectlist.append({thing: 'task'})
run_unp = None
master_index = 0
for item in objectlist:
try:
item.keys()[0].is_prepared.name
if item.keys()[0].is_prepared.name == sharedir:
logger.info('ShareDir %s is referenced by item #%s in %s repository' % (sharedir, stripProxy(item.keys()[0])._registry_id, item.values()[0]))
run_unp = item.keys()[0]
master_index += 1
except AttributeError as err:
logger.debug("Err: %s" % err)
try:
item.keys()[0].application.is_prepared.name
if item.keys()[0].application.is_prepared.name == sharedir:
logger.info('ShareDir %s is referenced by item #%s in %s repository' % (sharedir, stripProxy(item.keys()[0])._registry_id, item.values()[0]))
run_unp = item.keys()[0].application
master_index += 1
except AttributeError as err2:
logger.debug("Err2: %s" % err2)
try:
item.keys()[0].analysis.application.is_prepared.name
if item.keys()[0].analysis.application.is_prepared.name == sharedir:
logger.info('ShareDir %s is referenced by item #%s in %s repository' % (sharedir, stripProxy(item.keys()[0])._registry_id, item.values()[0]))
run_unp = item.keys()[0].analysis.application
master_index += 1
except AttributeError as err3:
logger.debug("Err3: %s" % err3)
pass
if run_unp is not None and unprepare is True:
logger.info('Unpreparing %s repository object #%s associated with ShareDir %s' % (item.values()[0], stripProxy(item.keys()[0])._registry_id, sharedir))
# stripProxy(item.keys()[0]).unprepare()
run_unp.unprepare()
run_unp = None
if unprepare is not True:
logger.info('%s item(s) found referencing ShareDir %s', master_index, sharedir)
def isCredentialRequired(credObj):
"""
The logic to decide if a given invalid credential
should trigger the deactivation of Ganga internal services.
"""
from Ganga.Runtime import Workspace_runtime
from Ganga.Runtime import Repository_runtime
if getName(credObj) == 'AfsToken':
return Workspace_runtime.requiresAfsToken() or Repository_runtime.requiresAfsToken()
if getName(credObj) == 'GridProxy':
from Ganga.Core.GangaRepository import getRegistryProxy
from Ganga.Runtime.GPIFunctions import typename
from Ganga.GPIDev.Base.Proxy import stripProxy
from Ganga.GPIDev.Lib.Job.Job import lazyLoadJobBackend, lazyLoadJobStatus
for j in getRegistryProxy('jobs'):
ji = stripProxy(j)
this_status = lazyLoadJobStatus(ji)
if this_status in ['submitted', 'running', 'completing']:
this_backend = lazyLoadJobBackend(ji)
if getName(this_backend) == 'LCG':
return True
return False
log.warning("Unknown credential object : %s" % credObj)
def isCredentialRequired(credObj):
"""
The logic to decide if a given invalid credential
should trigger the deactivation of Ganga internal services.
"""
from Ganga.Runtime import Workspace_runtime
from Ganga.Runtime import Repository_runtime
if getName(credObj) == 'AfsToken':
return Workspace_runtime.requiresAfsToken(
) or Repository_runtime.requiresAfsToken()
if getName(credObj) == 'GridProxy':
from Ganga.Core.GangaRepository import getRegistryProxy
from Ganga.Runtime.GPIFunctions import typename
from Ganga.GPIDev.Base.Proxy import stripProxy
from Ganga.GPIDev.Lib.Job.Job import lazyLoadJobBackend, lazyLoadJobStatus
for j in getRegistryProxy('jobs'):
ji = stripProxy(j)
this_status = lazyLoadJobStatus(ji)
if this_status in ['submitted', 'running', 'completing']:
this_backend = lazyLoadJobBackend(ji)
if getName(this_backend) == 'LCG':
return True
return False
log.warning("Unknown credential object : %s" % credObj)
def get_subjobs_in_time_range(jobid, fromDate=None, toDate=None):
from Ganga.Core.GangaRepository import getRegistryProxy
subjobs = []
for subjob in getRegistryProxy('jobs')(jobid).subjobs:
timeCreated = subjob.time.timestamps['new']
if fromDate is None and toDate is None:
subjobs.append(subjob)
elif fromDate is not None and toDate is not None:
if timeCreated >= fromDate and timeCreated <= toDate:
subjobs.append(subjob)
elif fromDate is not None and toDate is None:
if timeCreated >= fromDate:
subjobs.append(subjob)
return subjobs
def test_0_testRegistryAsserts(self):
from Ganga.GPI import jobs, box, tasks, prep
from Ganga.GPIDev.Base.Proxy import stripProxy
from Ganga.Core.GangaRepository import getRegistryProxy, getRegistrySlice, getRegistry
assert getRegistryProxy('jobs') is jobs
assert getRegistryProxy('box') is box
assert getRegistryProxy('tasks') is tasks
assert getRegistryProxy('prep') is prep
assert getRegistrySlice('jobs') is stripProxy(jobs)
assert getRegistrySlice('box') is stripProxy(box)
assert getRegistrySlice('tasks') is stripProxy(tasks)
assert getRegistrySlice('prep') is stripProxy(prep)
assert getRegistry('jobs') is stripProxy(jobs).objects
assert getRegistry('tasks') is stripProxy(tasks).objects
assert getRegistry('box') is stripProxy(box).objects
def test_0_testRegistryAsserts(self):
from Ganga.GPI import jobs, box, tasks, prep
from Ganga.GPIDev.Base.Proxy import stripProxy
from Ganga.Core.GangaRepository import getRegistryProxy, getRegistrySlice, getRegistry
assert getRegistryProxy('jobs') is jobs
assert getRegistryProxy('box') is box
assert getRegistryProxy('tasks') is tasks
assert getRegistryProxy('prep') is prep
assert getRegistrySlice('jobs') is stripProxy(jobs)
assert getRegistrySlice('box') is stripProxy(box)
assert getRegistrySlice('tasks') is stripProxy(tasks)
assert getRegistrySlice('prep') is stripProxy(prep)
assert getRegistry('jobs') is stripProxy(jobs).objects
assert getRegistry('tasks') is stripProxy(tasks).objects
assert getRegistry('box') is stripProxy(box).objects
def sleep_until_state(j,
timeout=None,
state='completed',
break_states=None,
sleep_period=1,
verbose=False):
'''
Wait until the job reaches the specified state
Returns:
True: if the state has been reached in the given timeout period
False: timeout occured or a break state has been reached
If break_states is specified, the call terminates when job enters in one of these state, returning False
'''
from Ganga.GPIDev.Base.Proxy import stripProxy
j = stripProxy(j)
if j.master is not None:
j = j.master
if timeout is None:
timeout = config['timeout']
from time import sleep
from Ganga.Core import monitoring_component
from Ganga.Core.GangaRepository import getRegistryProxy
jobs = getRegistryProxy('jobs')
current_status = None
while j.status != state and timeout > 0:
if not monitoring_component.isEnabled():
monitoring_component.runMonitoring(jobs=jobs.select(j.id, j.id),
_loadCredentials=False)
else:
monitoring_component.alive = True
monitoring_component.enabled = True
monitoring_component.steps = -1
monitoring_component.__updateTimeStamp = 0
monitoring_component.__sleepCounter = -0.5
if verbose and j.status != current_status:
logger.info("Job %s: status = %s" % (str(j.id), str(j.status)))
if current_status is None:
current_status = j.status
if type(break_states) == type([]) and j.status in break_states:
logger.info("Job finished with status: %s" % j.status)
return False
sleep(sleep_period)
timeout -= sleep_period
logger.debug("Status: %s" % j.status)
logger.info("Job finished with status: %s" % j.status)
logger.info("Timeout: %s" % str(timeout))
try:
j._getRegistry().updateLocksNow()
except:
pass
return j.status == state
def sleep_until_state(j, timeout=None, state='completed', break_states=None, sleep_period=1, verbose=False):
'''
Wait until the job reaches the specified state
Returns:
True: if the state has been reached in the given timeout period
False: timeout occured or a break state has been reached
If break_states is specified, the call terminates when job enters in one of these state, returning False
'''
from Ganga.GPIDev.Base.Proxy import stripProxy
j = stripProxy(j)
if j.master is not None:
j = j.master
if timeout is None:
timeout = config['timeout']
from time import sleep
from Ganga.Core import monitoring_component
from Ganga.Core.GangaRepository import getRegistryProxy
jobs = getRegistryProxy('jobs')
current_status = None
while j.status != state and timeout > 0:
if not monitoring_component.isEnabled():
monitoring_component.runMonitoring(jobs=jobs.select(j.id,j.id))
else:
monitoring_component.alive = True
monitoring_component.enabled = True
monitoring_component.steps = -1
monitoring_component.__updateTimeStamp = 0
monitoring_component.__sleepCounter = -0.5
if verbose and j.status != current_status:
logger.info("Job %s: status = %s" % (str(j.id), str(j.status)))
if current_status is None:
current_status = j.status
if type(break_states) == type([]) and j.status in break_states:
logger.info("Job finished with status: %s" % j.status )
return False
sleep(sleep_period)
timeout -= sleep_period
logger.debug("Status: %s" % j.status)
logger.info("Job finished with status: %s" % j.status )
logger.info("Timeout: %s" % str(timeout))
try:
j._getRegistry().updateLocksNow()
except:
pass
return j.status == state
def fill_jobs_dictionary():
from Ganga.Core.GangaRepository import getRegistryProxy
for job in getRegistryProxy('jobs'):
try:
# get the id -> it could cause RegistryKeyError and the code below
# will not be executed
jobid = job.id
try:
jobs_dictionary[jobid] = JobRelatedInfo(
job, job.time.timestamps['new'])
except RegistryKeyError:
jobs_dictionary[jobid] = JobRelatedInfo(job, None)
except RegistryKeyError:
pass
def update_jobs_dictionary():
from Ganga.Core.GangaRepository import getRegistryProxy
reg = getRegistry("jobs")
# get the changed jobs
changed_ids = reg.pollChangedJobs("WebGUI")
for job_id in changed_ids:
try:
job = getRegistryProxy('jobs')(job_id)
try:
jobs_dictionary[job_id] = JobRelatedInfo(
job, job.time.timestamps['new'])
except RegistryKeyError:
jobs_dictionary[job_id] = JobRelatedInfo(job, None)
except RegistryKeyError:
del jobs_dictionary[job_id]
def rebuild(self, unprepare=True, rmdir=False):
"""Rebuild the shareref table.
Clears the shareref table and then rebuilds it by iterating over all Ganga Objects
in the Job, Box and Task repositories. If an object has a ShareDir associated with it,
that ShareDir is added into the shareref table (the reference counter being incremented
accordingly). If called with the optional parameter 'unprepare=False', objects whose
ShareDirs are not present on disk will not be unprepared. Note that the default behaviour
is unprepare=True, i.e. the job/application would be unprepared.
After all Job/Box/Task objects have been checked, the inverse operation is performed,
i.e., for each directory in the ShareDir repository, a check is done to ensure there
is a matching entry in the shareref table. If not, and the optional parameter
'rmdir=True' is set, then the (orphaned) ShareDir will removed from the filesystem.
Otherwise, it will be added to the shareref table with a reference count of zero;
this results in the directory being deleted upon Ganga exit.
"""
self._getSessionLock()
# clear the shareref table
self.name = {}
lookup_input = []
from Ganga.GPIDev.Lib.File import getSharedPath
from Ganga.Core.GangaRepository import getRegistryProxy
objectlist = []
for thing in getRegistryProxy('jobs').select():
objectlist.append({thing: 'job'})
for thing in getRegistryProxy('box').select():
objectlist.append({thing: 'box'})
for thing in getRegistryProxy('tasks').select():
objectlist.append({thing: 'task'})
for item in objectlist:
shortname = None
try:
shortname = item.keys()[0].is_prepared.name
except AttributeError as err:
logger.debug("Err: %s" % err)
try:
shortname = item.keys()[0].application.is_prepared.name
except AttributeError as err2:
logger.debug("Err2: %s" % err2)
try:
shortname = item.keys(
)[0].analysis.application.is_prepared.name
except AttributeError as err3:
logger.debug("Err3: %s" % err3)
pass
try:
if shortname is not None and shortname is not True:
if os.path.basename(shortname) != shortname:
self.to_relative(item.keys()[0].is_prepared)
try:
numsubjobs = len(item.keys()[0].subjobs.ids())
except Exception as err:
logger.debug("Path Error: %s" % err)
Ganga.Utility.logging.log_unknown_exception()
numsubjobs = 0
self.helper(shortname,
unp=unprepare,
numsubjobs=numsubjobs)
except Exception as err:
logger.debug("-Error: %s" % err)
Ganga.Utility.logging.log_unknown_exception()
pass
# here we iterate over the lookup_input list and unprepare as
# necessary.
for item in lookup_input:
logger.info('Unpreparing objects referencing ShareDir %s' % item)
self.lookup(sharedir=item, unprepare=True)
# check to see that all sharedirs have an entry in the shareref. Otherwise, set their ref counter to 0
# so the user is made aware of them at shutdown
for this_dir in os.listdir(getSharedPath()):
if this_dir not in self.__getName().keys() and rmdir is False:
logger.debug(
"%s isn't referenced by a GangaObject in the Job or Box repository."
% this_dir)
self.__getName()[this_dir] = 0
elif this_dir not in self.__getName() and rmdir is True:
logger.debug(
"%s isn't referenced by a GangaObject in the Job or Box repository. Removing directory."
% this_dir)
shutil.rmtree(os.path.join(getSharedPath(), this_dir))
self._setDirty()
self._releaseSessionLockAndFlush()
def lookup(self, sharedir, unprepare=False):
"""
Report Job, Box and Task repository items which reference a given ShareDir object.
The optional parameter 'unprepare=True' can be set to call the unprepare method
on the returned objects.
"""
from Ganga.Core.GangaRepository import getRegistryProxy
objectlist = []
for thing in getRegistryProxy('jobs').select():
objectlist.append({thing: 'job'})
for thing in getRegistryProxy('box').select():
objectlist.append({thing: 'box'})
for thing in getRegistryProxy('tasks').select():
objectlist.append({thing: 'task'})
run_unp = None
master_index = 0
for item in objectlist:
try:
item.keys()[0].is_prepared.name
if item.keys()[0].is_prepared.name == sharedir:
logger.info(
'ShareDir %s is referenced by item #%s in %s repository'
% (sharedir, stripProxy(
item.keys()[0])._registry_id, item.values()[0]))
run_unp = item.keys()[0]
master_index += 1
except AttributeError as err:
logger.debug("Err: %s" % err)
try:
item.keys()[0].application.is_prepared.name
if item.keys()[0].application.is_prepared.name == sharedir:
logger.info(
'ShareDir %s is referenced by item #%s in %s repository'
%
(sharedir, stripProxy(item.keys()[0])._registry_id,
item.values()[0]))
run_unp = item.keys()[0].application
master_index += 1
except AttributeError as err2:
logger.debug("Err2: %s" % err2)
try:
item.keys()[0].analysis.application.is_prepared.name
if item.keys(
)[0].analysis.application.is_prepared.name == sharedir:
logger.info(
'ShareDir %s is referenced by item #%s in %s repository'
% (sharedir, stripProxy(
item.keys()[0])._registry_id,
item.values()[0]))
run_unp = item.keys()[0].analysis.application
master_index += 1
except AttributeError as err3:
logger.debug("Err3: %s" % err3)
pass
if run_unp is not None and unprepare is True:
logger.info(
'Unpreparing %s repository object #%s associated with ShareDir %s'
% (item.values()[0], stripProxy(
item.keys()[0])._registry_id, sharedir))
# stripProxy(item.keys()[0]).unprepare()
run_unp.unprepare()
run_unp = None
if unprepare is not True:
logger.info('%s item(s) found referencing ShareDir %s',
master_index, sharedir)
def table(self):
from Ganga.Core.GangaRepository import getRegistryProxy
t = getRegistryProxy('tasks').table(id=self.id)
def getObjectByID(this_id, reg_name):
"""returns proxies object from a repo based upon it's ID"""
from Ganga.Core.GangaRepository import getRegistryProxy
jobs_reg = getRegistryProxy(reg_name)
this_job = jobs_reg(this_id)
return this_job
def getObjectByID(this_id, reg_name):
"""returns proxies object from a repo based upon it's ID"""
from Ganga.Core.GangaRepository import getRegistryProxy
jobs_reg = getRegistryProxy(reg_name)
this_job = jobs_reg( this_id )
return this_job
def table(self):
from Ganga.Core.GangaRepository import getRegistryProxy
t = getRegistryProxy('tasks').table(id=self.id)
def rebuild(self, unprepare=True, rmdir=False):
"""Rebuild the shareref table.
Clears the shareref table and then rebuilds it by iterating over all Ganga Objects
in the Job, Box and Task repositories. If an object has a ShareDir associated with it,
that ShareDir is added into the shareref table (the reference counter being incremented
accordingly). If called with the optional parameter 'unprepare=False', objects whose
ShareDirs are not present on disk will not be unprepared. Note that the default behaviour
is unprepare=True, i.e. the job/application would be unprepared.
After all Job/Box/Task objects have been checked, the inverse operation is performed,
i.e., for each directory in the ShareDir repository, a check is done to ensure there
is a matching entry in the shareref table. If not, and the optional parameter
'rmdir=True' is set, then the (orphaned) ShareDir will removed from the filesystem.
Otherwise, it will be added to the shareref table with a reference count of zero;
this results in the directory being deleted upon Ganga exit.
"""
self._getWriteAccess()
# clear the shareref table
self.name = {}
lookup_input = []
from Ganga.GPIDev.Lib.File import getSharedPath
from Ganga.Core.GangaRepository import getRegistryProxy
objectlist = []
for thing in getRegistryProxy('jobs').select():
objectlist.append({thing: 'job'})
for thing in getRegistryProxy('box').select():
objectlist.append({thing: 'box'})
for thing in getRegistryProxy('tasks').select():
objectlist.append({thing: 'task'})
for item in objectlist:
shortname = None
try:
shortname = item.keys()[0].is_prepared.name
except AttributeError as err:
logger.debug("Err: %s" % err)
try:
shortname = item.keys()[0].application.is_prepared.name
except AttributeError as err2:
logger.debug("Err2: %s" % err2)
try:
shortname = item.keys()[0].analysis.application.is_prepared.name
except AttributeError as err3:
logger.debug("Err3: %s" % err3)
pass
try:
if shortname is not None and shortname is not True:
if os.path.basename(shortname) != shortname:
self.to_relative(item.keys()[0].is_prepared)
try:
numsubjobs = len(item.keys()[0].subjobs.ids())
except Exception as err:
logger.debug("Path Error: %s" % err)
Ganga.Utility.logging.log_unknown_exception()
numsubjobs = 0
self.helper(shortname, unp=unprepare, numsubjobs=numsubjobs)
except Exception as err:
logger.debug("-Error: %s" % err)
Ganga.Utility.logging.log_unknown_exception()
pass
# here we iterate over the lookup_input list and unprepare as
# necessary.
for item in lookup_input:
logger.info('Unpreparing objects referencing ShareDir %s' % item)
self.lookup(sharedir=item, unprepare=True)
# check to see that all sharedirs have an entry in the shareref. Otherwise, set their ref counter to 0
# so the user is made aware of them at shutdown
for this_dir in os.listdir(getSharedPath()):
if this_dir not in self.__getName().keys() and rmdir is False:
logger.debug("%s isn't referenced by a GangaObject in the Job or Box repository." % this_dir)
self.__getName()[this_dir] = 0
elif this_dir not in self.__getName().keys() and rmdir is True:
logger.debug("%s isn't referenced by a GangaObject in the Job or Box repository. Removing directory." % this_dir)
shutil.rmtree(os.path.join(getSharedPath(), this_dir))
self._setDirty()
self._releaseWriteAccess()