Example #1
0
    def lookup(self, sharedir, unprepare=False):
        """
        Report Job, Box and Task repository items which reference a given ShareDir object. 
        The optional parameter 'unprepare=True' can be set to call the unprepare method 
        on the returned objects.
        """
        from Ganga.Core.GangaRepository import getRegistryProxy
        objectlist = []
        for thing in getRegistryProxy('jobs').select():
            objectlist.append({thing: 'job'})
        for thing in getRegistryProxy('box').select():
            objectlist.append({thing: 'box'})
        for thing in getRegistryProxy('tasks').select():
            objectlist.append({thing: 'task'})

        run_unp = None
        master_index = 0
        for item in objectlist:
            try:
                item.keys()[0].is_prepared.name
                if item.keys()[0].is_prepared.name == sharedir:
                    logger.info('ShareDir %s is referenced by item #%s in %s repository' % (sharedir, stripProxy(item.keys()[0])._registry_id, item.values()[0]))
                    run_unp = item.keys()[0]
                    master_index += 1

            except AttributeError as err:
                logger.debug("Err: %s" % err)
                try:
                    item.keys()[0].application.is_prepared.name
                    if item.keys()[0].application.is_prepared.name == sharedir:
                        logger.info('ShareDir %s is referenced by item #%s in %s repository' % (sharedir, stripProxy(item.keys()[0])._registry_id, item.values()[0]))
                        run_unp = item.keys()[0].application
                        master_index += 1
                except AttributeError as err2:
                    logger.debug("Err2: %s" % err2)
                    try:
                        item.keys()[0].analysis.application.is_prepared.name
                        if item.keys()[0].analysis.application.is_prepared.name == sharedir:
                            logger.info('ShareDir %s is referenced by item #%s in %s repository' % (sharedir, stripProxy(item.keys()[0])._registry_id, item.values()[0]))
                            run_unp = item.keys()[0].analysis.application
                            master_index += 1
                    except AttributeError as err3:
                        logger.debug("Err3: %s" % err3)
                        pass

            if run_unp is not None and unprepare is True:
                logger.info('Unpreparing %s repository object #%s associated with ShareDir %s' % (item.values()[0],  stripProxy(item.keys()[0])._registry_id, sharedir))
#                stripProxy(item.keys()[0]).unprepare()
                run_unp.unprepare()
                run_unp = None

        if unprepare is not True:
            logger.info('%s item(s) found referencing ShareDir %s', master_index, sharedir)
Example #2
0
def isCredentialRequired(credObj):
    """
    The logic to decide if a given invalid credential
    should trigger the deactivation of Ganga internal services.  
    """

    from Ganga.Runtime import Workspace_runtime
    from Ganga.Runtime import Repository_runtime

    if getName(credObj) == 'AfsToken':
        return Workspace_runtime.requiresAfsToken() or Repository_runtime.requiresAfsToken()

    if getName(credObj) == 'GridProxy':
        from Ganga.Core.GangaRepository import getRegistryProxy
        from Ganga.Runtime.GPIFunctions import typename
        from Ganga.GPIDev.Base.Proxy import stripProxy
        from Ganga.GPIDev.Lib.Job.Job import lazyLoadJobBackend, lazyLoadJobStatus
        for j in getRegistryProxy('jobs'):
            ji = stripProxy(j)
            this_status = lazyLoadJobStatus(ji)
            if this_status in ['submitted', 'running', 'completing']:
                this_backend = lazyLoadJobBackend(ji)
                if getName(this_backend) == 'LCG':
                    return True
        return False

    log.warning("Unknown credential object : %s" % credObj)
Example #3
0
def isCredentialRequired(credObj):
    """
    The logic to decide if a given invalid credential
    should trigger the deactivation of Ganga internal services.  
    """

    from Ganga.Runtime import Workspace_runtime
    from Ganga.Runtime import Repository_runtime

    if getName(credObj) == 'AfsToken':
        return Workspace_runtime.requiresAfsToken(
        ) or Repository_runtime.requiresAfsToken()

    if getName(credObj) == 'GridProxy':
        from Ganga.Core.GangaRepository import getRegistryProxy
        from Ganga.Runtime.GPIFunctions import typename
        from Ganga.GPIDev.Base.Proxy import stripProxy
        from Ganga.GPIDev.Lib.Job.Job import lazyLoadJobBackend, lazyLoadJobStatus
        for j in getRegistryProxy('jobs'):
            ji = stripProxy(j)
            this_status = lazyLoadJobStatus(ji)
            if this_status in ['submitted', 'running', 'completing']:
                this_backend = lazyLoadJobBackend(ji)
                if getName(this_backend) == 'LCG':
                    return True
        return False

    log.warning("Unknown credential object : %s" % credObj)
Example #4
0
def get_subjobs_in_time_range(jobid, fromDate=None, toDate=None):
    from Ganga.Core.GangaRepository import getRegistryProxy

    subjobs = []

    for subjob in getRegistryProxy('jobs')(jobid).subjobs:

        timeCreated = subjob.time.timestamps['new']

        if fromDate is None and toDate is None:

            subjobs.append(subjob)

        elif fromDate is not None and toDate is not None:

            if timeCreated >= fromDate and timeCreated <= toDate:

                subjobs.append(subjob)

        elif fromDate is not None and toDate is None:

            if timeCreated >= fromDate:

                subjobs.append(subjob)

    return subjobs
Example #5
0
    def test_0_testRegistryAsserts(self):
        from Ganga.GPI import jobs, box, tasks, prep
        from Ganga.GPIDev.Base.Proxy import stripProxy
        from Ganga.Core.GangaRepository import getRegistryProxy, getRegistrySlice, getRegistry
        assert getRegistryProxy('jobs') is jobs
        assert getRegistryProxy('box') is box
        assert getRegistryProxy('tasks') is tasks
        assert getRegistryProxy('prep') is prep

        assert getRegistrySlice('jobs') is stripProxy(jobs)
        assert getRegistrySlice('box') is stripProxy(box)
        assert getRegistrySlice('tasks') is stripProxy(tasks)
        assert getRegistrySlice('prep') is stripProxy(prep)

        assert getRegistry('jobs') is stripProxy(jobs).objects
        assert getRegistry('tasks') is stripProxy(tasks).objects
        assert getRegistry('box') is stripProxy(box).objects
Example #6
0
    def test_0_testRegistryAsserts(self):
        from Ganga.GPI import jobs, box, tasks, prep
        from Ganga.GPIDev.Base.Proxy import stripProxy
        from Ganga.Core.GangaRepository import getRegistryProxy, getRegistrySlice, getRegistry
        assert getRegistryProxy('jobs') is jobs
        assert getRegistryProxy('box') is box
        assert getRegistryProxy('tasks') is tasks
        assert getRegistryProxy('prep') is prep

        assert getRegistrySlice('jobs') is stripProxy(jobs)
        assert getRegistrySlice('box') is stripProxy(box)
        assert getRegistrySlice('tasks') is stripProxy(tasks)
        assert getRegistrySlice('prep') is stripProxy(prep)

        assert getRegistry('jobs') is stripProxy(jobs).objects
        assert getRegistry('tasks') is stripProxy(tasks).objects
        assert getRegistry('box') is stripProxy(box).objects
Example #7
0
def sleep_until_state(j,
                      timeout=None,
                      state='completed',
                      break_states=None,
                      sleep_period=1,
                      verbose=False):
    '''
    Wait until the job reaches the specified state
    Returns:
     True: if the state has been reached in the given timeout period
     False: timeout occured or a break state has been reached
    If break_states is specified, the call terminates when job enters in one of these state, returning False
    '''

    from Ganga.GPIDev.Base.Proxy import stripProxy
    j = stripProxy(j)
    if j.master is not None:
        j = j.master

    if timeout is None:
        timeout = config['timeout']

    from time import sleep
    from Ganga.Core import monitoring_component
    from Ganga.Core.GangaRepository import getRegistryProxy

    jobs = getRegistryProxy('jobs')

    current_status = None
    while j.status != state and timeout > 0:
        if not monitoring_component.isEnabled():
            monitoring_component.runMonitoring(jobs=jobs.select(j.id, j.id),
                                               _loadCredentials=False)
        else:
            monitoring_component.alive = True
            monitoring_component.enabled = True
            monitoring_component.steps = -1
            monitoring_component.__updateTimeStamp = 0
            monitoring_component.__sleepCounter = -0.5
        if verbose and j.status != current_status:
            logger.info("Job %s: status = %s" % (str(j.id), str(j.status)))
        if current_status is None:
            current_status = j.status
        if type(break_states) == type([]) and j.status in break_states:
            logger.info("Job finished with status: %s" % j.status)
            return False
        sleep(sleep_period)
        timeout -= sleep_period
        logger.debug("Status: %s" % j.status)
    logger.info("Job finished with status: %s" % j.status)
    logger.info("Timeout: %s" % str(timeout))
    try:
        j._getRegistry().updateLocksNow()
    except:
        pass
    return j.status == state
Example #8
0
def sleep_until_state(j, timeout=None, state='completed', break_states=None, sleep_period=1, verbose=False):
    '''
    Wait until the job reaches the specified state
    Returns:
     True: if the state has been reached in the given timeout period
     False: timeout occured or a break state has been reached
    If break_states is specified, the call terminates when job enters in one of these state, returning False
    '''

    from Ganga.GPIDev.Base.Proxy import stripProxy
    j = stripProxy(j)
    if j.master is not None:
        j = j.master

    if timeout is None:
        timeout = config['timeout']
        
    from time import sleep
    from Ganga.Core import monitoring_component
    from Ganga.Core.GangaRepository import getRegistryProxy
    
    jobs = getRegistryProxy('jobs')

    current_status = None
    while j.status != state and timeout > 0:
        if not monitoring_component.isEnabled():
            monitoring_component.runMonitoring(jobs=jobs.select(j.id,j.id))
        else:
            monitoring_component.alive = True
            monitoring_component.enabled = True
            monitoring_component.steps = -1
            monitoring_component.__updateTimeStamp = 0
            monitoring_component.__sleepCounter = -0.5
        if verbose and j.status != current_status:
            logger.info("Job %s: status = %s" % (str(j.id), str(j.status)))
        if current_status is None:
            current_status = j.status
        if type(break_states) == type([]) and j.status in break_states:
            logger.info("Job finished with status: %s" % j.status )
            return False
        sleep(sleep_period)
        timeout -= sleep_period
        logger.debug("Status: %s" % j.status)
    logger.info("Job finished with status: %s" % j.status )
    logger.info("Timeout: %s" % str(timeout))
    try:
        j._getRegistry().updateLocksNow()
    except:
        pass
    return j.status == state
Example #9
0
def fill_jobs_dictionary():
    from Ganga.Core.GangaRepository import getRegistryProxy

    for job in getRegistryProxy('jobs'):
        try:
            # get the id -> it could cause RegistryKeyError and the code below
            # will not be executed
            jobid = job.id

            try:
                jobs_dictionary[jobid] = JobRelatedInfo(
                    job, job.time.timestamps['new'])
            except RegistryKeyError:
                jobs_dictionary[jobid] = JobRelatedInfo(job, None)

        except RegistryKeyError:
            pass
Example #10
0
def update_jobs_dictionary():
    from Ganga.Core.GangaRepository import getRegistryProxy

    reg = getRegistry("jobs")
    # get the changed jobs
    changed_ids = reg.pollChangedJobs("WebGUI")

    for job_id in changed_ids:
        try:

            job = getRegistryProxy('jobs')(job_id)

            try:
                jobs_dictionary[job_id] = JobRelatedInfo(
                    job, job.time.timestamps['new'])
            except RegistryKeyError:
                jobs_dictionary[job_id] = JobRelatedInfo(job, None)

        except RegistryKeyError:

            del jobs_dictionary[job_id]
Example #11
0
    def rebuild(self, unprepare=True, rmdir=False):
        """Rebuild the shareref table. 
        Clears the shareref table and then rebuilds it by iterating over all Ganga Objects 
        in the Job, Box and Task repositories. If an object has a ShareDir associated with it, 
        that ShareDir is added into the shareref table (the reference counter being incremented 
        accordingly). If called with the optional parameter 'unprepare=False', objects whose
        ShareDirs are not present on disk will not be unprepared. Note that the default behaviour
        is unprepare=True, i.e. the job/application would be unprepared.
        After all Job/Box/Task objects have been checked, the inverse operation is performed, 
        i.e., for each directory in the ShareDir repository, a check is done to ensure there 
        is a matching entry in the shareref table. If not, and the optional parameter 
        'rmdir=True' is set, then the (orphaned) ShareDir will removed from the filesystem. 
        Otherwise, it will be added to the shareref table with a reference count of zero; 
        this results in the directory being deleted upon Ganga exit.
        """
        self._getSessionLock()
        # clear the shareref table
        self.name = {}
        lookup_input = []

        from Ganga.GPIDev.Lib.File import getSharedPath
        from Ganga.Core.GangaRepository import getRegistryProxy

        objectlist = []
        for thing in getRegistryProxy('jobs').select():
            objectlist.append({thing: 'job'})
        for thing in getRegistryProxy('box').select():
            objectlist.append({thing: 'box'})
        for thing in getRegistryProxy('tasks').select():
            objectlist.append({thing: 'task'})

        for item in objectlist:
            shortname = None
            try:
                shortname = item.keys()[0].is_prepared.name
            except AttributeError as err:
                logger.debug("Err: %s" % err)
                try:
                    shortname = item.keys()[0].application.is_prepared.name
                except AttributeError as err2:
                    logger.debug("Err2: %s" % err2)
                    try:
                        shortname = item.keys(
                        )[0].analysis.application.is_prepared.name
                    except AttributeError as err3:
                        logger.debug("Err3: %s" % err3)
                        pass
            try:
                if shortname is not None and shortname is not True:
                    if os.path.basename(shortname) != shortname:
                        self.to_relative(item.keys()[0].is_prepared)
                    try:
                        numsubjobs = len(item.keys()[0].subjobs.ids())
                    except Exception as err:
                        logger.debug("Path Error: %s" % err)
                        Ganga.Utility.logging.log_unknown_exception()
                        numsubjobs = 0
                    self.helper(shortname,
                                unp=unprepare,
                                numsubjobs=numsubjobs)
            except Exception as err:
                logger.debug("-Error: %s" % err)
                Ganga.Utility.logging.log_unknown_exception()
                pass

        # here we iterate over the lookup_input list and unprepare as
        # necessary.
        for item in lookup_input:
            logger.info('Unpreparing objects referencing ShareDir %s' % item)
            self.lookup(sharedir=item, unprepare=True)

        # check to see that all sharedirs have an entry in the shareref. Otherwise, set their ref counter to 0
        # so the user is made aware of them at shutdown
        for this_dir in os.listdir(getSharedPath()):
            if this_dir not in self.__getName().keys() and rmdir is False:
                logger.debug(
                    "%s isn't referenced by a GangaObject in the Job or Box repository."
                    % this_dir)
                self.__getName()[this_dir] = 0
            elif this_dir not in self.__getName() and rmdir is True:
                logger.debug(
                    "%s isn't referenced by a GangaObject in the Job or Box repository. Removing directory."
                    % this_dir)
                shutil.rmtree(os.path.join(getSharedPath(), this_dir))

        self._setDirty()
        self._releaseSessionLockAndFlush()
Example #12
0
    def lookup(self, sharedir, unprepare=False):
        """
        Report Job, Box and Task repository items which reference a given ShareDir object. 
        The optional parameter 'unprepare=True' can be set to call the unprepare method 
        on the returned objects.
        """
        from Ganga.Core.GangaRepository import getRegistryProxy
        objectlist = []
        for thing in getRegistryProxy('jobs').select():
            objectlist.append({thing: 'job'})
        for thing in getRegistryProxy('box').select():
            objectlist.append({thing: 'box'})
        for thing in getRegistryProxy('tasks').select():
            objectlist.append({thing: 'task'})

        run_unp = None
        master_index = 0
        for item in objectlist:
            try:
                item.keys()[0].is_prepared.name
                if item.keys()[0].is_prepared.name == sharedir:
                    logger.info(
                        'ShareDir %s is referenced by item #%s in %s repository'
                        % (sharedir, stripProxy(
                            item.keys()[0])._registry_id, item.values()[0]))
                    run_unp = item.keys()[0]
                    master_index += 1

            except AttributeError as err:
                logger.debug("Err: %s" % err)
                try:
                    item.keys()[0].application.is_prepared.name
                    if item.keys()[0].application.is_prepared.name == sharedir:
                        logger.info(
                            'ShareDir %s is referenced by item #%s in %s repository'
                            %
                            (sharedir, stripProxy(item.keys()[0])._registry_id,
                             item.values()[0]))
                        run_unp = item.keys()[0].application
                        master_index += 1
                except AttributeError as err2:
                    logger.debug("Err2: %s" % err2)
                    try:
                        item.keys()[0].analysis.application.is_prepared.name
                        if item.keys(
                        )[0].analysis.application.is_prepared.name == sharedir:
                            logger.info(
                                'ShareDir %s is referenced by item #%s in %s repository'
                                % (sharedir, stripProxy(
                                    item.keys()[0])._registry_id,
                                   item.values()[0]))
                            run_unp = item.keys()[0].analysis.application
                            master_index += 1
                    except AttributeError as err3:
                        logger.debug("Err3: %s" % err3)
                        pass

            if run_unp is not None and unprepare is True:
                logger.info(
                    'Unpreparing %s repository object #%s associated with ShareDir %s'
                    % (item.values()[0], stripProxy(
                        item.keys()[0])._registry_id, sharedir))
                #                stripProxy(item.keys()[0]).unprepare()
                run_unp.unprepare()
                run_unp = None

        if unprepare is not True:
            logger.info('%s item(s) found referencing ShareDir %s',
                        master_index, sharedir)
Example #13
0
 def table(self):
     from Ganga.Core.GangaRepository import getRegistryProxy
     t = getRegistryProxy('tasks').table(id=self.id)
Example #14
0
def getObjectByID(this_id, reg_name):
    """returns proxies object from a repo based upon it's ID"""
    from Ganga.Core.GangaRepository import getRegistryProxy
    jobs_reg = getRegistryProxy(reg_name)
    this_job = jobs_reg(this_id)
    return this_job
Example #15
0
def getObjectByID(this_id, reg_name):
    """returns proxies object from a repo based upon it's ID"""
    from Ganga.Core.GangaRepository import getRegistryProxy
    jobs_reg = getRegistryProxy(reg_name)
    this_job = jobs_reg( this_id )
    return this_job
Example #16
0
 def table(self):
     from Ganga.Core.GangaRepository import getRegistryProxy
     t = getRegistryProxy('tasks').table(id=self.id)
Example #17
0
    def rebuild(self, unprepare=True, rmdir=False):
        """Rebuild the shareref table. 
        Clears the shareref table and then rebuilds it by iterating over all Ganga Objects 
        in the Job, Box and Task repositories. If an object has a ShareDir associated with it, 
        that ShareDir is added into the shareref table (the reference counter being incremented 
        accordingly). If called with the optional parameter 'unprepare=False', objects whose
        ShareDirs are not present on disk will not be unprepared. Note that the default behaviour
        is unprepare=True, i.e. the job/application would be unprepared.
        After all Job/Box/Task objects have been checked, the inverse operation is performed, 
        i.e., for each directory in the ShareDir repository, a check is done to ensure there 
        is a matching entry in the shareref table. If not, and the optional parameter 
        'rmdir=True' is set, then the (orphaned) ShareDir will removed from the filesystem. 
        Otherwise, it will be added to the shareref table with a reference count of zero; 
        this results in the directory being deleted upon Ganga exit.
        """
        self._getWriteAccess()
        # clear the shareref table
        self.name = {}
        lookup_input = []

        from Ganga.GPIDev.Lib.File import getSharedPath
        from Ganga.Core.GangaRepository import getRegistryProxy

        objectlist = []
        for thing in getRegistryProxy('jobs').select():
            objectlist.append({thing: 'job'})
        for thing in getRegistryProxy('box').select():
            objectlist.append({thing: 'box'})
        for thing in getRegistryProxy('tasks').select():
            objectlist.append({thing: 'task'})

        for item in objectlist:
            shortname = None
            try:
                shortname = item.keys()[0].is_prepared.name
            except AttributeError as err:
                logger.debug("Err: %s" % err)
                try:
                    shortname = item.keys()[0].application.is_prepared.name
                except AttributeError as err2:
                    logger.debug("Err2: %s" % err2)
                    try:
                        shortname = item.keys()[0].analysis.application.is_prepared.name
                    except AttributeError as err3:
                        logger.debug("Err3: %s" % err3)
                        pass
            try:
                if shortname is not None and shortname is not True:
                    if os.path.basename(shortname) != shortname:
                        self.to_relative(item.keys()[0].is_prepared)
                    try:
                        numsubjobs = len(item.keys()[0].subjobs.ids())
                    except Exception as err:
                        logger.debug("Path Error: %s" % err)
                        Ganga.Utility.logging.log_unknown_exception()
                        numsubjobs = 0
                    self.helper(shortname, unp=unprepare, numsubjobs=numsubjobs)
            except Exception as err:
                logger.debug("-Error: %s" % err)
                Ganga.Utility.logging.log_unknown_exception()
                pass

        # here we iterate over the lookup_input list and unprepare as
        # necessary.
        for item in lookup_input:
            logger.info('Unpreparing objects referencing ShareDir %s' % item)
            self.lookup(sharedir=item, unprepare=True)

        # check to see that all sharedirs have an entry in the shareref. Otherwise, set their ref counter to 0
        # so the user is made aware of them at shutdown
        for this_dir in os.listdir(getSharedPath()):
            if this_dir not in self.__getName().keys() and rmdir is False:
                logger.debug("%s isn't referenced by a GangaObject in the Job or Box repository." % this_dir)
                self.__getName()[this_dir] = 0
            elif this_dir not in self.__getName().keys() and rmdir is True:
                logger.debug("%s isn't referenced by a GangaObject in the Job or Box repository. Removing directory." % this_dir)
                shutil.rmtree(os.path.join(getSharedPath(), this_dir))

        self._setDirty()
        self._releaseWriteAccess()