예제 #1
0
    def test_c_JobManipulation(self):

        from Ganga.GPI import runMonitoring, Job, jobs, export, load

        runMonitoring()

        # -- JOBMANIPULATION JOBCOPY START
        j = Job(name='original')
        j2 = j.copy()
        j2.name = 'copy'
        j.submit()
        j3 = Job(j, name='copy2')
        jobs
        # -- JOBMANIPULATION JOBCOPY STOP

        # -- JOBMANIPULATION REPOACCESS START
        jobs(2)
        # -- JOBMANIPULATION REPOACCESS STOP

        # -- JOBMANIPULATION JOBSLICING START
        jobs[2]
        jobs[2:]
        jobs['copy2']
        # -- JOBMANIPULATION JOBSLICING STOP

        jobs(0).kill()
        # -- JOBMANIPULATION RESUBMIT START
        jobs(0).resubmit()
        # -- JOBMANIPULATION RESUBMIT STOP

        # -- JOBMANIPULATION FORCESTATUS START
        jobs(1).force_status('failed')
        # -- JOBMANIPULATION FORCESTATUS STOP

        # -- JOBMANIPULATION JOBREMOVE START
        jobs(2).remove()
        # -- JOBMANIPULATION JOBREMOVE STOP

        # -- JOBMANIPULATION JOBSELECT START
        # can select on ids, name, status, backend, application
        jobs.select(status='new')
        jobs.select(backend='Local')
        jobs.select(ids=[1, 3])

        # can restrict on min/max id
        jobs.select(1, 3, application='Executable')
        # -- JOBMANIPULATION JOBSELECT STOP

        # -- JOBMANIPULATION JOBSELECTOP START
        jobs.select(status='new').submit()
        # -- JOBMANIPULATION JOBSELECTOP STOP

        # -- JOBMANIPULATION EXPORTJOB START
        export(jobs(0), 'my_job.txt')
        jlist = load('my_job.txt')
        jlist[0].submit()
예제 #2
0
    def test_c_JobManipulation(self):

        from Ganga.GPI import runMonitoring, Job, jobs, export, load

        runMonitoring()

        # -- JOBMANIPULATION JOBCOPY START
        j = Job(name = 'original')
        j2 = j.copy()
        j2.name = 'copy'
        j.submit()
        j3 = Job(j, name = 'copy2')
        jobs
        # -- JOBMANIPULATION JOBCOPY STOP

        # -- JOBMANIPULATION REPOACCESS START
        jobs(2)
        # -- JOBMANIPULATION REPOACCESS STOP

        # -- JOBMANIPULATION JOBSLICING START
        jobs[2]
        jobs[2:]
        jobs['copy2']
        # -- JOBMANIPULATION JOBSLICING STOP

        jobs(0).kill()
        # -- JOBMANIPULATION RESUBMIT START
        jobs(0).resubmit()
        # -- JOBMANIPULATION RESUBMIT STOP

        # -- JOBMANIPULATION FORCESTATUS START
        jobs(1).force_status('failed')
        # -- JOBMANIPULATION FORCESTATUS STOP

        # -- JOBMANIPULATION JOBREMOVE START
        jobs(2).remove()
        # -- JOBMANIPULATION JOBREMOVE STOP

        # -- JOBMANIPULATION JOBSELECT START
        # can select on ids, name, status, backend, application
        jobs.select(status='new')
        jobs.select(backend='Local')
        jobs.select(ids=[1,3])

        # can restrict on min/max id
        jobs.select(1,3, application='Executable')
        # -- JOBMANIPULATION JOBSELECT STOP

        # -- JOBMANIPULATION JOBSELECTOP START
        jobs.select(status='new').submit()
        # -- JOBMANIPULATION JOBSELECTOP STOP

        # -- JOBMANIPULATION EXPORTJOB START
        export(jobs(0), 'my_job.txt')
        jlist = load('my_job.txt')
        jlist[0].submit()
예제 #3
0
def getGangaIDs(jobname):

    print "Getting Job IDs matching name", jobname

    from Ganga.GPI import jobs

    jobids = [j.id for j in jobs.select(name=jobname)]

    return jobids
예제 #4
0
    def lookup(self, sharedir, unprepare=False):
        """
        Report Job, Box and Task repository items which reference a given ShareDir object. 
        The optional parameter 'unprepare=True' can be set to call the unprepare method 
        on the returned objects.
        """
        from Ganga.GPI import jobs, box, tasks
        objectlist = []
        for thing in jobs.select():
            objectlist.append({thing: 'job'})
        for thing in box.select():
            objectlist.append({thing: 'box'})
        for thing in tasks.select():
            objectlist.append({thing: 'task'})

        run_unp = None
        master_index = 0
        for item in objectlist:
            try:
                item.keys()[0].is_prepared.name
                if item.keys()[0].is_prepared.name == sharedir:
                    logger.info('ShareDir %s is referenced by item #%s in %s repository', sharedir,
                                stripProxy(item.keys()[0])._registry_id, item.values()[0])
                    run_unp = item.keys()[0]
                    master_index += 1
            except AttributeError as err:
                logger.debug("Err: %s" % str(err))
                try:
                    item.keys()[0].application.is_prepared.name
                    if item.keys()[0].application.is_prepared.name == sharedir:
                        logger.info('ShareDir %s is referenced by item #%s in %s repository', sharedir,
                                    stripProxy(item.keys()[0])._registry_id, item.values()[0])
                        run_unp = item.keys()[0].application
                        master_index += 1
                except AttributeError as err2:
                    logger.debug("Err2: %s" % str(err2))
                    try:
                        item.keys()[0].analysis.application.is_prepared.name
                        if item.keys()[0].analysis.application.is_prepared.name == sharedir:
                            logger.info('ShareDir %s is referenced by item #%s in %s repository', sharedir,
                                        stripProxy(item.keys()[0])._registry_id, item.values()[0])
                            run_unp = item.keys()[0].analysis.application
                            master_index += 1
                    except AttributeError as err3:
                        logger.debug("Err3: %s" % str(err3))
                        pass

            if run_unp is not None and unprepare is True:
                logger.info('Unpreparing %s repository object #%s associated with ShareDir %s',
                            item.values()[0],  stripProxy(item.keys()[0])._registry_id, sharedir)
#                stripProxy(item.keys()[0]).unprepare()
                run_unp.unprepare()
                run_unp = None

        if unprepare is not True:
            logger.info('%s item(s) found referencing ShareDir %s', master_index, sharedir)
예제 #5
0
    def test_SelectJob(self):
        
        from Ganga.GPI import Job, jobs
        j=Job()

        job_slice =  jobs.select(status='new')

        assert len(job_slice) == 1

        j.submit()

        from GangaTest.Framework.utils import sleep_until_completed
        sleep_until_completed(j)

        assert j.status == 'completed'

        job_slice2 =  jobs.select(status='completed')

        assert len(job_slice2) == 1
예제 #6
0
    def run(self):
        config = Config.getConfig("LGI")

        # wait for GPI to become ready (taken from GangaJEM)
        while not self.should_stop():
            try:
                from Ganga.GPI import jobs

                break
            except:
                pass
            time.sleep(1)
        from Ganga.GPI import LGI

        # LGI update loop
        self.log.debug("Starting LGI StatsThread main loop")
        self.data = []
        self._writeStats(self.data, config["StatsFile"])
        while not self.should_stop():
            now = time.time()

            try:
                # add new line of data
                lgiQueued = LGI.resource.queued
                if lgiQueued is None:
                    lgiQueued = 0
                lgiRunning = 0  # TODO
                pilotQueued = sum([len(jobs.select(status=s)) for s in ["submitted", "submitting"]])
                pilotRunning = len(jobs.select(status="running"))
                self.data.append(
                    [int(now), lgiQueued + lgiRunning, lgiRunning, pilotQueued + pilotRunning, pilotRunning]
                )
                # trash too old lines
                self.data = filter(lambda x: now - x[0] < config["StatsHistory"], self.data)
                # write data
                self._writeStats(self.data, config["StatsFile"])
            except Exception, e:
                self.log.warn(e)

                # and wait for next iteration
            while not self.should_stop() and time.time() - now < config["StatsInterval"]:
                time.sleep(1)
예제 #7
0
파일: utils.py 프로젝트: slangrock/ganga
def sleep_until_state(j,
                      timeout=None,
                      state='completed',
                      break_states=None,
                      sleep_period=1,
                      verbose=False):
    '''
    Wait until the job reaches the specified state
    Returns:
     True: if the state has been reached in the given timeout period
     False: timeout occured or a break state has been reached
    If break_states is specified, the call terminates when job enters in one of these state, returning False
    '''

    from Ganga.GPIDev.Base.Proxy import stripProxy
    j = stripProxy(j)
    if j.master is not None:
        j = j.master

    if timeout is None:
        timeout = config['timeout']

    from time import sleep
    from Ganga.Core import monitoring_component
    from Ganga.GPI import jobs

    current_status = None
    while j.status != state and timeout > 0:
        if not monitoring_component.isEnabled():
            monitoring_component.runMonitoring(jobs=jobs.select(j.id, j.id),
                                               _loadCredentials=False)
        else:
            monitoring_component.alive = True
            monitoring_component.enabled = True
            monitoring_component.steps = -1
            monitoring_component.__updateTimeStamp = 0
            monitoring_component.__sleepCounter = -0.5
        if verbose and j.status != current_status:
            logger.info("Job %s: status = %s" % (str(j.id), str(j.status)))
        if current_status is None:
            current_status = j.status
        if type(break_states) == type([]) and j.status in break_states:
            logger.info("Job finished with status: %s" % j.status)
            return False
        sleep(sleep_period)
        timeout -= sleep_period
        logger.debug("Status: %s" % j.status)
    logger.info("Job finished with status: %s" % j.status)
    logger.info("Timeout: %s" % str(timeout))
    try:
        j._getRegistry().updateLocksNow()
    except:
        pass
    return j.status == state
예제 #8
0
    def test_b_SelectTests(self):
        """ Check select methods"""
        from Ganga.GPI import jobs, Executable

        self.assertEqual(len(jobs), job_num)

        mySlice = jobs.select(application=Executable)

        assert len(mySlice) == job_num

        mySlice2 = jobs.select(id=1)

        assert len(mySlice2) == 1
        assert mySlice2[1].id == 1

        for i in range(job_num):

            mySlice3 = jobs.select(name=job_names[i])

            assert len(mySlice3) == 1
            assert mySlice3[i].name == job_names[i]
예제 #9
0
    def test_b_SelectTests(self):
        """ Check select methods"""
        from Ganga.GPI import jobs, Executable

        self.assertEqual(len(jobs), job_num)

        mySlice = jobs.select(application=Executable)

        assert len(mySlice) == job_num

        mySlice2 = jobs.select(id=1)

        assert len(mySlice2) == 1
        assert mySlice2[1].id == 1

        for i in range(job_num):

            mySlice3 = jobs.select(name=job_names[i])

            assert len(mySlice3) == 1
            assert mySlice3[i].name == job_names[i]
예제 #10
0
def sleep_until_state(j, timeout=None, state='completed', break_states=None, sleep_period=1, verbose=False):
    '''
    Wait until the job reaches the specified state
    Returns:
     True: if the state has been reached in the given timeout period
     False: timeout occured or a break state has been reached
    If break_states is specified, the call terminates when job enters in one of these state, returning False
    '''

    from Ganga.GPIDev.Base.Proxy import stripProxy
    j = stripProxy(j)
    if j.master is not None:
        j = j.master

    if timeout is None:
        timeout = config['timeout']
        
    from time import sleep
    from Ganga.Core import monitoring_component
    from Ganga.GPI import jobs

    current_status = None
    while j.status != state and timeout > 0:
        if not monitoring_component.isEnabled():
            monitoring_component.runMonitoring( jobs=jobs.select(j.id,j.id), _loadCredentials=False )
        else:
            monitoring_component.alive = True
            monitoring_component.enabled = True
            monitoring_component.steps = -1
            monitoring_component.__updateTimeStamp = 0
            monitoring_component.__sleepCounter = -0.5
        if verbose and j.status != current_status:
            logger.info("Job %s: status = %s" % (str(j.id), str(j.status)))
        if current_status is None:
            current_status = j.status
        if type(break_states) == type([]) and j.status in break_states:
            logger.info("Job finished with status: %s" % j.status )
            return False
        sleep(sleep_period)
        timeout -= sleep_period
        logger.debug("Status: %s" % j.status)
    logger.info("Job finished with status: %s" % j.status )
    logger.info("Timeout: %s" % str(timeout))
    try:
        j._getRegistry().updateLocksNow()
    except:
        pass
    return j.status == state
예제 #11
0
    def Savannah74531(self):
        from Ganga.GPI import Job, jobs, jobtree

        index = 10
        while index > 0:
            Job()
            index -= 1

        testSlice = jobs[0:4]
        testList1 = jobs.select(4, 7)
        testList2 = [jobs[8], jobs[9]]

        jobtree.cd()
        jobtree.mkdir('testTreeOne')

        jobtree.add(testSlice, 'testTreeOne')
        jobtree.add(testList1, 'testTreeOne')
        jobtree.add(testList2, 'testTreeOne')

        jobtree.rm('testTreeOne')
예제 #12
0
    def Savannah74531(self):
        from Ganga.GPI import Job, jobs, jobtree

        index = 10
        while index > 0:
           Job()
           index -= 1

        testSlice = jobs[0:4]
        testList1 = jobs.select(4, 7)
        testList2 = [jobs[8], jobs[9]]

        jobtree.cd()
        jobtree.mkdir('testTreeOne')

        jobtree.add(testSlice, 'testTreeOne')
        jobtree.add(testList1, 'testTreeOne')
        jobtree.add(testList2, 'testTreeOne')

        jobtree.rm('testTreeOne')
예제 #13
0
    def test_b_Configuration(self):
        from Ganga.GPI import config, jobs

        # -- CONFIGURATION VIEWCHANGE START
        # print full config
        config

        # print config section
        config.Logging

        # edit a config option
        config.Logging['Ganga.Lib'] = 'DEBUG'
        # -- CONFIGURATION VIEWCHANGE STOP

        # -- CONFIGURATION DEFAULTCHANGE START
        config.defaults_Executable.exe = 'ls'
        # -- CONFIGURATION DEFAULTCHANGE STOP

        # -- CONFIGURATION STARTUPSCRIPT START
        slice = jobs.select(status='running')
        print slice
예제 #14
0
    def test_b_Configuration(self):
        from Ganga.GPI import config, jobs

        # -- CONFIGURATION VIEWCHANGE START
        # print full config
        config

        # print config section
        config.Logging

        # edit a config option
        config.Logging['Ganga.Lib'] = 'DEBUG'
        # -- CONFIGURATION VIEWCHANGE STOP

        # -- CONFIGURATION DEFAULTCHANGE START
        config.defaults_Executable.exe = 'ls'
        # -- CONFIGURATION DEFAULTCHANGE STOP

        # -- CONFIGURATION STARTUPSCRIPT START
        slice = jobs.select(status='running')
        print slice
예제 #15
0
def jobExists(jobname):
    from Ganga.GPI import jobs
    slice = jobs.select(name=jobname)
    return len(slice) > 0
예제 #16
0
    def test_Savannah96158(self):
        from Ganga.GPI import Job, jobs

        #The first two tests check the new functionality, the remainder just check that we didn't break existing functionality with this bug-fix
        a = Job()
        a.name = 'TestName'
        tmpList = jobs.select(name='*stN*')
        self.assertEqual(len(tmpList), 1, 'Test 1: jobs.select using wildcard returned unexpected number of results')

        a = Job()
        a.name = 'ekdicjsheeksoawoq1'
        a = Job()
        a.name = 'ekdicjsheeksoawoq2'
        a = Job()
        a.name = 'ekdicjsheeksoawoq3'
        a = Job()
        a.name = 'ekdicjsheeksoawoq4'
        tmpList = jobs.select(name='ekdicjsheeksoawoq?')
        self.assertEqual(len(tmpList), 4, 'Test 2: jobs.select using wildcard returned unexpected number of results')

        jobs.select(1)
        jobs.select(1, 4)
        jobs.select(status='new')
        jobs.select(backend='Local')
        jobs.select(application='Executable')
예제 #17
0
    def rebuild(self, unprepare=True, rmdir=False):
        """Rebuild the shareref table. 
        Clears the shareref table and then rebuilds it by iterating over all Ganga Objects 
        in the Job, Box and Task repositories. If an object has a ShareDir associated with it, 
        that ShareDir is added into the shareref table (the reference counter being incremented 
        accordingly). If called with the optional parameter 'unprepare=False', objects whose
        ShareDirs are not present on disk will not be unprepared. Note that the default behaviour
        is unprepare=True, i.e. the job/application would be unprepared.
        After all Job/Box/Task objects have been checked, the inverse operation is performed, 
        i.e., for each directory in the ShareDir repository, a check is done to ensure there 
        is a matching entry in the shareref table. If not, and the optional parameter 
        'rmdir=True' is set, then the (orphaned) ShareDir will removed from the filesystem. 
        Otherwise, it will be added to the shareref table with a reference count of zero; 
        this results in the directory being deleted upon Ganga exit.
        """
        self._getWriteAccess()
        from Ganga.GPI import jobs, box, tasks
        # clear the shareref table
        self.name = {}
        lookup_input = []

        from Ganga.GPIDev.Lib.File import getSharedPath


        objectlist = []
        for thing in jobs.select():
            objectlist.append({thing: 'job'})
        for thing in box.select():
            objectlist.append({thing: 'box'})
        for thing in tasks.select():
            objectlist.append({thing: 'task'})

        for item in objectlist:
            shortname = None
            try:
                shortname = item.keys()[0].is_prepared.name
            except AttributeError as err:
                logger.debug("Err: %s" % str(err))
                try:
                    shortname = item.keys()[0].application.is_prepared.name
                except AttributeError as err2:
                    logger.debug("Err2: %s" % str(err2))
                    try:
                        shortname = item.keys()[0].analysis.application.is_prepared.name
                    except AttributeError as err3:
                        logger.debug("Err3: %s" % str(err3))
                        pass
            try:
                if shortname is not None and shortname is not True:
                    if os.path.basename(shortname) != shortname:
                        self.to_relative(item.keys()[0].is_prepared)
                    try:
                        numsubjobs = len(item.keys()[0].subjobs.ids())
                    except Exception as err:
                        logger.debug("Path Error: %s" % str(err))
                        Ganga.Utility.logging.log_unknown_exception()
                        numsubjobs = 0
                    self.helper(shortname, unp=unprepare, numsubjobs=numsubjobs)
            except Exception as err:
                logger.debug("-Error: %s" % str(err))
                Ganga.Utility.logging.log_unknown_exception()
                pass

        # here we iterate over the lookup_input list and unprepare as
        # necessary.
        for item in lookup_input:
            logger.info('Unpreparing objects referencing ShareDir %s' % item)
            self.lookup(sharedir=item, unprepare=True)

        # check to see that all sharedirs have an entry in the shareref. Otherwise, set their ref counter to 0
        # so the user is made aware of them at shutdown
        for this_dir in os.listdir(getSharedPath()):
            if this_dir not in self.__getName().keys() and rmdir is False:
                logger.debug("%s isn't referenced by a GangaObject in the Job or Box repository." % this_dir)
                self.__getName()[this_dir] = 0
            elif this_dir not in self.__getName().keys() and rmdir is True:
                logger.debug("%s isn't referenced by a GangaObject in the Job or Box repository. Removing directory." % this_dir)
                shutil.rmtree(os.path.join(getSharedPath(), this_dir))

        self._setDirty()
        self._releaseWriteAccess()
예제 #18
0
    def test_Savannah96158(self):
        from Ganga.GPI import Job, jobs

        #The first two tests check the new functionality, the remainder just check that we didn't break existing functionality with this bug-fix
        a = Job()
        a.name = 'TestName'
        tmpList = jobs.select(name='*stN*')
        self.assertEqual(
            len(tmpList), 1,
            'Test 1: jobs.select using wildcard returned unexpected number of results'
        )

        a = Job()
        a.name = 'ekdicjsheeksoawoq1'
        a = Job()
        a.name = 'ekdicjsheeksoawoq2'
        a = Job()
        a.name = 'ekdicjsheeksoawoq3'
        a = Job()
        a.name = 'ekdicjsheeksoawoq4'
        tmpList = jobs.select(name='ekdicjsheeksoawoq?')
        self.assertEqual(
            len(tmpList), 4,
            'Test 2: jobs.select using wildcard returned unexpected number of results'
        )

        jobs.select(1)
        jobs.select(1, 4)
        jobs.select(status='new')
        jobs.select(backend='Local')
        jobs.select(application='Executable')
예제 #19
0
    def lookup(self, sharedir, unprepare=False):
        """
        Report Job, Box and Task repository items which reference a given ShareDir object. 
        The optional parameter 'unprepare=True' can be set to call the unprepare method 
        on the returned objects.
        """
        from Ganga.GPI import jobs, box, tasks
        objectlist = []
        for thing in jobs.select():
            objectlist.append({thing: 'job'})
        for thing in box.select():
            objectlist.append({thing: 'box'})
        for thing in tasks.select():
            objectlist.append({thing: 'task'})

        run_unp = None
        master_index = 0
        for item in objectlist:
            try:
                item.keys()[0].is_prepared.name
                if item.keys()[0].is_prepared.name == sharedir:
                    logger.info(
                        'ShareDir %s is referenced by item #%s in %s repository'
                        % (sharedir, stripProxy(
                            item.keys()[0])._registry_id, item.values()[0]))
                    run_unp = item.keys()[0]
                    master_index += 1

            except AttributeError as err:
                logger.debug("Err: %s" % str(err))
                try:
                    item.keys()[0].application.is_prepared.name
                    if item.keys()[0].application.is_prepared.name == sharedir:
                        logger.info(
                            'ShareDir %s is referenced by item #%s in %s repository'
                            %
                            (sharedir, stripProxy(item.keys()[0])._registry_id,
                             item.values()[0]))
                        run_unp = item.keys()[0].application
                        master_index += 1
                except AttributeError as err2:
                    logger.debug("Err2: %s" % str(err2))
                    try:
                        item.keys()[0].analysis.application.is_prepared.name
                        if item.keys(
                        )[0].analysis.application.is_prepared.name == sharedir:
                            logger.info(
                                'ShareDir %s is referenced by item #%s in %s repository'
                                % (sharedir, stripProxy(
                                    item.keys()[0])._registry_id,
                                   item.values()[0]))
                            run_unp = item.keys()[0].analysis.application
                            master_index += 1
                    except AttributeError as err3:
                        logger.debug("Err3: %s" % str(err3))
                        pass

            if run_unp is not None and unprepare is True:
                logger.info(
                    'Unpreparing %s repository object #%s associated with ShareDir %s'
                    % (item.values()[0], stripProxy(
                        item.keys()[0])._registry_id, sharedir))
                #                stripProxy(item.keys()[0]).unprepare()
                run_unp.unprepare()
                run_unp = None

        if unprepare is not True:
            logger.info('%s item(s) found referencing ShareDir %s',
                        master_index, sharedir)
예제 #20
0
	def run(self):
		# wait for GPI to become ready (taken from GangaJEM)
		while not self.should_stop():
			try:        
				from Ganga.GPI import jobs
				break
			except: pass
			time.sleep(1)
		from Ganga.GPI import LGI

		# main pilotjob loop
		self.nlgijobs = nlgijobs = InterpoList()
		self.nlgijobs[time.time()-1] = 0
		curpilotsrun = len(jobs.select(status='running'))
		curpilotswait = sum([len(jobs.select(status=s)) for s in ['submitted', 'submitting']])
		self.log.debug('Starting PilotThread main loop')
		self.log.info('pilotjobs: %d queued, %d running'%(curpilotswait, curpilotsrun))
		while not self.should_stop():
			# update historical list of number pilotjobs
			now = time.time()

			# TODO truncate history to avoid memory hog
			if LGI.resource.queued is not None:
				nlgijobs[now] = LGI.resource.queued
			else:
				nlgijobs[now] = 0 # fallback to no jobs to continue baseline
			# first make sure baseline of non-terminating pilotjobs is present
			nbaseline = sum([len(jobs.select(status=s, name='LGIpilot@')) for s in ['running', 'submitted', 'submitting']])
			newbaseline = max(0, config['SchedMin']-nbaseline)
			if newbaseline > 0:
				self.log.info('%d/%d baseline pilotjobs present, submitting %d new', nbaseline, config['SchedMin'], newbaseline)
				submitpilots(newbaseline, False)
				# To avoid to keep submitting new pilot jobs when lgi jobs
				# are waiting, do as if each pilotjob takes care of one lgi
				# job. The max(nlgijobs[<some history>]) below does the rest.
				nlgijobs[now] -= newbaseline

			curpilotsrun = len(jobs.select(status='running'))
			curpilotswait = sum([len(jobs.select(status=s)) for s in ['submitted', 'submitting']])
			curpilots = curpilotsrun + curpilotswait

			# find out how many pilotjobs we want right now
			# main rule: #new_pilotjobs = #lgijobs_waiting/2 - #current_pilots_waiting
			# since number of lgijobs waiting is influenced by running
			# pilots, only submit half of jobs to approach it exponentially,
			# so as to avoid spawning too many pilots during job bursts.
			# The number of pilotjobs that's submitted but not yet running is
			# substracted to avoid over-submission when the grid has long waiting
			# queues.
			newpilots = int((min(nlgijobs[now-config['WaitNew']:])+1)/2 - curpilotswait)

			newpilots = min(newpilots, config['SchedMax']-config['SchedMin']-curpilots)
			newpilots = max(newpilots, 0)

			# some debugging
			if self.log.isEnabledFor('DEBUG'):
				self.log.debug('LGI: %d - %d pending in last %d s --> pilots: %d wanted, %d queued, %d running'%(
					int(min(nlgijobs[now-config['WaitNew']:])), int(max(nlgijobs[now-config['WaitNew']:])),
					config['WaitNew'], newpilots, curpilotswait, curpilotsrun))

			# submit any auto-terminating pilotjobs
			if newpilots > 0:
				self.log.info('Submitting %d new pilotjobs'%(newpilots))
				submitpilots(newpilots)
				# don't submit any new pilotjobs for WaitNew seconds
				#  by setting the number of currently waiting LGI jobs to zero
				#nlgijobs[now] -= newpilots
				nlgijobs[now] = 0

			# cleanup finished jobs
			utcnow = datetime.datetime.utcnow()
			for s in ['finished', 'completed', 'killed', 'failed']:
				for j in jobs.select(status=s):
					delta = utcnow - max(j.time.timestamps.values())
					if delta.seconds/60/60 >= config['KeepJobs']:
						j.remove()
			
			# and wait for next iteration
			while not self.should_stop() and time.time()-now < config['Update']:
				time.sleep(1)
예제 #21
0
    def rebuild(self, unprepare=True, rmdir=False):
        """Rebuild the shareref table. 
        Clears the shareref table and then rebuilds it by iterating over all Ganga Objects 
        in the Job, Box and Task repositories. If an object has a ShareDir associated with it, 
        that ShareDir is added into the shareref table (the reference counter being incremented 
        accordingly). If called with the optional parameter 'unprepare=False', objects whose
        ShareDirs are not present on disk will not be unprepared. Note that the default behaviour
        is unprepare=True, i.e. the job/application would be unprepared.
        After all Job/Box/Task objects have been checked, the inverse operation is performed, 
        i.e., for each directory in the ShareDir repository, a check is done to ensure there 
        is a matching entry in the shareref table. If not, and the optional parameter 
        'rmdir=True' is set, then the (orphaned) ShareDir will removed from the filesystem. 
        Otherwise, it will be added to the shareref table with a reference count of zero; 
        this results in the directory being deleted upon Ganga exit.
        """
        self._getWriteAccess()
        from Ganga.GPI import jobs, box, tasks
        # clear the shareref table
        self.name = {}
        lookup_input = []

        from Ganga.GPIDev.Lib.File import getSharedPath

        objectlist = []
        for thing in jobs.select():
            objectlist.append({thing: 'job'})
        for thing in box.select():
            objectlist.append({thing: 'box'})
        for thing in tasks.select():
            objectlist.append({thing: 'task'})

        for item in objectlist:
            shortname = None
            try:
                shortname = item.keys()[0].is_prepared.name
            except AttributeError as err:
                logger.debug("Err: %s" % str(err))
                try:
                    shortname = item.keys()[0].application.is_prepared.name
                except AttributeError as err2:
                    logger.debug("Err2: %s" % str(err2))
                    try:
                        shortname = item.keys(
                        )[0].analysis.application.is_prepared.name
                    except AttributeError as err3:
                        logger.debug("Err3: %s" % str(err3))
                        pass
            try:
                if shortname is not None and shortname is not True:
                    if os.path.basename(shortname) != shortname:
                        self.to_relative(item.keys()[0].is_prepared)
                    try:
                        numsubjobs = len(item.keys()[0].subjobs.ids())
                    except Exception as err:
                        logger.debug("Path Error: %s" % str(err))
                        Ganga.Utility.logging.log_unknown_exception()
                        numsubjobs = 0
                    self.helper(shortname,
                                unp=unprepare,
                                numsubjobs=numsubjobs)
            except Exception as err:
                logger.debug("-Error: %s" % str(err))
                Ganga.Utility.logging.log_unknown_exception()
                pass

        # here we iterate over the lookup_input list and unprepare as
        # necessary.
        for item in lookup_input:
            logger.info('Unpreparing objects referencing ShareDir %s' % item)
            self.lookup(sharedir=item, unprepare=True)

        # check to see that all sharedirs have an entry in the shareref. Otherwise, set their ref counter to 0
        # so the user is made aware of them at shutdown
        for this_dir in os.listdir(getSharedPath()):
            if this_dir not in self.__getName().keys() and rmdir is False:
                logger.debug(
                    "%s isn't referenced by a GangaObject in the Job or Box repository."
                    % this_dir)
                self.__getName()[this_dir] = 0
            elif this_dir not in self.__getName().keys() and rmdir is True:
                logger.debug(
                    "%s isn't referenced by a GangaObject in the Job or Box repository. Removing directory."
                    % this_dir)
                shutil.rmtree(os.path.join(getSharedPath(), this_dir))

        self._setDirty()
        self._releaseWriteAccess()
예제 #22
0
    def rebuild(self, unprepare=True, rmdir=False):
        """Rebuild the shareref table. 
        Clears the shareref table and then rebuilds it by iterating over all Ganga Objects 
        in the Job, Box and Task repositories. If an object has a ShareDir associated with it, 
        that ShareDir is added into the shareref table (the reference counter being incremented 
        accordingly). If called with the optional parameter 'unprepare=False', objects whose
        ShareDirs are not present on disk will not be unprepared. Note that the default behaviour
        is unprepare=True, i.e. the job/application would be unprepared.
        After all Job/Box/Task objects have been checked, the inverse operation is performed, 
        i.e., for each directory in the ShareDir repository, a check is done to ensure there 
        is a matching entry in the shareref table. If not, and the optional parameter 
        'rmdir=True' is set, then the (orphaned) ShareDir will removed from the filesystem. 
        Otherwise, it will be added to the shareref table with a reference count of zero; 
        this results in the directory being deleted upon Ganga exit.
        """
        self._getWriteAccess()
        from Ganga.GPI import jobs, box, tasks
        #clear the shareref table
        self.name={}
        lookup_input=[]


        def helper(object, unp=True, numsubjobs=0):
            shareddir =  os.path.join(ShareDir._root_shared_path,os.path.basename(object))
            logger.debug('Adding %s to the shareref table.' % shareddir)
            if self.name.has_key(os.path.basename(object)):
                self.name[os.path.basename(object)] +=1
            else:
                self.name[os.path.basename(object)] = 1
            if numsubjobs > 0:
                self.name[os.path.basename(object)] += numsubjobs
            if not os.path.isdir(shareddir) and os.path.basename(object) not in lookup_input:
                logger.info('Shared directory %s not found on disk.' % shareddir)
                if unp == True: 
                    lookup_input.append(os.path.basename(object))


        def to_relative(object):
            logger.info('Absolute ShareDir().name attribute found in Job #%s', object.id)
            logger.info('Converting to relative path and moving associated directory if it exists.')
            try:
                shutil.move(object.is_prepared.name,\
                    os.path.join(ShareDir._root_shared_path,os.path.basename(object.is_prepared.name)))
            except:
                logger.warn('Unable to move directory %s to %s', object.is_prepared.name,\
                    os.path.join(ShareDir._root_shared_path,os.path.basename(object.is_prepared.name)))
            try:
                object._impl.is_prepared.name = os.path.basename(object.is_prepared.name)
            except:
                logger.warn("Unable to convert object's is_prepared.name attribute to a relative path")
        

        objectlist = []
        for thing in jobs.select():
            objectlist.append({thing:'job'})
        for thing in box.select():
            objectlist.append({thing:'box'})
        for thing in tasks.select():
            objectlist.append({thing:'task'})

        for item in objectlist:
            shortname = None
            try:
                shortname = item.keys()[0].is_prepared.name 
            except AttributeError:
                try:
                    shortname = item.keys()[0].application.is_prepared.name 
                except AttributeError:
                    try:
                        shortname = item.keys()[0].analysis.application.is_prepared.name 
                    except AttributeError:
                        pass
            try:
                if shortname is not None and shortname is not True:
                    if os.path.basename(shortname) != shortname:
                        to_relative(item.keys()[0].is_prepared)
                    try:
                        numsubjobs = len(item.keys()[0].subjobs.ids())
                    except:
                        numsubjobs = 0
                    helper(shortname, unp=unprepare, numsubjobs=numsubjobs)
            except:
                pass


        #here we iterate over the lookup_input list and unprepare as necessary.
        for item in lookup_input:
            logger.info('Unpreparing objects referencing ShareDir %s' % item)
            self.lookup(sharedir=item, unprepare=True)

        #check to see that all sharedirs have an entry in the shareref. Otherwise, set their ref counter to 0 
        #so the user is made aware of them at shutdown
        for dir in os.listdir(ShareDir._root_shared_path):
            if not self.name.has_key(dir) and rmdir is False:
                logger.debug("%s isn't referenced by a GangaObject in the Job or Box repository." % dir)
                self.name[dir] = 0
            elif not self.name.has_key(dir) and rmdir is True:
                logger.debug("%s isn't referenced by a GangaObject in the Job or Box repository. Removing directory." % dir)
                shutil.rmtree(os.path.join(ShareDir._root_shared_path,dir))

        self._setDirty()
        self._releaseWriteAccess()