예제 #1
0
    def __init__(self, workDir, setup, esPath, token, experiment, userid, sitename, outputDir=None, yodaToOS=False, threads=10, isDaemon=False):
        super(MVEventStager, self).__init__(workDir, setup, esPath, token, experiment, userid, sitename, outputDir, yodaToOS, threads, isDaemon)

        self.__workDir = workDir
        self.__updateEventRangesDir = os.path.join(self.__workDir, 'MVupdateEventRanges')
        if not os.path.exists(self.__updateEventRangesDir):
            os.makedirs(self.__updateEventRangesDir)
        self.__logFile = os.path.join(workDir, 'EventStager.log')
        self.__setup = setup
        self.__siteMover = objectstoreSiteMover(setup)
        self.__esPath = esPath
        self.__token = token
        self.__experiment = experiment
        self.__outputDir = outputDir

        self.__userid = userid
        self.__sitename = sitename

        self.__report =  getInitialTracingReport(userid=self.__userid, sitename=self.__sitename, dsname=None, eventType="objectstore", analysisJob=False, jobId=None, jobDefId=None, dn=self.__userid)

        self.__eventRanges = {}
        self.__eventRanges_staged = {}
        self.__eventRanges_faileStaged = {}

        self.__eventStager = None
        self.__canFinish = False
        self.__status = 'new'
        self.__threads = threads
        self.__yodaToOS = yodaToOS
        self.__isDaemon = isDaemon

        self.__threadpool = ThreadPool(self.__threads)
        logging.info("Init EventStager workDir %s setup %s esPath %s token %s experiment %s userid %s sitename %s threads %s outputDir %s isDaemond %s" % (self.__workDir, self.__setup, self.__esPath, self.__token, self.__experiment, self.__userid, self.__sitename, self.__threads, self.__outputDir, self.__isDaemon))
예제 #2
0
파일: DroidStager.py 프로젝트: vokac/pilot
    def setup(self, job):
        try:
            self.__jobId = job.get("JobId", None)
            self.__yodaToOS = job.get('yodaToOS', False)
            self.__yodaToZip = job.get('yodaToZip', False)
            self.__zipFileName = job.get('zipFileName', None)
            self.__zipEventRangesName = job.get('zipEventRangesName', None)
            self.__tmpLog.debug("Rank %s: zip file %s" % (self.__rank, self.__zipFileName))
            self.__tmpLog.debug("Rank %s: zip event range file %s" % (self.__rank, self.__zipEventRangesName))
            if self.__zipFileName is None or self.__zipEventRangesName is None:
                self.__tmpLog.debug("Rank %s: either zipFileName(%s) is None or zipEventRanagesName(%s) is None, will not use zip output" % (self.__rank, self.__zipFileName, self.__zipEventRangesName))
                self.__yodaToZip = False
            self.__copyOutputToGlobal =  job.get('copyOutputToGlobal', False)

            if self.__yodaToOS:
                setup = job.get('setup', None)
                self.__esPath = job.get('esPath', None)
                self.__os_bucket_id = job.get('os_bucket_id', None)
                self.__report =  getInitialTracingReport(userid='Yoda', sitename='Yoda', dsname=None, eventType="objectstore", analysisJob=False, jobId=None, jobDefId=None, dn='Yoda')
                self.__siteMover = objectstoreSiteMover(setup, useTimerCommand=False)
                self.__cores = int(job.get('ATHENA_PROC_NUMBER', 1))

                try:
                    self.__stageout_threads = int(job.get('stageout_threads', None))
                except:
                    self.__stageout_threads = self.__cores/8
                self.__tmpLog.debug("Rank %s: start threadpool with %s threads" % (self.__rank, self.__stageout_threads))
                self.__threadpool = ThreadPool(self.__stageout_threads)

        except:
            self.__tmpLog.error("Failed to setup Droid stager: %s" % str(traceback.format_exc()))
예제 #3
0
    def setupStageOutHPCEvent(self):
        if self.__job.prodDBlockTokenForOutput is not None and len(
                self.__job.prodDBlockTokenForOutput
        ) > 0 and self.__job.prodDBlockTokenForOutput[0] != 'NULL':
            siteInfo = getSiteInformation(self.getExperiment())
            objectstore_orig = siteInfo.readpar("objectstore")
            #siteInfo.replaceQueuedataField("objectstore", self.__job.prodDBlockTokenForOutput[0])
            espath = getFilePathForObjectStore(filetype="eventservice")
        else:
            #siteInfo = getSiteInformation(self.getExperiment())
            #objectstore = siteInfo.readpar("objectstore")
            espath = getFilePathForObjectStore(filetype="eventservice")
        self.__espath = getFilePathForObjectStore(filetype="eventservice")
        tolog("EventServer objectstore path: " + espath)

        siteInfo = getSiteInformation(self.getExperiment())
        # get the copy tool
        setup = siteInfo.getCopySetup(stageIn=False)
        tolog("Copy Setup: %s" % (setup))

        dsname, datasetDict = self.getDatasets()
        self.__report = getInitialTracingReport(
            userid=self.__job.prodUserID,
            sitename=self.__jobSite.sitename,
            dsname=dsname,
            eventType="objectstore",
            analysisJob=self.__analysisJob,
            jobId=self.__job.jobId,
            jobDefId=self.__job.jobDefinitionID,
            dn=self.__job.prodUserID)
        self.__siteMover = objectstoreSiteMover(setup)
예제 #4
0
파일: Droid.py 프로젝트: complynx/pilot
    def setup(self, job):
        try:
            self.__jobId = job.get("JobId", None)
            self.__poolFileCatalog = job.get('PoolFileCatalog', None)
            self.__inputFiles = job.get('InputFiles', None)
            self.__copyInputFiles = job.get('CopyInputFiles', False)
            self.__preSetup = job.get('PreSetup', None)
            self.__postRun = job.get('PostRun', None)
            self.__yodaToOS = job.get('yodaToOS', False)
            self.__copyOutputToGlobal =  job.get('copyOutputToGlobal', False)

            if self.__yodaToOS:
                setup = job.get('setup', None)
                self.__esPath = job.get('esPath', None)
                self.__report =  getInitialTracingReport(userid='Yoda', sitename='Yoda', dsname=None, eventType="objectstore", analysisJob=False, jobId=None, jobDefId=None, dn='Yoda')
                self.__siteMover = objectstoreSiteMover(setup, useTimerCommand=False)

            self.__ATHENA_PROC_NUMBER = int(job.get('ATHENA_PROC_NUMBER', 1))
            self.__ATHENA_PROC_NUMBER -= self.reserveCores
            if self.__ATHENA_PROC_NUMBER < 0:
                self.__ATHENA_PROC_NUMBER = 1
            job["AthenaMPCmd"] = "export ATHENA_PROC_NUMBER=" + str(self.__ATHENA_PROC_NUMBER) + "; " + job["AthenaMPCmd"]
            self.__jobWorkingDir = job.get('GlobalWorkingDir', None)
            if self.__jobWorkingDir:
                self.__jobWorkingDir = os.path.join(self.__jobWorkingDir, 'rank_%s' % self.__rank)
                if not os.path.exists(self.__jobWorkingDir):
                    os.makedirs(self.__jobWorkingDir)
                os.chdir(self.__jobWorkingDir)
                logFile = os.path.join(self.__jobWorkingDir, 'Droid.log')
                logging.basicConfig(filename=logFile, level=logging.DEBUG)
                self.__tmpLog = Logger.Logger()

            if self.__copyInputFiles and self.__inputFiles is not None and self.__poolFileCatalog is not None:
                for inputFile in self.__inputFiles:
                    shutil.copy(inputFile, './')

                pfc_name = os.path.basename(self.__poolFileCatalog)
                pfc_name = os.path.join(os.getcwd(), pfc_name)
                pfc_name_back = pfc_name + ".back"
                shutil.copy2(self.__poolFileCatalog, pfc_name_back)
                with open(pfc_name, 'wt') as pfc_out:
                    with open(pfc_name_back, 'rt') as pfc_in:
                        for line in pfc_in:
                            pfc_out.write(line.replace('HPCWORKINGDIR', os.getcwd()))
                    
                job["AthenaMPCmd"] = job["AthenaMPCmd"].replace('HPCWORKINGDIR', os.getcwd())
            
            self.__esJobManager = EventServerJobManager(self.__rank, self.__ATHENA_PROC_NUMBER)
            status, output = self.__esJobManager.preSetup(self.__preSetup)
            if status != 0:
                return False, output
            # self.__esJobManager.initMessageThread(socketname='EventService_EventRanges', context='local')
            # self.__esJobManager.initTokenExtractorProcess(job["TokenExtractCmd"])
            # self.__esJobManager.initAthenaMPProcess(job["AthenaMPCmd"])
            ret = self.__esJobManager.init(socketname='EventService_EventRanges', context='local', athenaMPCmd=job["AthenaMPCmd"], tokenExtractorCmd=job["TokenExtractCmd"])
            return True, None
        except:
            errMsg = "Failed to init EventServerJobManager: %s" % str(traceback.format_exc())
            self.__esJobManager.terminate()
            return False, errMsg
예제 #5
0
    def __init__(self,
                 workDir,
                 setup,
                 esPath,
                 token,
                 experiment,
                 userid,
                 sitename,
                 outputDir=None,
                 yodaToOS=False,
                 threads=10,
                 isDaemon=False):
        super(MVEventStager,
              self).__init__(workDir, setup, esPath, token, experiment, userid,
                             sitename, outputDir, yodaToOS, threads, isDaemon)

        self.__workDir = workDir
        self.__updateEventRangesDir = os.path.join(self.__workDir,
                                                   'MVupdateEventRanges')
        if not os.path.exists(self.__updateEventRangesDir):
            os.makedirs(self.__updateEventRangesDir)
        self.__logFile = os.path.join(workDir, 'EventStager.log')
        self.__setup = setup
        self.__siteMover = objectstoreSiteMover(setup)
        self.__esPath = esPath
        self.__token = token
        self.__experiment = experiment
        self.__outputDir = outputDir

        self.__userid = userid
        self.__sitename = sitename

        self.__report = getInitialTracingReport(userid=self.__userid,
                                                sitename=self.__sitename,
                                                dsname=None,
                                                eventType="objectstore",
                                                analysisJob=False,
                                                jobId=None,
                                                jobDefId=None,
                                                dn=self.__userid)

        self.__eventRanges = {}
        self.__eventRanges_staged = {}
        self.__eventRanges_faileStaged = {}

        self.__eventStager = None
        self.__canFinish = False
        self.__status = 'new'
        self.__threads = threads
        self.__yodaToOS = yodaToOS
        self.__isDaemon = isDaemon

        self.__threadpool = ThreadPool(self.__threads)
        logging.info(
            "Init EventStager workDir %s setup %s esPath %s token %s experiment %s userid %s sitename %s threads %s outputDir %s isDaemond %s"
            % (self.__workDir, self.__setup, self.__esPath, self.__token,
               self.__experiment, self.__userid, self.__sitename,
               self.__threads, self.__outputDir, self.__isDaemon))
예제 #6
0
    def setupStageOutHPCEvent(self):
        if self.__job.prodDBlockTokenForOutput is not None and len(self.__job.prodDBlockTokenForOutput) > 0 and self.__job.prodDBlockTokenForOutput[0] != 'NULL':
            siteInfo = getSiteInformation(self.getExperiment())
            objectstore_orig = siteInfo.readpar("objectstore")
            #siteInfo.replaceQueuedataField("objectstore", self.__job.prodDBlockTokenForOutput[0])
            espath = getFilePathForObjectStore(filetype="eventservice")
        else:
            #siteInfo = getSiteInformation(self.getExperiment())
            #objectstore = siteInfo.readpar("objectstore")
            espath = getFilePathForObjectStore(filetype="eventservice")
        self.__espath = getFilePathForObjectStore(filetype="eventservice")
        tolog("EventServer objectstore path: " + espath)

        siteInfo = getSiteInformation(self.getExperiment())
        # get the copy tool
        setup = siteInfo.getCopySetup(stageIn=False)
        tolog("Copy Setup: %s" % (setup))

        dsname, datasetDict = self.getDatasets()
        self.__report = getInitialTracingReport(userid=self.__job.prodUserID, sitename=self.__jobSite.sitename, dsname=dsname, eventType="objectstore", analysisJob=self.__analysisJob, jobId=self.__job.jobId, jobDefId=self.__job.jobDefinitionID, dn=self.__job.prodUserID)
        self.__siteMover = objectstoreSiteMover(setup)