def storageInterface( self, endpoint, protocol ): """ Create the storage interface. """ if self.debug : print 'storageInterface():\n' try: interface = SElement( FullPath(endpoint), protocol ) except ProtocolUnknown, ex: msg = "ERROR : Unable to create interface with %s protocol"%protocol msg += str(ex) raise Exception(msg)
class PostMortemServer(PostMortem): def __init__(self, cfg_params, nj_list): PostMortem.__init__(self, cfg_params, nj_list) # init client server params... CliServerParams(self) self.copyTout = setLcgTimeout() if common.scheduler.name().upper() in ['LSF', 'CAF']: self.copyTout = ' ' if self.storage_path[0] != '/': self.storage_path = '/' + self.storage_path return def collectLogging(self): # get updated status from server try: from StatusServer import StatusServer stat = StatusServer(self.cfg_params) warning_msg = stat.resynchClientSide() if warning_msg is not None: common.logger.info(warning_msg) except: pass #create once storage interaction object seEl = None loc = None try: seEl = SElement(self.storage_name, self.storage_proto, self.storage_port) except Exception, ex: common.logger.debug(str(ex)) msg = "ERROR: Unable to create SE source interface \n" raise CrabException(msg) try: loc = SElement("localhost", "local") except Exception, ex: common.logger.debug(str(ex)) msg = "ERROR: Unable to create destination interface \n" raise CrabException(msg)
def retrieveFiles(self, filesToRetrieve): """ Real get output from server storage """ self.taskuuid = str(common._db.queryTask('name')) common.logger.debug("Task name: " + self.taskuuid) # create the list with the actual filenames remotedir = os.path.join(self.storage_path, self.taskuuid) # list of file to retrieve osbTemplate = remotedir + '/out_files_%s.tgz' osbFiles = [osbTemplate % str(jid) for jid in filesToRetrieve] if self.cfg_params['CRAB.scheduler'].lower() in ["condor_g"]: osbTemplate = remotedir + '/CMSSW_%s.stdout' osbFiles.extend( [osbTemplate % str(jid) for jid in filesToRetrieve]) osbTemplate = remotedir + '/CMSSW_%s.stderr' osbFiles.extend( [osbTemplate % str(jid) for jid in filesToRetrieve]) common.logger.debug("List of OSB files: " + str(osbFiles)) copyHere = self.outDir destTemplate = copyHere + '/out_files_%s.tgz' destFiles = [destTemplate % str(jid) for jid in filesToRetrieve] if self.cfg_params['CRAB.scheduler'].lower() in ["condor_g"]: destTemplate = copyHere + '/CMSSW_%s.stdout' destFiles.extend( [destTemplate % str(jid) for jid in filesToRetrieve]) destTemplate = copyHere + '/CMSSW_%s.stderr' destFiles.extend( [destTemplate % str(jid) for jid in filesToRetrieve]) common.logger.info("Starting retrieving output from server " + str(self.storage_name) + "...") try: seEl = SElement(self.storage_name, self.storage_proto, self.storage_port) except Exception, ex: common.logger.debug(str(ex)) msg = "ERROR : Unable to create SE source interface \n" raise CrabException(msg)
def moveISB_SEAPI(self, listOffiles=[]): ## get task info from BL ## common.logger.debug("Task name: " + self.taskuuid) if len(listOffiles) > 0: isblist = listOffiles else: isblist = str(common._db.queryTask('globalSandbox')).split(',') common.logger.debug("List of ISB files: " + str(isblist)) # init SE interface common.logger.info("Starting sending the project to the storage " + str(self.storage_name) + "...") try: seEl = SElement(self.storage_name, self.storage_proto, self.storage_port) except Exception, ex: common.logger.debug(str(ex)) msg = "ERROR : Unable to create SE destination interface \n" msg += "Project " + self.taskuuid + " not Submitted \n" raise CrabException(msg)
def listRemoteDir(self, endpoint): """ """ #protocol = self.protocolDict[common.scheduler.name().upper()] #### FEDE for rfio and xrootd at caf ####################################################### if (common.scheduler.name().upper() == 'CAF' and str.split(str(endpoint), ':')[0] == 'root'): protocol = 'xrootd' else: protocol = self.protocolDict[common.scheduler.name().upper()] msg = 'protocol = ' + protocol + '\n' msg += 'endpoint = ' + endpoint + '\n' common.logger.debug(msg) try: Storage = SElement(FullPath(string.strip(endpoint)), protocol) except Exception, ex: common.logger.debug(traceback.format_exc()) raise Exception(str(ex))
def collectLogging(self): # get updated status from server try: from StatusServer import StatusServer stat = StatusServer(self.cfg_params) warning_msg = stat.resynchClientSide() if warning_msg is not None: common.logger.info(warning_msg) except: pass #create once storage interaction object seEl = None loc = None try: seEl = SElement(self.storage_name, self.storage_proto, self.storage_port) except Exception, ex: common.logger.debug(str(ex)) msg = "ERROR: Unable to create SE source interface \n" raise CrabException(msg)
class SubmitterServer(Submitter): def __init__(self, cfg_params, parsed_range, val): self.srvCfg = {} self.cfg_params = cfg_params self.submitRange = [] self.credentialType = 'Proxy' self.copyTout = setLcgTimeout() self.extended = int(cfg_params.get('CMSSW.extend', 0)) #wmbs self.type = int(cfg_params.get('WMBS.automation', 0)) self.taskType = 'fullySpecified' if self.type == 1: self.taskType = 'partiallySpecified' if common.scheduler.name().upper() in ['LSF', 'CAF']: self.credentialType = 'Token' self.copyTout = ' ' if common.scheduler.name().upper() == 'REMOTEGLIDEIN': msg = "FATAL ERROR: remoteGlidein scheduler requires use_server=0" raise CrabException(msg) Submitter.__init__(self, cfg_params, parsed_range, val) # init client server params... CliServerParams(self) # path fix if self.storage_path[0] != '/': self.storage_path = '/' + self.storage_path self.taskuuid = str(common._db.queryTask('name')) self.limitJobs = False return def run(self): """ The main method of the class: submit jobs in range self.nj_list """ common.logger.debug("SubmitterServer::run() called") start = time.time() #wmbs self.BuildJobList(self.type) self.submitRange = self.nj_list ## wmbs check = self.checkIfCreate(self.type) if check == 0: self.remotedir = os.path.join(self.storage_path, self.taskuuid) self.manageCredential() # check if it is the first submission isFirstSubmission = common._db.checkIfNeverSubmittedBefore() # standard submission to the server self.performSubmission(isFirstSubmission) stop = time.time() common.logger.debug("Submission Time: " + str(stop - start)) #wmbs if self.type == 0: msg = 'Total of %d jobs submitted' % len(self.submitRange) else: msg = 'Request submitted to the server.' common.logger.info(msg) if int(self.type) == 1: common._db.updateTask_({'jobType': 'Submitted'}) return def moveISB_SEAPI(self, listOffiles=[]): ## get task info from BL ## common.logger.debug("Task name: " + self.taskuuid) if len(listOffiles) > 0: isblist = listOffiles else: isblist = str(common._db.queryTask('globalSandbox')).split(',') common.logger.debug("List of ISB files: " + str(isblist)) # init SE interface common.logger.info("Starting sending the project to the storage " + str(self.storage_name) + "...") try: seEl = SElement(self.storage_name, self.storage_proto, self.storage_port) except Exception, ex: common.logger.debug(str(ex)) msg = "ERROR : Unable to create SE destination interface \n" msg += "Project " + self.taskuuid + " not Submitted \n" raise CrabException(msg) try: loc = SElement("localhost", "local") except Exception, ex: common.logger.debug(str(ex)) msg = "ERROR : Unable to create SE source interface \n" msg += "Project " + self.taskuuid + " not Submitted \n" raise CrabException(msg)
class GetOutputServer(GetOutput, StatusServer): def __init__(self, *args): GetOutput.__init__(self, *args) # init client server params... CliServerParams(self) if self.storage_path[0] != '/': self.storage_path = '/' + self.storage_path self.copyTout = setLcgTimeout() if common.scheduler.name().upper() in ['LSF', 'CAF']: self.copyTout = ' ' return def getOutput(self): # get updated status from server #inherited from StatusServer warning_msg = self.resynchClientSide() if warning_msg is not None: common.logger.info(warning_msg) # understand whether the required output are available self.checkBeforeGet() # retrive files filesAndJodId = {} filesAndJodId.update(self.retrieveFiles(self.list_id)) common.logger.debug("Files to be organized and notified " + str(filesAndJodId)) # load updated task task = common._db.getTask() self.organizeOutput(task, self.list_id) self.notifyRetrievalToServer(filesAndJodId) return def retrieveFiles(self, filesToRetrieve): """ Real get output from server storage """ self.taskuuid = str(common._db.queryTask('name')) common.logger.debug("Task name: " + self.taskuuid) # create the list with the actual filenames remotedir = os.path.join(self.storage_path, self.taskuuid) # list of file to retrieve osbTemplate = remotedir + '/out_files_%s.tgz' osbFiles = [osbTemplate % str(jid) for jid in filesToRetrieve] if self.cfg_params['CRAB.scheduler'].lower() in ["condor_g"]: osbTemplate = remotedir + '/CMSSW_%s.stdout' osbFiles.extend( [osbTemplate % str(jid) for jid in filesToRetrieve]) osbTemplate = remotedir + '/CMSSW_%s.stderr' osbFiles.extend( [osbTemplate % str(jid) for jid in filesToRetrieve]) common.logger.debug("List of OSB files: " + str(osbFiles)) copyHere = self.outDir destTemplate = copyHere + '/out_files_%s.tgz' destFiles = [destTemplate % str(jid) for jid in filesToRetrieve] if self.cfg_params['CRAB.scheduler'].lower() in ["condor_g"]: destTemplate = copyHere + '/CMSSW_%s.stdout' destFiles.extend( [destTemplate % str(jid) for jid in filesToRetrieve]) destTemplate = copyHere + '/CMSSW_%s.stderr' destFiles.extend( [destTemplate % str(jid) for jid in filesToRetrieve]) common.logger.info("Starting retrieving output from server " + str(self.storage_name) + "...") try: seEl = SElement(self.storage_name, self.storage_proto, self.storage_port) except Exception, ex: common.logger.debug(str(ex)) msg = "ERROR : Unable to create SE source interface \n" raise CrabException(msg) try: loc = SElement("localhost", "local") except Exception, ex: common.logger.debug(str(ex)) msg = "ERROR : Unable to create destination interface \n" raise CrabException(msg)