def doMaster(self): """ This method calls the doNew method for each hostname that exists in the DowntimeCache table of the local database. :return: S_OK / S_ERROR """ # Query DB for all downtimes result = self.rmClient.selectDowntimeCache() if not result['OK']: return result for data in result['Value']: # If already processed don't do it again if data[0] in self.seenHostnames: continue # data[0] contains the hostname gLogger.verbose("Checking if the downtime of %s has been changed" % data[0]) result = self.doNew(data[0]) if not result['OK']: return result self.seenHostnames.add(data[0]) return S_OK()
def initialize( self ): for serviceName in self.__services: gLogger.verbose( "Initializing %s" % serviceName ) result = self.__services[ serviceName ].initialize() if not result[ 'OK' ]: return result return S_OK()
def getLocalProtocols( self ): """ Get the list of all the local access protocols defined for this Storage Element """ if not self.valid: return S_ERROR( self.errorReason ) gLogger.verbose( "StorageElement.getLocalProtocols: Obtaining local protocols for %s." % self.name ) return S_OK( self.localProtocols )
def _rebuildDirectoryUsage( self ): """ Recreate and replenish the Storage Usage tables """ req = "DROP TABLE IF EXISTS FC_DirectoryUsage_backup" result = self.db._update( req ) req = "RENAME TABLE FC_DirectoryUsage TO FC_DirectoryUsage_backup" result = self.db._update( req ) req = "CREATE TABLE `FC_DirectoryUsage` LIKE `FC_DirectoryUsage_backup`" result = self.db._update( req ) if not result['OK']: return result result = self.__rebuildDirectoryUsageLeaves() if not result['OK']: return result result = self.db.dtree.findDir( '/' ) if not result['OK']: return result if not result['Value']: return S_ERROR( 'Directory / not found' ) dirID = result['Value'] result = self.__rebuildDirectoryUsage( dirID ) gLogger.verbose( 'Finished rebuilding Directory Usage' ) return result
def __submitFTSTransfer( self ): """ create and execute glite-transfer-submit CLI command :param self: self reference """ comm = [ 'glite-transfer-submit', '-s', self.ftsServer, '-f', self.surlFile, '-o' ] if self.targetToken: comm += [ '-t', self.targetToken ] if self.sourceToken: comm += [ '-S', self.sourceToken ] if self.__cksmTest: comm.append( "--compare-checksums" ) gLogger.verbose( 'Executing %s' % ' '.join( comm ) ) res = executeGridCommand( '', comm ) os.remove( self.surlFile ) if not res['OK']: return res returnCode, output, errStr = res['Value'] if not returnCode == 0: return S_ERROR( errStr ) guid = output.replace( '\n', '' ) if not checkGuid( guid ): return S_ERROR( 'Wrong GUID format returned' ) self.ftsGUID = guid # if self.priority != 3: # comm = ['glite-transfer-setpriority','-s', self.ftsServer,self.ftsGUID,str(self.priority)] # executeGridCommand('',comm) return res
def _getPoolCatalogs( directory = '' ): patterns = ['*.xml', '*.xml*gz'] omissions = ['\.bak$'] # to be ignored for production files #First obtain valid list of unpacked catalog files in directory poolCatalogList = [] for pattern in patterns: fileList = glob.glob( os.path.join( directory, pattern ) ) for fname in fileList: if fname.endswith( '.bak' ): gLogger.verbose( 'Ignoring BAK file: %s' % fname ) elif tarfile.is_tarfile( fname ): try: gLogger.debug( 'Unpacking catalog XML file %s' % ( os.path.join( directory, fname ) ) ) tarFile = tarfile.open( os.path.join( directory, fname ), 'r' ) for member in tarFile.getmembers(): tarFile.extract( member, directory ) poolCatalogList.append( os.path.join( directory, member.name ) ) except Exception, x : gLogger.error( 'Could not untar with exception', ' %s: %s' % ( fname, str( x ) ) ) else: poolCatalogList.append( fname )
def __setParam( self, value ): change = False if self.item_called in self.paramTypes: if self.paramValues[self.item_called] != value: if type( value ) in self.paramTypes[self.item_called]: change = True else: raise TypeError( "%s %s %s expected one of %s" % ( self.item_called, value, type( value ), self.paramTypes[self.item_called] ) ) else: if self.item_called not in self.paramValues: change = True else: if self.paramValues[self.item_called] != value: change = True if not change: gLogger.verbose( "No change of parameter %s required" % self.item_called ) else: gLogger.verbose( "Parameter %s to be changed" % self.item_called ) transID = self.paramValues['TransformationID'] if self.exists and transID: res = self.transClient.setTransformationParameter( transID, self.item_called, value ) if not res['OK']: return res self.paramValues[self.item_called] = value return S_OK()
def execute(self): """ Main execution method """ delta = time.time() - self.last_update if delta > self.reportPeriod: result = self.jobDB.getCounters("Jobs", ["Status", "Site"], {}, "") if not result["OK"]: return S_ERROR("Failed to get data from the Job Database") self.resultDB = result["Value"] self.last_update = time.time() totalDict = {} for status in MONITOR_STATUS: totalDict[status] = 0 for row in self.resultDB: site = row[0]["Site"] status = row[0]["Status"] count = row[1] if site in MONITOR_SITES and status in MONITOR_STATUS: gLogger.verbose("Adding mark %s-%s: " % (status, site) + str(count)) gMonitor.addMark("%s-%s" % (status, site), count) if status in totalDict: totalDict[status] += count for status in MONITOR_STATUS: gLogger.verbose("Adding mark %s-All sites: " % status + str(totalDict[status])) gMonitor.addMark("%s-All sites" % status, totalDict[status]) return S_OK()
def findGenericPilotCredentials(vo=False, group=False, pilotDN='', pilotGroup=''): """ Looks into the Operations/<>/Pilot section of CS to find the pilot credentials. Then check if the user has a registered proxy in ProxyManager. if pilotDN or pilotGroup are specified, use them """ if not group and not vo: return S_ERROR("Need a group or a VO to determine the Generic pilot credentials") if not vo: vo = Registry.getVOForGroup(group) if not vo: return S_ERROR("Group %s does not have a VO associated" % group) opsHelper = Operations.Operations(vo=vo) if not pilotGroup: pilotGroup = opsHelper.getValue("Pilot/GenericPilotGroup", "") if not pilotDN: pilotDN = opsHelper.getValue("Pilot/GenericPilotDN", "") if not pilotDN: pilotUser = opsHelper.getValue("Pilot/GenericPilotUser", "") if pilotUser: result = Registry.getDNForUsername(pilotUser) if result['OK']: pilotDN = result['Value'][0] if pilotDN and pilotGroup: gLogger.verbose("Pilot credentials: %s@%s" % (pilotDN, pilotGroup)) result = gProxyManager.userHasProxy(pilotDN, pilotGroup, 86400) if not result['OK']: return S_ERROR("%s@%s has no proxy in ProxyManager") return S_OK((pilotDN, pilotGroup)) if pilotDN: return S_ERROR("DN %s does not have group %s" % (pilotDN, pilotGroup)) return S_ERROR("No generic proxy in the Proxy Manager with groups %s" % pilotGroup)
def initialize( self ): #Build the URLs self._url = self._cfg.getURL() if not self._url: return S_ERROR( "Could not build service URL for %s" % self._name ) gLogger.verbose( "Service URL is %s" % self._url ) #Load handler result = self._loadHandlerInit() if not result[ 'OK' ]: return result self._handler = result[ 'Value' ] #Initialize lock manager self._lockManager = LockManager( self._cfg.getMaxWaitingPetitions() ) self._initMonitoring() self._threadPool = ThreadPool( max( 1, self._cfg.getMinThreads() ), max( 0, self._cfg.getMaxThreads() ), self._cfg.getMaxWaitingPetitions() ) self._threadPool.daemonize() self._msgBroker = MessageBroker( "%sMSB" % self._name, threadPool = self._threadPool ) #Create static dict self._serviceInfoDict = { 'serviceName' : self._name, 'serviceSectionPath' : PathFinder.getServiceSection( self._name ), 'URL' : self._cfg.getURL(), 'messageSender' : MessageSender( self._name, self._msgBroker ), 'validNames' : self._validNames, 'csPaths' : [ PathFinder.getServiceSection( svcName ) for svcName in self._validNames ] } #Call static initialization function try: self._handler[ 'class' ]._rh__initializeClass( dict( self._serviceInfoDict ), self._lockManager, self._msgBroker, self._monitor ) if self._handler[ 'init' ]: for initFunc in self._handler[ 'init' ]: gLogger.verbose( "Executing initialization function" ) try: result = initFunc( dict( self._serviceInfoDict ) ) except Exception as excp: gLogger.exception( "Exception while calling initialization function", lException = excp ) return S_ERROR( "Exception while calling initialization function: %s" % str( excp ) ) if not isReturnStructure( result ): return S_ERROR( "Service initialization function %s must return S_OK/S_ERROR" % initFunc ) if not result[ 'OK' ]: return S_ERROR( "Error while initializing %s: %s" % ( self._name, result[ 'Message' ] ) ) except Exception as e: errMsg = "Exception while initializing %s" % self._name gLogger.exception( e ) gLogger.exception( errMsg ) return S_ERROR( errMsg ) #Load actions after the handler has initialized itself result = self._loadActions() if not result[ 'OK' ]: return result self._actions = result[ 'Value' ] gThreadScheduler.addPeriodicTask( 30, self.__reportThreadPoolContents ) return S_OK()
def __checkoutFromSVN( self, moduleName = None, sourceURL = None, tagVersion = None ): """ This method checkout a given tag from a SVN repository. Note: we can checkout any project form a SVN repository :param str moduleName: The name of the Module :param str sourceURL: The code repository :param str tagVersion: the tag for example: v4r3p6 """ if not moduleName: moduleName = self.params.name if not sourceURL: sourceURL = self.params.sourceURL if not tagVersion: tagVersion = self.params.version cmd = "svn export --trust-server-cert --non-interactive '%s/%s' '%s'" % ( sourceURL, tagVersion, os.path.join( self.params.destination, moduleName ) ) gLogger.verbose( "Executing: %s" % cmd ) result = Subprocess.systemCall( 900, shlex.split(cmd) ) if not result[ 'OK' ]: return S_ERROR( "Error while retrieving sources from SVN: %s" % result[ 'Message' ] ) exitStatus, stdData, errData = result[ 'Value' ] if exitStatus: return S_ERROR( "Error while retrieving sources from SVN: %s" % "\n".join( [ stdData, errData ] ) ) return S_OK()
def setPlotsLocation( self, plotsDir ): self.plotsLocation = plotsDir for plot in os.listdir( self.plotsLocation ): if plot.find( ".png" ) > 0: plotLocation = "%s/%s" % ( self.plotsLocation, plot ) gLogger.verbose( "Purging %s" % plotLocation ) os.unlink( plotLocation )
def __createListeners(self): for serviceName in self.__services: svcCfg = self.__services[serviceName].getConfig() protocol = svcCfg.getProtocol() port = svcCfg.getPort() if not port: return S_ERROR("No port defined for service %s" % serviceName) if protocol not in gProtocolDict: return S_ERROR("Protocol %s is not known for service %s" % (protocol, serviceName)) self.__listeningConnections[serviceName] = {'port': port, 'protocol': protocol} transportArgs = {} for kw in ServiceReactor.__transportExtraKeywords: value = svcCfg.getOption(kw) if value: ikw = ServiceReactor.__transportExtraKeywords[kw] if ikw: kw = ikw if kw == 'timeout': value = int(value) transportArgs[kw] = value gLogger.verbose("Initializing %s transport" % protocol, svcCfg.getURL()) transport = gProtocolDict[protocol]['transport'](("", port), bServerMode=True, **transportArgs) retVal = transport.initAsServer() if not retVal['OK']: return S_ERROR("Cannot start listening connection for service %s: %s" % (serviceName, retVal['Message'])) self.__listeningConnections[serviceName]['transport'] = transport self.__listeningConnections[serviceName]['socket'] = transport.getSocket() return S_OK()
def export_sendSMS( self, userName, body, fromAddress ): """ Send an SMS with supplied body to the specified DIRAC user using the Mail utility via an SMS switch. """ gLogger.verbose( 'Received signal to send the following SMS to %s:\n%s' % ( userName, body ) ) mobile = gConfig.getValue( '/Registry/Users/%s/Mobile' % userName, '' ) if not mobile: return S_ERROR( 'No registered mobile number for %s' % userName ) csSection = PathFinder.getServiceSection( 'Framework/Notification' ) smsSwitch = gConfig.getValue( '%s/SMSSwitch' % csSection, '' ) if not smsSwitch: return S_ERROR( 'No SMS switch is defined in CS path %s/SMSSwitch' % csSection ) address = '%s@%s' % ( mobile, smsSwitch ) subject = 'DIRAC SMS' m = Mail() m._subject = subject m._message = body m._mailAddress = address if not fromAddress == 'None': m._fromAddress = fromAddress result = m._send() if not result['OK']: gLogger.warn( 'Could not send SMS to %s with the following message:\n%s' % ( userName, result['Message'] ) ) else: gLogger.info( 'SMS sent successfully to %s ' % ( userName ) ) gLogger.debug( result['Value'] ) return result
def getTransformationTasks( self, condDict = None, older = None, newer = None, timeStamp = None, orderAttribute = None, limit = 10000, inputVector = False ): """ gets all the transformation tasks for a transformation, incrementally. "limit" here is just used to determine the offset. """ rpcClient = self._getRPC() transformationTasks = [] if condDict is None: condDict = {} if timeStamp is None: timeStamp = 'CreationTime' # getting transformationFiles - incrementally offsetToApply = 0 while True: res = rpcClient.getTransformationTasks( condDict, older, newer, timeStamp, orderAttribute, limit, inputVector, offsetToApply ) if not res['OK']: return res else: gLogger.verbose( "Result for limit %d, offset %d: %d" % ( limit, offsetToApply, len( res['Value'] ) ) ) if res['Value']: transformationTasks = transformationTasks + res['Value'] offsetToApply += limit if len( res['Value'] ) < limit: break return S_OK( transformationTasks )
def __init__(self, vo=None): """ c'tor :param str vo: name of the virtual organization (community) """ if vo is None: vo = getVO() if not vo: raise Exception('No VO name given') self.vo = vo self.vomsVO = getVOOption(vo, "VOMSName") if not self.vomsVO: raise Exception("Can not get VOMS name for VO %s" % vo) self.urls = [] result = gConfig.getSections('/Registry/VO/%s/VOMSServers' % self.vo) if result['OK']: for server in result['Value']: gLogger.verbose("Adding 'https://%s:8443/voms/%s/apiv2/users'" % (server, self.vomsVO)) self.urls.append('https://%s:8443/voms/%s/apiv2/users' % (server, self.vomsVO)) else: gLogger.error("Section '/Registry/VO/%s/VOMSServers' not found" % self.vo) self.userDict = None
def _syncResources( self ): ''' Sync resources: compares CS with DB and does the necessary modifications. ( StorageElements, FTS, FileCatalogs and ComputingElements ) ''' gLogger.verbose( '-- Synchronizing Resources --' ) gLogger.verbose( '-> StorageElements' ) ses = self.__syncStorageElements() if not ses[ 'OK' ]: gLogger.error( ses[ 'Message' ] ) gLogger.verbose( '-> FTS' ) fts = self.__syncFTS() if not fts[ 'OK' ]: gLogger.error( fts[ 'Message' ] ) gLogger.verbose( '-> FileCatalogs' ) fileCatalogs = self.__syncFileCatalogs() if not fileCatalogs[ 'OK' ]: gLogger.error( fileCatalogs[ 'Message' ] ) gLogger.verbose( '-> ComputingElements' ) computingElements = self.__syncComputingElements() if not computingElements[ 'OK' ]: gLogger.error( computingElements[ 'Message' ] ) #FIXME: VOMS return S_OK()
def __getClientInitArgs( self, trid, proposalTuple ): clientTransport = self._transportPool.get( trid ) #Get the peer credentials credDict = clientTransport.getConnectingCredentials() if 'x509Chain' not in credDict: return S_OK() cKey = ( credDict[ 'DN' ], credDict.get( 'group', False ), credDict.get( 'extraCredentials', False ), credDict[ 'isLimitedProxy' ] ) dP = self.__delegatedCredentials.get( cKey, 3600 ) idString = self._createIdentityString( credDict, clientTransport ) if dP: gLogger.verbose( "Proxy for %s is cached" % idString ) return S_OK( dP ) result = self.__requestDelegation( clientTransport, credDict ) if not result[ 'OK' ]: gLogger.warn( "Could not get proxy for %s: %s" % ( idString, result[ 'Message' ] ) ) return result delChain = result[ 'Value' ] delegatedChain = delChain.dumpAllToString()[ 'Value' ] secsLeft = delChain.getRemainingSecs()[ 'Value' ] - 1 clientInitArgs = { BaseClient.KW_SETUP : proposalTuple[0][1], BaseClient.KW_TIMEOUT : 600, BaseClient.KW_IGNORE_GATEWAYS : True, BaseClient.KW_USE_CERTIFICATES : False, BaseClient.KW_PROXY_STRING : delegatedChain } if BaseClient.KW_EXTRA_CREDENTIALS in credDict: clientInitArgs[ BaseClient.KW_EXTRA_CREDENTIALS ] = credDict[ BaseClient.KW_EXTRA_CREDENTIALS ] gLogger.warn( "Got delegated proxy for %s: %s secs left" % ( idString, secsLeft ) ) self.__delegatedCredentials.add( cKey, secsLeft, clientInitArgs ) return S_OK( clientInitArgs )
def __fillOptimizers( self, opName ): gLogger.verbose( "Filling %s optimizers" % opName ) while True: #Get first job if any jid = self.__jobOpQueue.popJob( opName ) if jid == None: gLogger.verbose( "No more jobs for optimizers %s" % opName ) #No more jobs return S_OK() result = self.__bookJobInOptimizer( opName, jid ) if not result[ 'OK' ]: gLogger.info( "No empty %s optimizers now" % opName ) self.__jobOpQueue.pushJob( opName, jid, ahead = True ) return S_OK() opState = result[ 'Value' ] result = self.__jobsState.getJob( jid ) if not result[ 'OK' ]: gLogger.warn( "Could not load job data", "for jid %s" % jid ) self.__jobsState.forgetJob( jid ) continue job = result[ 'Value' ] result = opState.requestOptimization( job ) if not result[ 'OK' ]: self.__jobOpQueue.pushJob( opName, jid, ahead = True ) gLogger.warn( "Could not request optimization", result[ 'Message' ] ) self.optimizerDisconnected( opState.getTrid() ) return S_OK()
def forgetJob( self, jid ): gLogger.verbose( "Forgetting about jid %s" % jid ) self.__lock.acquire() try: return self.__innerForgetJob( jid ) finally: self.__lock.release()
def __uploadInputSandbox( self, classAdJob, jobDescriptionObject = None ): """Checks the validity of the job Input Sandbox. The function returns the list of Input Sandbox files. The total volume of the input sandbox is evaluated """ inputSandbox = self.__getInputSandboxEntries( classAdJob ) realFiles = [] badFiles = [] diskFiles = [] for isFile in inputSandbox: valid = True for tag in ( 'lfn:', 'LFN:', 'SB:', '%s' ): # in case of parametric input sandbox, there is %s passed, so have to ignore it also if isFile.find( tag ) == 0: valid = False break if valid: realFiles.append( isFile ) stringIOFiles = [] stringIOFilesSize = 0 if jobDescriptionObject is not None: if isinstance( jobDescriptionObject, StringIO.StringIO ): stringIOFiles = [jobDescriptionObject] stringIOFilesSize = len( jobDescriptionObject.buf ) gLogger.debug( "Size of the stringIOFiles: " + str( stringIOFilesSize ) ) else: return S_ERROR( "jobDescriptionObject is not a StringIO object" ) # Check real files for isFile in realFiles: if not os.path.exists( isFile ): # we are passing in real files, we expect them to be on disk badFiles.append( isFile ) gLogger.warn( "inputSandbox file/directory " + isFile + " not found. Keep looking for the others" ) continue diskFiles.append( isFile ) diskFilesSize = File.getGlobbedTotalSize( diskFiles ) gLogger.debug( "Size of the diskFiles: " + str( diskFilesSize ) ) totalSize = diskFilesSize + stringIOFilesSize gLogger.verbose( "Total size of the inputSandbox: " + str( totalSize ) ) okFiles = stringIOFiles + diskFiles if badFiles: result = S_ERROR( 'Input Sandbox is not valid' ) result['BadFile'] = badFiles result['TotalSize'] = totalSize return result if okFiles: if not self.sandboxClient: self.sandboxClient = SandboxStoreClient( useCertificates = self.useCertificates ) result = self.sandboxClient.uploadFilesAsSandbox( okFiles ) if not result[ 'OK' ]: return result inputSandbox.append( result[ 'Value' ] ) classAdJob.insertAttributeVectorString( "InputSandbox", inputSandbox ) return S_OK()
def __checkoutFromGit( self ): if self.params.vcsBranch: brCmr = "-b %s" % self.params.vcsBranch else: brCmr = "" fDirName = os.path.join( self.params.destination, self.params.name ) cmd = "git clone %s '%s' '%s'" % ( brCmr, self.params.sourceURL, fDirName ) gLogger.verbose( "Executing: %s" % cmd ) if os.system( cmd ): return S_ERROR( "Error while retrieving sources from git" ) branchName = "DIRACDistribution-%s" % os.getpid() cmd = "( cd '%s'; git checkout -b '%s' '%s' )" % ( fDirName, branchName, self.params.version ) gLogger.verbose( "Executing: %s" % cmd ) exportRes = os.system( cmd ) shutil.rmtree( "%s/.git" % fDirName ) if exportRes: return S_ERROR( "Error while exporting from git" ) return S_OK()
def extendTransformation( self, transID, maxTasks ): gLogger.info( "Considering extension of transformation %d" % transID ) # Get the current count of tasks submitted for this transformation res = self.transClient.getTransformationTaskStats( transID ) if not res['OK']: if res['Message'] != 'No records found': gLogger.error( "Failed to get task statistics", "%s %s" % ( transID, res['Message'] ) ) return res else: statusDict = {} else: statusDict = res['Value'] gLogger.verbose( "Current task count for transformation %d" % transID ) for status in sortList( statusDict.keys() ): statusCount = statusDict[status] gLogger.verbose( "%s : %s" % ( status.ljust( 20 ), str( statusCount ).rjust( 8 ) ) ) # Determine the number of tasks to be created numberOfTasks = self.calculateTaskNumber( maxTasks, statusDict ) if not numberOfTasks: gLogger.info( "No tasks required for transformation %d" % transID ) return S_OK() # Extend the transformation by the determined number of tasks res = self.transClient.extendTransformation( transID, numberOfTasks ) if not res['OK']: gLogger.error( "Failed to extend transformation", "%s %s" % ( transID, res['Message'] ) ) return res gLogger.info( "Successfully extended transformation %d by %d tasks" % ( transID, numberOfTasks ) ) return S_OK()
def updateTaskStatus(self): gLogger.info("updateTaskStatus: Updating the Status of tasks") # Get the transformations to be updated status = self.am_getOption('UpdateTasksStatus',['Active','Completing','Stopped']) res = self._selectTransformations(transType=self.transType,status=status,agentType=[]) if not res['OK']: return res for transformation in res['Value']: transID = transformation['TransformationID'] # Get the tasks which are in a UPDATE state updateStatus = self.am_getOption('TaskUpdateStatus',['Checking','Deleted','Killed','Staging','Stalled','Matched','Rescheduled','Completed','Submitted','Received','Waiting','Running']) condDict = {"TransformationID":transID,"ExternalStatus":updateStatus} timeStamp = str(datetime.datetime.utcnow() - datetime.timedelta(minutes=10)) res = self.transClient.getTransformationTasks(condDict=condDict,older=timeStamp, timeStamp='LastUpdateTime') if not res['OK']: gLogger.error("updateTaskStatus: Failed to get tasks to update for transformation", "%s %s" % (transID,res['Message'])) continue if not res['Value']: gLogger.verbose("updateTaskStatus: No tasks found to update for transformation %s" % transID) continue res = self.getSubmittedTaskStatus(res['Value']) if not res['OK']: gLogger.error("updateTaskStatus: Failed to get updated task statuses for transformation", "%s %s" % (transID,res['Message'])) continue statusDict = res['Value'] for status in sortList(statusDict.keys()): taskIDs = statusDict[status] gLogger.info("updateTaskStatus: Updating %d task(s) from transformation %d to %s" % (len(taskIDs),transID,status)) res = self.transClient.setTaskStatus(transID,taskIDs,status) if not res['OK']: gLogger.error("updateTaskStatus: Failed to update task status for transformation", "%s %s" % (transID,res['Message'])) gLogger.info("updateTaskStatus: Transformation task status update complete") return S_OK()
def cleanUpLFNPath( lfn ): """ Normalise LFNs """ gLogger.debug("LFN before Cleanup", lfn) lfn = posixpath.normpath(lfn) gLogger.verbose("LFN after Cleanup", lfn) return lfn
def __checkoutFromGit(self): if self.params.vcsBranch: brCmr = "-b %s" % self.params.vcsBranch else: brCmr = "" fDirName = os.path.join(self.params.destination, self.params.name) cmd = "git clone %s '%s' '%s'" % (brCmr, self.params.sourceURL, fDirName) gLogger.verbose("Executing: %s" % cmd) if os.system(cmd): return S_ERROR("Error while retrieving sources from git") branchName = "DIRACDistribution-%s" % os.getpid() isTagCmd = "( cd '%s'; git tag -l | grep '%s' )" % (fDirName, self.params.version) if os.system(isTagCmd): # No tag found, assume branch branchSource = "origin/%s" % self.params.version else: branchSource = self.params.version cmd = "( cd '%s'; git checkout -b '%s' '%s' )" % (fDirName, branchName, branchSource) gLogger.verbose("Executing: %s" % cmd) exportRes = os.system(cmd) # Add the keyword substitution gLogger.notice("Replacing keywords (can take a while)...") self.replaceKeywordsWithGit(fDirName) shutil.rmtree("%s/.git" % fDirName) if exportRes: return S_ERROR("Error while exporting from git") return S_OK()
def setGraphsLocation( self, graphsDir ): self.graphsLocation = graphsDir for graphName in os.listdir( self.graphsLocation ): if graphName.find( ".png" ) > 0: graphLocation = "%s/%s" % ( self.graphsLocation, graphName ) gLogger.verbose( "Purging %s" % graphLocation ) os.unlink( graphLocation )
def getLogPath(paramDict, basePath=None): """ Can construct log file paths even if job fails e.g. no output files available. :param dict paramDict: dictionary with at least the keys ``PRODUCTION_ID``, ``JOB_ID``, ``LogFilePath`` :param str basePath: Optional, base path for the log file failover, of not set LogFilePath from paramDict is used as a base :returns: S_OK with dict with LogFilePath and LogTargetPath """ result = checkForMandatoryKeys(paramDict, ['PRODUCTION_ID', 'JOB_ID', 'LogFilePath']) if not result['OK']: return result productionID = paramDict['PRODUCTION_ID'] jobID = paramDict['JOB_ID'] logFileName = "%s_%s.tar" %( str(productionID).zfill(8), str(int(jobID)).zfill(4) ) if basePath: logTargetPath = [ cleanUpLFNPath( os.path.join( basePath, "LOG", str(productionID).zfill(8), logFileName ) ) ] else: #need to built logPath from logFilePath, as it's not there, and must be as in method above logPathtemp = cleanUpLFNPath(paramDict['LogFilePath']).split("/") logPath = "/"+os.path.join(*logPathtemp[0:-1]) logTargetPath = ['%s/%s_%s.tar' % ( logPath, str(productionID).zfill(8), str(int(jobID)).zfill(3))] #this is not doing anything except return the same string as was passed into the function logFilePath = paramDict['LogFilePath'] gLogger.verbose('Log file path is: %s' % logFilePath) gLogger.verbose('Log target path is: %s' % logTargetPath) jobOutputs = {'LogFilePath' : logFilePath, 'LogTargetPath' : logTargetPath} return S_OK(jobOutputs)
def __setParam(self, value): change = False if self.item_called in self.paramTypes: oldValue = self.paramValues[self.item_called] if oldValue != value: if type(value) in self.paramTypes[self.item_called]: change = True else: raise TypeError, "%s %s %s expected one of %s" % ( self.item_called, value, type(value), self.paramTypes[self.item_called], ) if not self.item_called in self.paramTypes.keys(): if not self.paramValues.has_key(self.item_called): change = True else: oldValue = self.paramValues[self.item_called] if oldValue != value: change = True if not change: gLogger.verbose("No change of parameter %s required" % self.item_called) else: gLogger.verbose("Parameter %s to be changed" % self.item_called) transID = self.paramValues["TransformationID"] if self.exists and transID: res = self.transClient.setTransformationParameter(transID, self.item_called, value) if not res["OK"]: return res self.paramValues[self.item_called] = value return S_OK()
def isSubRequestDone( self, ind, rType ): """ Check if the request contains more operations to be performed """ if not self.subRequests.has_key( rType ): return S_ERROR( "No requests of type specified found." ) elif len( self.subRequests[rType] ) <= ind: return S_ERROR( "Subrequest index is out of range." ) else: status = self.getSubRequestAttributeValue( ind, rType, "Status" )['Value'] if status == 'Done': return S_OK( 1 ) files = self.getSubRequestFiles( ind, rType )['Value'] for rFile in files: if not rFile.has_key( 'Status' ): gLogger.error( "!!! The file has no status information !!!" ) gLogger.error( "Ind:%s Type:%s" % ( ind, rType ), self.toXML()['Value'] ) elif rFile['Status'] not in ( 'Done', 'Failed' ): gLogger.verbose( 'Found file in a non-Done or non-Failed state' ) return S_OK( 0 ) datasets = self.getSubRequestDatasets( ind, rType )['Value'] for dataset in datasets: if dataset['Status'] != 'Done': return S_OK( 0 ) if files or datasets: return S_OK( 1 ) else: return S_OK( 0 )
def getType(fileNames, directory=''): """ This function searches the directory for POOL XML catalog files and extracts the type of the pfn. fileNames can be a string or a list, directory defaults to PWD. """ if not directory: directory = os.getcwd() if not os.path.isdir(directory): return S_ERROR('%s is not a directory' % directory) if not isinstance(fileNames, list): fileNames = [fileNames] gLogger.verbose('Will look for POOL XML Catalog file types in %s for %s' % (directory, ', '.join(fileNames))) finalCatList = _getPoolCatalogs(directory) # Create POOL catalog with final list of catalog files and extract GUIDs generated = [] pfnTypes = {} catalog = PoolXMLCatalog(finalCatList) for fname in fileNames: typeFile = str(catalog.getTypeByPfn(fname)) if not typeFile: typeFile = 'ROOT_All' generated.append(fname) pfnTypes[fname] = typeFile if not generated: gLogger.info('Found Types from POOL XML Catalogue for all files: %s' % ', '.join(fileNames)) else: gLogger.info( 'GUIDs not found from POOL XML Catalogue (and were generated) for: %s' % ', '.join(generated)) result = S_OK(pfnTypes) result['directory'] = directory result['generated'] = generated return result
def __sendAccounting(self, regSuc, regTotal, regTime, transEndTime): transSuc = 0 transSize = 0 missingSize = [] for lfn in self.fileDict.keys(): if self.fileDict[lfn].get('Status') == 'Finished': transSuc += 1 if not self.catalogMetadata.has_key(lfn): missingSize.append(lfn) if missingSize: self.__updateMetadataCache(missingSize) for lfn in self.fileDict.keys(): if self.fileDict[lfn].get('Status') == 'Finished': transSize += self.catalogMetadata[lfn]['Size'] transTotal = 0 for state in (self.statusSummary.keys()): transTotal += self.statusSummary[state] submitTime = fromString(self.submitTime) endTime = fromString(transEndTime) oAccounting = DataOperation() #oAccounting.setEndTime(endTime) oAccounting.setEndTime(transEndTime) oAccounting.setStartTime(submitTime) accountingDict = {} accountingDict['OperationType'] = 'replicateAndRegister' accountingDict['User'] = '******' accountingDict['Protocol'] = 'FTS' accountingDict['RegistrationTime'] = regTime accountingDict['RegistrationOK'] = regSuc accountingDict['RegistrationTotal'] = regTotal accountingDict['TransferOK'] = transSuc accountingDict['TransferTotal'] = transTotal accountingDict['TransferSize'] = transSize accountingDict['FinalStatus'] = self.requestStatus accountingDict['Source'] = self.sourceSE accountingDict['Destination'] = self.targetSE c = transEndTime - submitTime transferTime = c.days * 86400 + c.seconds accountingDict['TransferTime'] = transferTime oAccounting.setValuesFromDict(accountingDict) gLogger.verbose("Attempting to commit accounting message...") oAccounting.commit() gLogger.verbose("...committed.") return S_OK()
def __init__(self, host, port, user=None, password=None, indexPrefix='', useSSL=True): """ c'tor :param self: self reference :param str host: name of the database for example: MonitoringDB :param str port: The full name of the database for example: 'Monitoring/MonitoringDB' :param str user: user name to access the db :param str password: if the db is password protected we need to provide a password :param str indexPrefix: it is the indexPrefix used to get all indexes :param bool useSSL: We can disable using secure connection. By default we use secure connection. """ self.__indexPrefix = indexPrefix self._connected = False if user and password: gLogger.debug("Specified username and password") self.__url = "https://%s:%s@%s:%d" % (user, password, host, port) else: gLogger.debug("Username and password not specified") self.__url = "http://%s:%d" % (host, port) gLogger.verbose("Connecting to %s:%s, useSSL = %s" % (host, port, useSSL)) if useSSL: bd = BundleDeliveryClient() retVal = bd.getCAs() casFile = None if not retVal['OK']: gLogger.error("CAs file does not exists:", retVal['Message']) casFile = certifi.where() else: casFile = retVal['Value'] self.__client = Elasticsearch(self.__url, timeout=self.__timeout, use_ssl=True, verify_certs=True, ca_certs=casFile) else: self.__client = Elasticsearch(self.__url, timeout=self.__timeout) gLogger.verbose("ElasticSearchDB URL: %s" % self.__url) self.__tryToConnect()
def __compileWebApp(self): """ This method is compile the DIRAC web framework """ dctArgs = [] if self.params.extjspath: dctArgs.append("-P '%s'" % self.params.extjspath) destDir = self.params.destination dctArgs.append("-D '%s'" % destDir) scriptName = os.path.join("%s/WebAppDIRAC/scripts/" % destDir, "dirac-webapp-compile.py") if not os.path.isfile(scriptName): return S_ERROR("%s file does not exists!" % scriptName) dctArgs.append("-n '%s'" % self.params.name) cmd = "'%s' %s" % (scriptName, " ".join(dctArgs)) gLogger.verbose("Executing %s" % cmd) if os.system(cmd) != 0: return S_ERROR("Failed to execute the command") return S_OK()
def commit(self): """ Send the registers in a bundle mode """ rpcClient = self.__getRPCClient() sent = 0 # create a local reference and prevent other running commits # to take the same data second time self.__registersListLock.acquire() registersList = self.__registersList self.__registersList = [] self.__registersListLock.release() try: while registersList: registersToSend = registersList[:self.__maxRecordsInABundle] retVal = rpcClient.commitRegisters(registersToSend) if retVal['OK']: self.__lastSuccessfulCommit = time.time() else: gLogger.error('Error sending accounting record', retVal['Message']) if self.__failoverEnabled and time.time( ) - self.__lastSuccessfulCommit > self.__maxTimeRetrying: gLogger.verbose( "Sending accounting records to failover") result = _sendToFailover(retVal['rpcStub']) if not result['OK']: return result else: return S_ERROR( "Cannot commit data to DataStore service") sent += len(registersToSend) del registersList[:self.__maxRecordsInABundle] except Exception as e: # pylint: disable=broad-except gLogger.exception("Error committing", lException=e) return S_ERROR("Error committing %s" % repr(e).replace(',)', ')')) finally: # if something is left because of an error return it to the main list self.__registersList.extend(registersList) return S_OK(sent)
def _createReplication(targetSE, sourceSE, prodID, datatype, extraname=''): """Creates the replication transformation based on the given parameters""" from DIRAC.TransformationSystem.Client.Transformation import Transformation from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient metadata = {"Datatype": datatype, "ProdID": prodID} trans = Transformation() transName = 'replicate_%s_%s' % (str(prodID), ",".join(targetSE)) if extraname: transName += "_%s" % extraname trans.setTransformationName(transName) description = 'Replicate files for prodID %s to %s' % (str(prodID), ",".join(targetSE)) trans.setDescription(description) trans.setLongDescription(description) trans.setType('Replication') trans.setGroup('Replication') trans.setPlugin('Broadcast') res = trans.setSourceSE(sourceSE) if not res['OK']: exit(1) res = trans.setTargetSE(targetSE) if not res['OK']: exit(1) res = trans.addTransformation() if not res['OK']: gLogger.error(res['Message']) exit(1) gLogger.verbose(res) trans.setStatus('Active') trans.setAgentType('Automatic') currtrans = trans.getTransformationID()['Value'] client = TransformationClient() res = client.createTransformationInputDataQuery(currtrans, metadata) if res['OK']: gLogger.always("Successfully created replication transformation") return S_OK() else: gLogger.error("Failure during replication creation", res['Message']) return S_ERROR("Failed to create transformation")
def addToCache(self, fileName, lifeTime=False): """ Adds a new file to the cache. If the file is already there, its timer is reset """ # Check to see if the file should be excluded. for pattern in self.__cacheExceptions: if re.match(pattern, fileName): gLogger.verbose( 'File matches exclusion pattern. Ignoring: %s' % fileName) return S_OK(fileName) if not lifeTime: lifeTime = self.__defaultLifeTime now = time.time() self.__cachedFiles[fileName] = [now, lifeTime] gLogger.verbose('File added to cache. File %s. Lifetime: %d' % (fileName, lifeTime)) return S_OK(fileName)
def _loadHandlerInit( self ): handlerClass = self._svcData[ 'classObj' ] handlerName = handlerClass.__name__ handlerInitMethods = self.__searchInitFunctions( handlerClass ) try: handlerInitMethods.append( getattr( self._svcData[ 'moduleObj' ], "initialize%s" % handlerName ) ) except AttributeError: gLogger.verbose( "Not found global initialization function for service" ) if handlerInitMethods: gLogger.info( "Found %s initialization methods" % len( handlerInitMethods ) ) handlerInfo = {} handlerInfo[ "name" ] = handlerName handlerInfo[ "module" ] = self._svcData[ 'moduleObj' ] handlerInfo[ "class" ] = handlerClass handlerInfo[ "init" ] = handlerInitMethods return S_OK( handlerInfo )
def userHasProxy(self, userDN, userGroup, validSeconds=0): """ Check if a user(DN-group) has a proxy in the proxy management Updates internal cache if needed to minimize queries to the service :param basestring userDN: user DN :param basestring userGroup: user group :param int validSeconds: proxy valid time in a seconds :return: S_OK()/S_ERROR() """ cacheKey = (userDN, userGroup) if self.__usersCache.exists(cacheKey, validSeconds): return S_OK(True) # Get list of users from the DB with proxys at least 300 seconds gLogger.verbose("Updating list of users in proxy management") retVal = self.__refreshUserCache(validSeconds) if not retVal['OK']: return retVal return S_OK(self.__usersCache.exists(cacheKey, validSeconds))
def makeGuid(fileNames): """ Function to retrieve a file GUID using Root. """ if isinstance(fileNames, basestring): fileNames = [fileNames] fileGUIDs = {} for fileName in fileNames: res = getRootFileGUID(fileName) if res['OK']: gLogger.verbose('GUID from ROOT', '%s' % res['Value']) fileGUIDs[fileName] = res['Value'] else: gLogger.error( 'Could not obtain GUID from file through Gaudi, using standard DIRAC method' ) fileGUIDs[fileName] = DIRACMakeGUID(fileName) return fileGUIDs
def _syncSites(self): ''' Sync sites: compares CS with DB and does the necessary modifications. ''' gLogger.info('-- Synchronizing sites --') # sites in CS res = getSites() if not res['OK']: return res sitesCS = res['Value'] gLogger.verbose('%s sites found in CS' % len(sitesCS)) # sites in RSS result = self.rStatus.selectStatusElement('Site', 'Status', meta={'columns': ['Name']}) if not result['OK']: return result sitesDB = [siteDB[0] for siteDB in result['Value']] # Sites that are in DB but not (anymore) in CS toBeDeleted = list(set(sitesDB).difference(set(sitesCS))) gLogger.verbose('%s sites to be deleted' % len(toBeDeleted)) # Delete sites for siteName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( 'Site', siteName) gLogger.verbose('Deleting site %s' % siteName) if not deleteQuery['OK']: return deleteQuery # Sites that are in CS but not (anymore) in DB toBeAdded = list(set(sitesCS).difference(set(sitesDB))) gLogger.verbose('%s site entries to be added' % len(toBeAdded)) for site in toBeAdded: query = self.rStatus.addIfNotThereStatusElement( 'Site', 'Status', name=site, statusType='all', status=self.defaultStatus, elementType='Site', tokenOwner=self.tokenOwner, reason='Synchronized') if not query['OK']: return query return S_OK()
def purge(self): """ Purge exired files """ now = time.time() filesToDelete = [] for fileName in self.__cachedFiles: fileData = self.__cachedFiles[fileName] if fileData[0] + fileData[1] < now: filesToDelete.append(fileName) while filesToDelete: fileName = filesToDelete.pop() try: gLogger.verbose("Purging %s" % fileName) os.unlink("%s/%s" % (self.__filesLocation, fileName)) except Exception, e: gLogger.error("Can't delete file file %s: %s" % (fileName, str(e))) del (self.__cachedFiles[fileName])
def generate(self, reportRequest): """ It retrives the data from the database and creates the plot :param dict reportRequest: contains the plot attributes """ reportHash = reportRequest["hash"] reportName = reportRequest["reportName"] if reportName in self.__reportNameMapping: reportRequest["reportName"] = self.__reportNameMapping[reportName] gLogger.info("Retrieving data for %s:%s" % (reportRequest["typeName"], reportRequest["reportName"])) sT = time.time() retVal = self.__retrieveReportData(reportRequest, reportHash) reportGenerationTime = time.time() - sT if not retVal["OK"]: return retVal if not reportRequest["generatePlot"]: return retVal reportData = retVal["Value"] gLogger.info("Plotting data for %s:%s" % (reportRequest["typeName"], reportRequest["reportName"])) sT = time.time() retVal = self.__generatePlotForReport(reportRequest, reportHash, reportData) plotGenerationTime = time.time() - sT gLogger.verbose( "Time for %s:%s - Report %.2f Plot %.2f (%.2f%% r/p)" % ( reportRequest["typeName"], reportRequest["reportName"], reportGenerationTime, plotGenerationTime, ((reportGenerationTime * 100 / plotGenerationTime) if plotGenerationTime else 0.0), )) if not retVal["OK"]: return retVal plotDict = retVal["Value"] if "retrieveReportData" in reportRequest["extraArgs"] and reportRequest[ "extraArgs"]["retrieveReportData"]: plotDict["reportData"] = reportData return S_OK(plotDict)
def getAncestorFiles(inputData, ancestorDepth): """ Returns S_OK(<list of files>) or S_ERROR(<Message>) after querying the Bookkeeping for ancestor files. Input data can be an LFN string or a list of LFNs. Ancestor depth is an integer or string that converts to an integer. If successful, the original input data LFNs are also returned in the list. """ res = getFileAncestors(inputData, ancestorDepth) if not res['OK']: return res inputDataWithAncestors = res['Value'].keys() for ancestorList in res['Value']: inputDataWithAncestors += res['Value'][ancestorList] totalFiles = len(inputDataWithAncestors) - len(inputData) gLogger.verbose('%s ancestor files retrieved from the bookkeeping \ for ancestor depth %s' % (totalFiles, ancestorDepth)) return S_OK(inputDataWithAncestors)
def haltInstances(cls, vmList): """ Common haltInstances for Running(from class VirtualMachineManagerHandler) and Stalled(from checkStalledInstances periodic task) to Halt """ failed = {} successful = {} for instanceID in vmList: instanceID = int(instanceID) result = cls.virtualMachineDB.getUniqueID(instanceID) if not result["OK"]: gLogger.error("haltInstances: on getUniqueID call: %s" % result["Message"]) continue uniqueID = result["Value"] result = cls.createEndpoint(uniqueID) if not result["OK"]: gLogger.error("haltInstances: on createEndpoint call: %s" % result["Message"]) continue endpoint = result["Value"] # Get proxy to be used to connect to the cloud endpoint authType = endpoint.parameters.get("Auth") if authType and authType.lower() in ["x509", "voms"]: siteName = endpoint.parameters["Site"] ceName = endpoint.parameters["CEName"] gLogger.verbose("Getting cloud proxy for %s/%s" % (siteName, ceName)) result = getProxyFileForCloud(endpoint) if not result["OK"]: continue endpoint.setProxy(result["Value"]) result = endpoint.stopVM(uniqueID) if result["OK"]: cls.virtualMachineDB.recordDBHalt(instanceID, 0) successful[instanceID] = True else: failed[instanceID] = result["Message"] return S_OK({"Successful": successful, "Failed": failed})
def submitTasks( self ): gLogger.info( "submitTasks: Submitting tasks for transformations" ) res = getProxyInfo( False, False ) if not res['OK']: gLogger.error( "submitTasks: Failed to determine credentials for submission", res['Message'] ) return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] gLogger.info( "submitTasks: Tasks will be submitted with the credentials %s:%s" % ( owner, ownerGroup ) ) # Get the transformations which should be submitted tasksPerLoop = self.am_getOption( 'TasksPerLoop', 50 ) status = self.am_getOption( 'SubmitStatus', ['Active', 'Completing'] ) res = self._selectTransformations( transType = self.transType, status = status ) if not res['OK']: return res for transformation in res['Value']: transID = transformation['TransformationID'] transBody = transformation['Body'] res = self.transClient.getTasksToSubmit( transID, tasksPerLoop ) if not res['OK']: gLogger.error( "submitTasks: Failed to obtain tasks for transformation", "%s %s" % ( transID, res['Message'] ) ) continue tasks = res['Value']['JobDictionary'] if not tasks: gLogger.verbose( "submitTasks: No tasks found for submission for transformation %s" % transID ) continue gLogger.info( "submitTasks: Obtained %d tasks for submission for transformation %s" % ( len( tasks ), transID ) ) res = self.taskManager.prepareTransformationTasks( transBody, tasks, owner, ownerGroup ) if not res['OK']: gLogger.error( "submitTasks: Failed to prepare tasks for transformation", "%s %s" % ( transID, res['Message'] ) ) continue res = self.taskManager.submitTransformationTasks( res['Value'] ) if not res['OK']: gLogger.error( "submitTasks: Failed to submit prepared tasks for transformation", "%s %s" % ( transID, res['Message'] ) ) continue res = self.taskManager.updateDBAfterTaskSubmission( res['Value'] ) if not res['OK']: gLogger.error( "submitTasks: Failed to update DB after task submission for transformation", "%s %s" % ( transID, res['Message'] ) ) continue gLogger.info( "submitTasks: Submission of transformation tasks complete" ) return S_OK()
def createCatalog(self, catalogName, useProxy=False): """ Create a file catalog object from its name and CS description """ if useProxy: catalog = FileCatalogProxyClient(catalogName) return S_OK(catalog) # get the CS description first catalogPath = getCatalogPath(catalogName) catalogType = gConfig.getValue(catalogPath + '/CatalogType', catalogName) catalogURL = gConfig.getValue(catalogPath + '/CatalogURL', '') self.log.verbose('Creating %s client' % catalogName) moduleRootPaths = getInstalledExtensions() for moduleRootPath in moduleRootPaths: gLogger.verbose("Trying to load from root path %s" % moduleRootPath) #moduleFile = os.path.join( rootPath, moduleRootPath, "Resources", "Catalog", "%sClient.py" % catalogType ) #gLogger.verbose( "Looking for file %s" % moduleFile ) #if not os.path.isfile( moduleFile ): # continue try: # This enforces the convention that the plug in must be named after the file catalog moduleName = "%sClient" % (catalogType) catalogModule = __import__( '%s.Resources.Catalog.%s' % (moduleRootPath, moduleName), globals(), locals(), [moduleName]) except ImportError, x: if "No module" in str(x): gLogger.debug('Catalog module %s not found in %s' % (catalogType, moduleRootPath)) else: errStr = "Failed attempt to import %s from the path %s: %s" % ( catalogType, moduleRootPath, x) gLogger.error(errStr) continue except Exception, x: errStr = "Failed attempt to import %s from the path %s: %s" % ( catalogType, moduleRootPath, x) gLogger.error(errStr) continue
def checkReservedTasks( self ): gLogger.info( "checkReservedTasks: Checking Reserved tasks" ) # Get the transformations which should be checked status = self.am_getOption( 'CheckReservedStatus', ['Active', 'Completing', 'Stopped'] ) res = self._selectTransformations( transType = self.transType, status = status, agentType = [] ) if not res['OK']: return res for transformation in res['Value']: transID = transformation['TransformationID'] # Select the tasks which have been in Reserved status for more than 1 hour for selected transformations condDict = {"TransformationID":transID, "ExternalStatus":'Reserved'} time_stamp_older = str( datetime.datetime.utcnow() - datetime.timedelta( hours = 1 ) ) time_stamp_newer = str( datetime.datetime.utcnow() - datetime.timedelta( days = 7 ) ) res = self.transClient.getTransformationTasks( condDict = condDict, older = time_stamp_older, newer = time_stamp_newer, timeStamp = 'LastUpdateTime' ) if not res['OK']: gLogger.error( "checkReservedTasks: Failed to get Reserved tasks for transformation", "%s %s" % ( transID, res['Message'] ) ) continue if not res['Value']: gLogger.verbose( "checkReservedTasks: No Reserved tasks found for transformation %s" % transID ) continue res = self.taskManager.updateTransformationReservedTasks( res['Value'] ) if not res['OK']: gLogger.info( "checkReservedTasks: No Reserved tasks found for transformation %s" % transID ) continue noTasks = res['Value']['NoTasks'] taskNameIDs = res['Value']['TaskNameIDs'] # For the tasks with no associated request found re-set the status of the task in the transformationDB for taskName in noTasks: transID, taskID = taskName.split( '_' ) gLogger.info( "checkReservedTasks: Resetting status of %s to Created as no associated task found" % ( taskName ) ) res = self.transClient.setTaskStatus( int( transID ), int( taskID ), 'Created' ) if not res['OK']: gLogger.warn( "checkReservedTasks: Failed to update task status and ID after recovery", "%s %s" % ( taskName, res['Message'] ) ) # For the tasks for which an associated request was found update the task details in the transformationDB for taskName, extTaskID in taskNameIDs.items(): transID, taskID = taskName.split( '_' ) gLogger.info( "checkReservedTasks: Resetting status of %s to Created with ID %s" % ( taskName, extTaskID ) ) res = self.transClient.setTaskStatusAndWmsID( int( transID ), int( taskID ), 'Submitted', str( extTaskID ) ) if not res['OK']: gLogger.warn( "checkReservedTasks: Failed to update task status and ID after recovery", "%s %s" % ( taskName, res['Message'] ) ) gLogger.info( "checkReservedTasks: Updating of reserved tasks complete" ) return S_OK()
def __checkoutFromGit(self): if self.params.vcsBranch: brCmr = "-b %s" % self.params.vcsBranch else: brCmr = "" fDirName = os.path.join(self.params.destination, self.params.name) cmd = "git clone %s '%s' '%s'" % (brCmr, self.params.sourceURL, fDirName) gLogger.verbose("Executing: %s" % cmd) if os.system(cmd): return S_ERROR("Error while retrieving sources from git") branchName = "DIRACDistribution-%s" % os.getpid() isTagCmd = "( cd '%s'; git tag -l | grep '%s' )" % ( fDirName, self.params.version) if os.system(isTagCmd): #No tag found, assume branch branchSource = 'origin/%s' % self.params.version else: branchSource = self.params.version cmd = "( cd '%s'; git checkout -b '%s' '%s' )" % (fDirName, branchName, branchSource) gLogger.verbose("Executing: %s" % cmd) exportRes = os.system(cmd) #Add the keyword substitution gLogger.notice("Replacing keywords (can take a while)...") self.replaceKeywordsWithGit(fDirName) shutil.rmtree("%s/.git" % fDirName, ignore_errors=True) shutil.rmtree("%s/tests" % fDirName, ignore_errors=True) shutil.rmtree("%s/docs" % fDirName, ignore_errors=True) shutil.rmtree("%s/tests" % self.params.destination, ignore_errors=True) shutil.rmtree("%s/docs" % self.params.destination, ignore_errors=True) if exportRes: return S_ERROR("Error while exporting from git") return S_OK()
def initialize(self): """This replaces the standard initialize from Service""" # Build the URLs self._url = self._cfg.getURL() if not self._url: return S_ERROR("Could not build service URL for %s" % GatewayService.GATEWAY_NAME) gLogger.verbose("Service URL is %s" % self._url) # Load handler result = self._loadHandlerInit() if not result["OK"]: return result self._handler = result["Value"] # Discover Handler self._threadPool = ThreadPoolExecutor(max(0, self._cfg.getMaxThreads())) self._msgBroker = MessageBroker("%sMSB" % GatewayService.GATEWAY_NAME, threadPool=self._threadPool) self._msgBroker.useMessageObjects(False) getGlobalMessageBroker().useMessageObjects(False) self._msgForwarder = MessageForwarder(self._msgBroker) return S_OK()
def _getProductionID(self, prodName, connection=False): """ Method returns ID of production specified by the prodName :param str prodName: the Production name """ try: prodName = long(prodName) cmd = "SELECT ProductionID from Productions WHERE ProductionID=%d;" % prodName except BaseException: if not isinstance(prodName, six.string_types): return S_ERROR("Production should be ID or name") cmd = "SELECT ProductionID from Productions WHERE ProductionName='%s';" % prodName res = self._query(cmd, connection) if not res['OK']: gLogger.error("Failed to obtain production ID for production", "%s: %s" % (prodName, res['Message'])) return res elif not res['Value']: gLogger.verbose("Production %s does not exist" % (prodName)) return S_ERROR("Production does not exist") return S_OK(res['Value'][0][0])
def getUserPersistence(self, userDN, userGroup, validSeconds=0): """ Check if a user(DN-group) has a proxy in the proxy management - Updates internal cache if needed to minimize queries to the service """ cacheKey = (userDN, userGroup) userData = self.__usersCache.get(cacheKey, validSeconds) if userData: if userData['persistent']: return S_OK(True) #Get list of users from the DB with proxys at least 300 seconds gLogger.verbose("Updating list of users in proxy management") retVal = self.__refreshUserCache(validSeconds) if not retVal['OK']: return retVal userData = self.__usersCache.get(cacheKey, validSeconds) if userData: return S_OK(userData['persistent']) return S_OK(False)
def getStorageParameters(self, protocol): """ Get protocol specific options """ gLogger.verbose( "StorageElement.getStorageParameters: Obtaining storage parameters for %s protocol %s." % (self.name, protocol)) res = self.getProtocols() availableProtocols = res['Value'] if not protocol in availableProtocols: errStr = "StorageElement.getStorageParameters: Requested protocol not available for SE." gLogger.error(errStr, '%s for %s' % (protocol, self.name)) return S_ERROR(errStr) for storage in self.storages: res = storage.getParameters() storageParameters = res['Value'] if storageParameters['ProtocolName'] == protocol: return S_OK(storageParameters) errStr = "StorageElement.getStorageParameters: Requested protocol supported but no object found." gLogger.error(errStr, "%s for %s" % (protocol, self.name)) return S_ERROR(errStr)
def __copyToExternalSE(self, localFilePath, sbPath): """ Copy uploaded file to external SE """ try: dm = DataManager() result = dm.put(sbPath, localFilePath, self.__externalSEName) if not result["OK"]: return result if "Successful" not in result["Value"]: gLogger.verbose("Oops, no successful transfers there", str(result)) return S_ERROR("RM returned OK to the action but no successful transfers were there") okTrans = result["Value"]["Successful"] if sbPath not in okTrans: gLogger.verbose("Ooops, SB transfer wasn't in the successful ones", str(result)) return S_ERROR("RM returned OK to the action but SB transfer wasn't in the successful ones") return S_OK((self.__externalSEName, okTrans[sbPath])) except Exception as e: gLogger.error("Error while moving sandbox to SE", "%s" % repr(e).replace(",)", ")")) return S_ERROR("Error while moving sandbox to SE")
def getGUID(fileNames, directory=''): """ This function searches the directory for POOL XML catalog files and extracts the GUID. fileNames can be a string or a list, directory defaults to PWD. """ if not directory: directory = os.getcwd() if not os.path.isdir(directory): return S_ERROR('%s is not a directory' % directory) if not type(fileNames) == type([]): fileNames = [fileNames] gLogger.verbose('Will look for POOL XML Catalog GUIDs in %s for %s' % (directory, string.join(fileNames, ', '))) patterns = ['*.xml', '*.xml*gz'] omissions = ['\.bak$'] # to be ignored for production files #First obtain valid list of unpacked catalog files in directory poolCatalogList = [] for pattern in patterns: fileList = glob.glob(os.path.join(directory, pattern)) for fname in fileList: if fname.endswith('.bak'): gLogger.verbose('Ignoring BAK file: %s' % fname) elif tarfile.is_tarfile(fname): try: gLogger.debug('Unpacking catalog XML file %s' % (os.path.join(directory, fname))) tarFile = tarfile.open(os.path.join(directory, fname), 'r') for member in tarFile.getmembers(): tarFile.extract(member, directory) poolCatalogList.append( os.path.join(directory, member.name)) except Exception, x: gLogger.error('Could not untar %s with exception %s' % (fname, str(x))) else: poolCatalogList.append(fname)
def __generateStorageObject( self, storageName, protocolName, protocol, path = None, host = None, port = None, spaceToken = None, wsUrl = None, parameters={} ): storageType = protocolName if self.proxy: storageType = 'Proxy' moduleRootPaths = getInstalledExtensions() moduleLoaded = False path = path.rstrip( '/' ) if not path: path = '/' for moduleRootPath in moduleRootPaths: if moduleLoaded: break gLogger.verbose( "Trying to load from root path %s" % moduleRootPath ) moduleFile = os.path.join( rootPath, moduleRootPath, "Resources", "Storage", "%sStorage.py" % storageType ) gLogger.verbose( "Looking for file %s" % moduleFile ) if not os.path.isfile( moduleFile ): continue try: # This inforces the convention that the plug in must be named after the protocol moduleName = "%sStorage" % ( storageType ) storageModule = __import__( '%s.Resources.Storage.%s' % ( moduleRootPath, moduleName ), globals(), locals(), [moduleName] ) except Exception, x: errStr = "StorageFactory._generateStorageObject: Failed to import %s: %s" % ( storageName, x ) gLogger.exception( errStr ) return S_ERROR( errStr ) try: evalString = "storageModule.%s(storageName,protocol,path,host,port,spaceToken,wsUrl)" % moduleName storage = eval( evalString ) if not storage.isOK(): errStr = "StorageFactory._generateStorageObject: Failed to instantiate storage plug in." gLogger.error( errStr, "%s" % ( moduleName ) ) return S_ERROR( errStr ) except Exception, x: errStr = "StorageFactory._generateStorageObject: Failed to instantiate %s(): %s" % ( moduleName, x ) gLogger.exception( errStr ) return S_ERROR( errStr )
def getTransformationFiles( self, condDict = {}, older = None, newer = None, timeStamp = 'LastUpdate', orderAttribute = None, limit = 10000, rpc = '', url = '', timeout = 1800 ): """ gets all the transformation files for a transformation, incrementally. "limit" here is just used to determine the offset. """ rpcClient = self._getRPC( rpc = rpc, url = url, timeout = timeout ) transformationFiles = [] # getting transformationFiles - incrementally offsetToApply = 0 while True: res = rpcClient.getTransformationFiles( condDict, older, newer, timeStamp, orderAttribute, limit, offsetToApply ) if not res['OK']: return res else: gLogger.verbose( "Result for limit %d, offset %d: %d" % ( limit, offsetToApply, len( res['Value'] ) ) ) if res['Value']: transformationFiles = transformationFiles + res['Value'] offsetToApply += limit if len( res['Value'] ) < limit: break return S_OK( transformationFiles )
def __getOutputLFNs(self, prodID='12345', prodJobID='6789', prodXMLFile=''): """ Will construct the output LFNs for the production for visual inspection. """ if not prodXMLFile: gLogger.verbose('Using workflow object to generate XML file') prodXMLFile = self.__createWorkflow() job = LHCbJob(prodXMLFile) result = preSubmissionLFNs( job._getParameters(), job.workflow.createCode(), # pylint: disable=protected-access productionID=prodID, jobID=prodJobID) if not result['OK']: return result lfns = result['Value'] gLogger.verbose(lfns) return result
def getPayloadProxyFromDIRACGroup(self, userDN, userGroup, requiredTimeLeft, token=None, proxyToConnect=None): """ Download a payload proxy with VOMS extensions depending on the group :param str userDN: user DN :param str userGroup: user group :param int requiredTimeLeft: required proxy live time in a seconds :param str token: valid token to get a proxy :param X509Chain proxyToConnect: proxy as a chain :return: S_OK(X509Chain)/S_ERROR() """ # Assign VOMS attribute vomsAttr = Registry.getVOMSAttributeForGroup(userGroup) if not vomsAttr: gLogger.verbose("No voms attribute assigned to group %s when requested payload proxy" % userGroup) return self.downloadProxy(userDN, userGroup, limited=True, requiredTimeLeft=requiredTimeLeft, proxyToConnect=proxyToConnect, token=token) else: return self.downloadVOMSProxy(userDN, userGroup, limited=True, requiredTimeLeft=requiredTimeLeft, requiredVOMSAttribute=vomsAttr, proxyToConnect=proxyToConnect, token=token)
def getProductionTransformations(self, prodName, condDict=None, older=None, newer=None, timeStamp=None, orderAttribute=None, limit=10000): """ Gets all the production transformations for a production, incrementally. "limit" here is just used to determine the offset. :param str prodName: the production name :return: the list of the transformations associated to the production """ rpcClient = self._getRPC() productionTransformations = [] if condDict is None: condDict = {} if timeStamp is None: timeStamp = 'CreationTime' # getting productionTransformations - incrementally offsetToApply = 0 while True: res = rpcClient.getProductionTransformations( prodName, condDict, older, newer, timeStamp, orderAttribute, limit, offsetToApply) if not res['OK']: return res else: gLogger.verbose("Result for limit %d, offset %d: %d" % (limit, offsetToApply, len(res['Value']))) if res['Value']: productionTransformations = productionTransformations + res[ 'Value'] offsetToApply += limit if len(res['Value']) < limit: break return S_OK(productionTransformations)