def removeLibc(libraryPath): """ Remove libraries that can be problematic, like libc.so :param string libraryPath: libraryPath to look for libraries to remove :returns: True on Success, False in case of error """ gLogger.debug("RemoveLibC: Trying to remove these libraries:") gLogger.debug("RemoveLibC - "+ "\nRemoveLibC - ".join(FILES_TO_REMOVE) ) curdir = os.getcwd() try: os.chdir(libraryPath) except OSError: return True listlibs = os.listdir(os.getcwd()) for lib in listlibs: for lib_to_remove in FILES_TO_REMOVE: if lib.count(lib_to_remove): try: libraryPath = os.getcwd() + os.sep + lib gLogger.info("RemoveLibC: Trying to remove: %s" % libraryPath) os.remove(libraryPath) except OSError: gLogger.error("RemoveLibC: Could not remove", lib) os.chdir(curdir) return False os.chdir(curdir) return True
def jobexec(jobxml, wfParameters): jobfile = os.path.abspath(jobxml) if not os.path.exists(jobfile): gLogger.warn('Path to specified workflow %s does not exist' % (jobfile)) sys.exit(1) workflow = fromXMLFile(jobfile) gLogger.debug(workflow) code = workflow.createCode() gLogger.debug(code) jobID = 0 if 'JOBID' in os.environ: jobID = os.environ['JOBID'] gLogger.info('DIRAC JobID %s is running at site %s' % (jobID, DIRAC.siteName())) workflow.addTool('JobReport', JobReport(jobID)) workflow.addTool('AccountingReport', DataStoreClient()) workflow.addTool('Request', Request()) # Propagate the command line parameters to the workflow if any for pName, pValue in wfParameters.items(): workflow.setValue(pName, pValue) # Propagate the command line parameters to the workflow module instances of each step for stepdefinition in workflow.step_definitions.itervalues(): for moduleInstance in stepdefinition.module_instances: for pName, pValue in wfParameters.iteritems(): if moduleInstance.parameters.find(pName): moduleInstance.parameters.setValue(pName, pValue) return workflow.execute()
def __irodsClient( self , user = None ): global IRODS_USER password = None cfgPath = self.serviceInfoDict[ 'serviceSectionPath' ] gLogger.debug( "cfgPath: %s" % cfgPath ) if not user: credentials = self.getRemoteCredentials() if credentials and ( "username" in credentials ): IRODS_USER = credentials[ "username" ] ## TODO: should get user password somehow elif user == "r": IRODS_USER = gConfig.getValue( "%s/read" % cfgPath , IRODS_USER ) elif user == "w": IRODS_USER = gConfig.getValue( "%s/write" % cfgPath , IRODS_USER ) if not IRODS_USER: return False , "Failed to get iRods user" gLogger.debug( "iRods user: %s" % IRODS_USER ) password = gConfig.getValue( "%s/%s" % ( cfgPath , IRODS_USER ) , password ) conn , errMsg = rcConnect( IRODS_HOST , IRODS_PORT , IRODS_USER , IRODS_ZONE ) status = clientLoginWithPassword( conn , password ) if not status == 0: return False , "Failed to authenticate user '%s'" % IRODS_USER return conn , errMsg
def __getStageSubmittedReplicas( self ): """ This obtains the StageSubmitted replicas from the Replicas table and the RequestID from the StageRequests table """ res = self.storageDB.getCacheReplicas( {'Status':'StageSubmitted'} ) if not res['OK']: gLogger.error( "StageRequest.__getStageSubmittedReplicas: Failed to get replicas with StageSubmitted status.", res['Message'] ) return res if not res['Value']: gLogger.debug( "StageRequest.__getStageSubmittedReplicas: No StageSubmitted replicas found to process." ) return S_OK() else: gLogger.debug( "StageRequest.__getStageSubmittedReplicas: Obtained %s StageSubmitted replicas(s) to process." % len( res['Value'] ) ) seReplicas = {} replicaIDs = res['Value'] for replicaID, info in replicaIDs.items(): storageElement = info['SE'] if not seReplicas.has_key( storageElement ): seReplicas[storageElement] = [] seReplicas[storageElement].append( replicaID ) # RequestID was missing from replicaIDs dictionary BUGGY? res = self.storageDB.getStageRequests( {'ReplicaID':replicaIDs.keys()} ) if not res['OK']: return res if not res['Value']: return S_ERROR( 'Could not obtain request IDs for replicas %s from StageRequests table' % ( replicaIDs.keys() ) ) for replicaID, info in res['Value'].items(): reqID = info['RequestID'] replicaIDs[replicaID]['RequestID'] = reqID return S_OK( {'SEReplicas':seReplicas, 'ReplicaIDs':replicaIDs} )
def __query( self, queryType, tableName, parameters ): ''' This method is a rather important one. It will format the input for the DB queries, instead of doing it on a decorator. Two dictionaries must be passed to the DB. First one contains 'columnName' : value pairs, being the key lower camel case. The second one must have, at lease, a key named 'table' with the right table name. ''' # Functions we can call, just a light safety measure. _gateFunctions = [ 'insert', 'update', 'select', 'delete', 'addOrModify', 'addIfNotThere' ] if not queryType in _gateFunctions: return S_ERROR( '"%s" is not a proper gate call' % queryType ) gateFunction = getattr( self.gate, queryType ) # If meta is None, we set it to {} meta = ( True and parameters.pop( 'meta' ) ) or {} # params = parameters # Remove self, added by locals() del parameters[ 'self' ] meta[ 'table' ] = tableName gLogger.debug( 'Calling %s, with \n params %s \n meta %s' % ( queryType, parameters, meta ) ) return gateFunction( parameters, meta ) #............................................................................... #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def export_update( self, params, meta ): ''' This method is a bridge to access :class:`ResourceManagementDB` remotely. It does not add neither processing nor validation. If you need to know more about this method, you must keep reading on the database documentation. :Parameters: **params** - `dict` arguments for the mysql query ( must match table columns ! ). **meta** - `dict` metadata for the mysql query. It must contain, at least, `table` key with the proper table name. :return: S_OK() || S_ERROR() ''' gLogger.info( 'update: %s %s' % ( params, meta ) ) try: res = db.update( params, meta ) gLogger.debug( 'update %s' % res ) except Exception, e: _msg = 'Exception calling db.update: \n %s' % e gLogger.exception( _msg ) res = S_ERROR( _msg )
def transfer_toClient( self, fileID, token, fileHelper ): """ Method to send files to clients. fileID is the local file name in the SE. token is used for access rights confirmation. """ conn , error = self.__irodsClient( "r" ) if not conn: return S_ERROR( error ) file_path = self.__resolveFileID( fileID ) file_path = IRODS_HOME + file_path gLogger.debug( "file_path to read: %s" % file_path ) fd = iRodsOpen( conn , file_path , "r" ) if not fd: rcDisconnect( conn ) gLogger.error( "Failed to get file object" ) return S_ERROR( "Failed to get file object" ) result = fileHelper.FileToNetwork( fd ) fd.close() rcDisconnect( conn ) if not result[ "OK" ]: gLogger.error( "Failed to get file " + fileID ) return S_ERROR( "Failed to get file " + fileID ) else: return result
def export_rescheduleJob( self, jobIDs ): """ Reschedule a single job. If the optional proxy parameter is given it will be used to refresh the proxy in the Proxy Repository """ jobList = self.__get_job_list( jobIDs ) if not jobList: return S_ERROR( 'Invalid job specification: ' + str( jobIDs ) ) validJobList, invalidJobList, nonauthJobList, ownerJobList = self.__evaluate_rights( jobList, RIGHT_RESCHEDULE ) for jobID in validJobList: gtaskQueueDB.deleteJob( jobID ) #gJobDB.deleteJobFromQueue(jobID) result = gJobDB.rescheduleJob( jobID ) gLogger.debug( str( result ) ) if not result['OK']: return result gJobLoggingDB.addLoggingRecord( result['JobID'], result['Status'], result['MinorStatus'], application = 'Unknown', source = 'JobManager' ) if invalidJobList or nonauthJobList: result = S_ERROR( 'Some jobs failed deletion' ) if invalidJobList: result['InvalidJobIDs'] = invalidJobList if nonauthJobList: result['NonauthorizedJobIDs'] = nonauthJobList return result result = S_OK( validJobList ) result[ 'requireProxyUpload' ] = len( ownerJobList ) > 0 and self.__checkIfProxyUploadIsRequired() self.__sendNewJobsToMind( validJobList ) return result
def run(self): """ The main watchdog execution method """ result = self.initialize() if not result['OK']: gLogger.always('Can not start wtchdog for the following reason') gLogger.always(result['Message']) return result try: while True: gLogger.debug('Starting agent loop # %d' % self.count) start_cycle_time = time.time() result = self.execute() exec_cycle_time = time.time() - start_cycle_time if not result['OK']: gLogger.error("Watchdog error during execution", result['Message']) break elif result['Value'] == "Ended": break self.count += 1 if exec_cycle_time < self.pollingTime: time.sleep(self.pollingTime - exec_cycle_time) return S_OK() except Exception, x: gLogger.exception() return S_ERROR('Exception')
def aftermath(self, actionSuccess, actionFailed, action, prefix): success = ", ".join(actionSuccess) failure = "\n".join(actionFailed) if len(actionSuccess) > 1: sText = prefix + "s" else: sText = prefix if len(actionFailed) > 1: fText = prefix + "s" else: fText = prefix if len(success) > 0 and len(failure) > 0: sMessage = "%s %sed successfully: " % (sText , action , success) fMessage = "Failed to %s %s:\n%s" % (action , fText , failure) result = sMessage + "\n\n" + fMessage return { "success" : "true" , "result" : result } elif len(success) > 0 and len(failure) < 1: result = "%s %sed successfully: %s" % (sText , action , success) return { "success" : "true" , "result" : result } elif len(success) < 1 and len(failure) > 0: result = "Failed to %s %s:\n%s" % (action , fText , failure) gLogger.always(result) return { "success" : "false" , "error" : result } else: result = "No action has performed due technical failure. Check the logs please" gLogger.debug(result) return { "success" : "false" , "error" : result }
def web_getHostData(self): """ Returns flatten list of components (services, agents) installed on hosts returned by getHosts function """ # checkUserCredentials() userData = self.getSessionData() DN = str(userData["user"]["DN"]) group = str(userData["user"]["group"]) callback = list() if not (self.request.arguments.has_key("hostname") and len(self.request.arguments["hostname"][0]) > 0): self.finish({ "success" : "false" , "error" : "Name of the host is absent" }) return host = self.request.arguments["hostname"][0] client = SystemAdministratorClient(host , None , delegatedDN=DN , delegatedGroup=group) result = yield self.threadTask(client.getOverallStatus) gLogger.debug("Result of getOverallStatus(): %s" % result) if not result[ "OK" ]: self.finish({ "success" : "false" , "error" : result[ "Message" ] }) return overall = result[ "Value" ] for record in self.flatten(overall): record[ "Host" ] = host callback.append(record) self.finish({ "success" : "true" , "result" : callback })
def getDirectoryMetadata( self, path ): """ Get the metadata for the directory """ res = self.__checkArgumentFormat( path ) if not res['OK']: return res urls = res['Value'] gLogger.debug( "RFIOStorage.getDirectoryMetadata: Attempting to get metadata for %s directories." % len( urls ) ) res = self.isDirectory( urls ) if not res['OK']: return res successful = {} failed = res['Value']['Failed'] directories = [] for url, isDirectory in res['Value']['Successful'].items(): if isDirectory: directories.append( url ) else: errStr = "RFIOStorage.getDirectoryMetadata: Directory does not exist." gLogger.error( errStr, url ) failed[url] = errStr res = self.__getPathMetadata( directories ) if not res['OK']: return res else: failed.update( res['Value']['Failed'] ) successful = res['Value']['Successful'] resDict = {'Failed':failed, 'Successful':successful} return S_OK( resDict )
def web_getHostErrors(self): userData = self.getSessionData() DN = str(userData["user"]["DN"]) group = str(userData["user"]["group"]) if not "host" in self.request.arguments: self.finish({ "success" : "false" , "error" : "Name of the host is missing or not defined" }) return host = str(self.request.arguments[ "host" ][0]) client = SystemAdministratorClient(host , None , delegatedDN=DN , delegatedGroup=group) result = yield self.threadTask(client.checkComponentLog, "*") gLogger.debug(result) if not result[ "OK" ]: self.finish({ "success" : "false" , "error" : result[ "Message" ] }) return result = result[ "Value" ] callback = list() for key, value in result.items(): system, component = key.split("/") value[ "System" ] = system value[ "Name" ] = component value[ "Host" ] = host callback.append(value) total = len(callback) self.finish({ "success" : "true" , "result" : callback , "total" : total })
def isFile( self, path ): """Check if the given path exists and it is a file """ res = self.__checkArgumentFormat( path ) if not res['OK']: return res urls = res['Value'] gLogger.debug( "RFIOStorage.isFile: Determining whether %s paths are files." % len( urls ) ) successful = {} failed = {} comm = "nsls -ld" for url in urls: comm = " %s %s" % ( comm, url ) res = shellCall( self.timeout, comm ) if not res['OK']: return res returncode, stdout, stderr = res['Value'] if returncode in [0, 1]: for line in stdout.splitlines(): permissions, _subdirs, _owner, _group, _size, _month, _date, _timeYear, pfn = line.split() if permissions[0] != 'd': successful[pfn] = True else: successful[pfn] = False for line in stderr.splitlines(): pfn, error = line.split( ': ' ) url = pfn.strip() failed[url] = error else: errStr = "RFIOStorage.isFile: Completely failed to determine whether path is file." gLogger.error( errStr, "%s %s" % ( self.name, stderr ) ) return S_ERROR( errStr ) resDict = {'Failed':failed, 'Successful':successful} return S_OK( resDict )
def removeDirectory( self, path, recursive = False ): """Remove a directory on the physical storage together with all its files and subdirectories. """ res = self.__checkArgumentFormat( path ) if not res['OK']: return res urls = res['Value'] gLogger.debug( "RFIOStorage.removeDirectory: Attempting to remove %s directories." % len( urls ) ) successful = {} failed = {} for url in urls: comm = "nsrm -r %s" % url res = shellCall( 100, comm ) if res['OK']: returncode, _stdout, stderr = res['Value'] if returncode == 0: successful[url] = {'FilesRemoved':0, 'SizeRemoved':0} elif returncode == 1: successful[url] = {'FilesRemoved':0, 'SizeRemoved':0} else: failed[url] = stderr else: errStr = "RFIOStorage.removeDirectory: Completely failed to remove directory." gLogger.error( errStr, "%s %s" % ( url, res['Message'] ) ) failed[url] = res['Message'] resDict = {'Failed':failed, 'Successful':successful} return S_OK( resDict )
def getDirectory( self, path, localPath = False ): """ Get locally a directory from the physical storage together with all its files and subdirectories. """ res = self.__checkArgumentFormat( path ) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} gLogger.debug( "RFIOStorage.getDirectory: Attempting to get local copies of %s directories." % len( urls ) ) for src_directory in urls: dirName = os.path.basename( src_directory ) if localPath: dest_dir = "%s/%s" % ( localPath, dirName ) else: dest_dir = "%s/%s" % ( os.getcwd(), dirName ) res = self.__getDir( src_directory, dest_dir ) if res['OK']: if res['Value']['AllGot']: gLogger.debug( "RFIOStorage.getDirectory: Successfully got local copy of %s" % src_directory ) successful[src_directory] = {'Files':res['Value']['Files'], 'Size':res['Value']['Size']} else: gLogger.error( "RFIOStorage.getDirectory: Failed to get entire directory.", src_directory ) failed[src_directory] = {'Files':res['Value']['Files'], 'Size':res['Value']['Size']} else: gLogger.error( "RFIOStorage.getDirectory: Completely failed to get local copy of directory.", src_directory ) failed[src_directory] = {'Files':0, 'Size':0} resDict = {'Failed':failed, 'Successful':successful} return S_OK( resDict )
def putDirectory( self, path ): """ Put a local directory to the physical storage together with all its files and subdirectories. """ res = checkArgumentFormat( path ) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} gLogger.debug( "RFIOStorage.putDirectory: Attemping to put %s directories to remote storage." % len( urls ) ) for destDir, sourceDir in urls.items(): res = self.__putDir( sourceDir, destDir ) if res['OK']: if res['Value']['AllPut']: gLogger.debug( "RFIOStorage.putDirectory: Successfully put directory to remote storage: %s" % destDir ) successful[destDir] = {'Files':res['Value']['Files'], 'Size':res['Value']['Size']} else: gLogger.error( "RFIOStorage.putDirectory: Failed to put entire directory to remote storage.", destDir ) failed[destDir] = {'Files':res['Value']['Files'], 'Size':res['Value']['Size']} else: gLogger.error( "RFIOStorage.putDirectory: Completely failed to put directory to remote storage.", destDir ) failed[destDir] = {'Files':0, 'Size':0} resDict = {'Failed':failed, 'Successful':successful} return S_OK( resDict )
def exists( self, path ): """ Check if the given path exists. The 'path' variable can be a string or a list of strings. """ res = self.__checkArgumentFormat( path ) if not res['OK']: return res urls = res['Value'] gLogger.debug( "RFIOStorage.exists: Determining the existance of %s files." % len( urls ) ) comm = "nsls -d" for url in urls: comm = " %s %s" % ( comm, url ) res = shellCall( self.timeout, comm ) successful = {} failed = {} if res['OK']: returncode, stdout, stderr = res['Value'] if returncode in [0, 1]: for line in stdout.splitlines(): url = line.strip() successful[url] = True for line in stderr.splitlines(): pfn, _ = line.split( ': ' ) url = pfn.strip() successful[url] = False else: errStr = "RFIOStorage.exists: Completely failed to determine the existance files." gLogger.error( errStr, "%s %s" % ( self.name, stderr ) ) return S_ERROR( errStr ) else: errStr = "RFIOStorage.exists: Completely failed to determine the existance files." gLogger.error( errStr, "%s %s" % ( self.name, res['Message'] ) ) return S_ERROR( errStr ) resDict = {'Failed':failed, 'Successful':successful} return S_OK( resDict )
def isDirectory( self, path ): """Check if the given path exists and it is a directory """ res = self.__checkArgumentFormat( path ) if not res['OK']: return res urls = res['Value'] gLogger.debug( "RFIOStorage.isDirectory: Determining whether %s paths are directories." % len( urls ) ) res = self.__getPathMetadata( urls ) if not res['OK']: return res failed = {} successful = {} for pfn, error in res['Value']['Failed'].items(): if error == 'No such file or directory': failed[pfn] = 'Directory does not exist' else: failed[pfn] = error for pfn, pfnDict in res['Value']['Successful'].items(): if pfnDict['Type'] == 'Directory': successful[pfn] = True else: successful[pfn] = False resDict = {'Failed':failed, 'Successful':successful} return S_OK( resDict )
def getFileMetadata( self, path ): """ Get metadata associated to the file """ res = self.__checkArgumentFormat( path ) if not res['OK']: return res urls = res['Value'] gLogger.debug( "RFIOStorage.getFileMetadata: Obtaining metadata for %s files." % len( urls ) ) res = self.__getPathMetadata( urls ) if not res['OK']: return res failed = {} successful = {} for pfn, error in res['Value']['Failed'].items(): if error == 'No such file or directory': failed[pfn] = 'File does not exist' else: failed[pfn] = error files = [] for pfn, pfnDict in res['Value']['Successful'].items(): if pfnDict['Type'] == 'Directory': failed[pfn] = "Supplied path is not a file" else: successful[pfn] = res['Value']['Successful'][pfn] files.append( pfn ) if files: res = self.__getFileMetadata( files ) if not res['OK']: return res for pfn, pfnDict in res['Value']['Successful'].items(): successful[pfn].update( pfnDict ) resDict = {'Failed':failed, 'Successful':successful} return S_OK( resDict )
def getFileSize( self, path ): """Get the physical size of the given file """ res = self.__checkArgumentFormat( path ) if not res['OK']: return res urls = res['Value'] gLogger.debug( "RFIOStorage.getFileSize: Determining the sizes for %s files." % len( urls ) ) res = self.__getPathMetadata( urls ) if not res['OK']: return res failed = {} successful = {} for pfn, error in res['Value']['Failed'].items(): if error == 'No such file or directory': failed[pfn] = 'File does not exist' else: failed[pfn] = error for pfn, pfnDict in res['Value']['Successful'].items(): if pfnDict['Type'] == 'Directory': failed[pfn] = "Supplied path is not a file" else: successful[pfn] = res['Value']['Successful'][pfn]['Size'] resDict = {'Failed':failed, 'Successful':successful} return S_OK( resDict )
def killJob( self, jobIDList ): """ Kill the specified jobs """ result = self._prepareProxy() self.usercfg.ProxyPath(os.environ['X509_USER_PROXY']) if not result['OK']: gLogger.error( 'ARCComputingElement: failed to set up proxy', result['Message'] ) return result js = arc.compute.JobSupervisor(self.usercfg) jobList = list( jobIDList ) if isinstance( jobIDList, basestring ): jobList = [ jobIDList ] for jobID in jobList: job = self.__getARCJob( jobID ) js.AddJob( job ) result = js.Cancel() # Cancel all jobs at once if not result: gLogger.debug("Failed to kill jobs %s. CE(?) not reachable?" % jobIDList) return S_ERROR( 'Failed to kill the job(s)' ) else: gLogger.debug("Killed jobs %s" % jobIDList) return S_OK()
def getSiteCEMapping( gridName = '' ): """ Returns a dictionary of all sites and their CEs as a list, e.g. {'LCG.CERN.ch':['ce101.cern.ch',...]} If gridName is specified, result is restricted to that Grid type. """ siteCEMapping = {} gridTypes = gConfig.getSections( 'Resources/Sites/', [] ) if not gridTypes['OK']: gLogger.warn( 'Problem retrieving sections in /Resources/Sites' ) return gridTypes gridTypes = gridTypes['Value'] if gridName: if not gridName in gridTypes: return S_ERROR( 'Could not get sections for /Resources/Sites/%s' % gridName ) gridTypes = [gridName] gLogger.debug( 'Grid Types are: %s' % ( ', '.join( gridTypes ) ) ) for grid in gridTypes: sites = gConfig.getSections( '/Resources/Sites/%s' % grid, [] ) if not sites['OK']: gLogger.warn( 'Problem retrieving /Resources/Sites/%s section' % grid ) return sites for candidate in sites['Value']: candidateCEs = gConfig.getValue( '/Resources/Sites/%s/%s/CE' % ( grid, candidate ), [] ) if candidateCEs: siteCEMapping[candidate] = candidateCEs else: gLogger.debug( 'No CEs defined for site %s' % candidate ) return S_OK( siteCEMapping )
def _getPoolCatalogs( directory = '' ): patterns = ['*.xml', '*.xml*gz'] omissions = ['\.bak$'] # to be ignored for production files #First obtain valid list of unpacked catalog files in directory poolCatalogList = [] for pattern in patterns: fileList = glob.glob( os.path.join( directory, pattern ) ) for fname in fileList: if fname.endswith( '.bak' ): gLogger.verbose( 'Ignoring BAK file: %s' % fname ) elif tarfile.is_tarfile( fname ): try: gLogger.debug( 'Unpacking catalog XML file %s' % ( os.path.join( directory, fname ) ) ) tarFile = tarfile.open( os.path.join( directory, fname ), 'r' ) for member in tarFile.getmembers(): tarFile.extract( member, directory ) poolCatalogList.append( os.path.join( directory, member.name ) ) except Exception, x : gLogger.error( 'Could not untar with exception', ' %s: %s' % ( fname, str( x ) ) ) else: poolCatalogList.append( fname )
def run(self): """ The main watchdog execution method """ result = self.initialize() if not result["OK"]: gLogger.always("Can not start watchdog for the following reason") gLogger.always(result["Message"]) return result try: while True: gLogger.debug("Starting watchdog loop # %d" % self.count) start_cycle_time = time.time() result = self.execute() exec_cycle_time = time.time() - start_cycle_time if not result["OK"]: gLogger.error("Watchdog error during execution", result["Message"]) break elif result["Value"] == "Ended": break self.count += 1 if exec_cycle_time < self.pollingTime: time.sleep(self.pollingTime - exec_cycle_time) return S_OK() except Exception: gLogger.exception() return S_ERROR("Exception")
def insertCSSection( self, path, pardict ): """ insert a section and values (or subsections) into the CS :param str path: full path of the new section :param str pardict: dictionary of key values in the new section, values can also be dictionaries :return: S_OK(), S_ERROR() """ from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI if self.csapi is None: self.csapi = CSAPI() for key, value in pardict.iteritems(): newSectionPath = os.path.join(path,key) gLogger.debug( "Adding to cs %s : %s " % ( newSectionPath, value ) ) self.csapi.createSection( path ) if isinstance( value, dict ): res = self.insertCSSection( newSectionPath, value ) else: res = self.csapi.setOption( newSectionPath, value ) if not res['OK']: return res else: gLogger.notice( "Added to CS: %s " % res['Value'] ) return S_OK("Added all things to cs")
def _receiveAndCheckProposal( self, trid ): clientTransport = self._transportPool.get( trid ) #Get the peer credentials credDict = clientTransport.getConnectingCredentials() #Receive the action proposal retVal = clientTransport.receiveData( 1024 ) if not retVal[ 'OK' ]: gLogger.error( "Invalid action proposal", "%s %s" % ( self._createIdentityString( credDict, clientTransport ), retVal[ 'Message' ] ) ) return S_ERROR( "Invalid action proposal" ) proposalTuple = retVal[ 'Value' ] gLogger.debug( "Received action from client", "/".join( list( proposalTuple[1] ) ) ) #Check if there are extra credentials if proposalTuple[2]: clientTransport.setExtraCredentials( proposalTuple[2] ) #Check if this is the requested service requestedService = proposalTuple[0][0] if requestedService not in self._validNames: return S_ERROR( "%s is not up in this server" % requestedService ) #Check if the action is valid requestedActionType = proposalTuple[1][0] if requestedActionType not in Service.SVC_VALID_ACTIONS: return S_ERROR( "%s is not a known action type" % requestedActionType ) #Check if it's authorized result = self._authorizeProposal( proposalTuple[1], trid, credDict ) if not result[ 'OK' ]: return result #Proposal is OK return S_OK( proposalTuple )
def getCESiteMapping( gridName = '' ): """ Returns a dictionary of all CEs and their associated site, e.g. {'ce101.cern.ch':'LCG.CERN.ch', ...]} Assumes CS structure of: /Resources/Sites/<GRIDNAME>/<SITENAME> """ ceSiteMapping = {} gridTypes = gConfig.getSections( '/Resources/Sites/', [] ) if not gridTypes['OK']: gLogger.warn( 'Problem retrieving sections in /Resources/Sites' ) return gridTypes gridTypes = gridTypes['Value'] if gridName: if not gridName in gridTypes: return S_ERROR( 'Could not get sections for /Resources/Sites/%s' % gridName ) gridTypes = [gridName] gLogger.debug( 'Grid Types are: %s' % ( ', '.join( gridTypes ) ) ) for grid in gridTypes: sites = gConfig.getSections( '/Resources/Sites/%s' % grid, [] ) if not sites['OK']: #gConfig returns S_ERROR for empty sections until version gLogger.warn( 'Problem retrieving /Resources/Sites/%s section' % grid ) return sites if sites: for candidate in sites['Value']: siteCEs = gConfig.getValue( '/Resources/Sites/%s/%s/CE' % ( grid, candidate ), [] ) for ce in siteCEs: if ceSiteMapping.has_key( ce ): current = ceSiteMapping[ce] gLogger.warn( 'CE %s already has a defined site %s but it is also defined for %s' % ( ce, current, candidate ) ) else: ceSiteMapping[ce] = candidate return S_OK( ceSiteMapping )
def cleanUpLFNPath( lfn ): """ Normalise LFNs """ gLogger.debug("LFN before Cleanup", lfn) lfn = posixpath.normpath(lfn) gLogger.verbose("LFN after Cleanup", lfn) return lfn
def _getDirectoryContent(directory): """ Inner function: recursively scan a directory, returns list of LFNs """ filesInDirectory = {} gLogger.debug("Examining %s" % directory) res = self.fileCatalog.listDirectory(directory) if not res['OK']: gLogger.error('Failed to get directory contents', res['Message']) return res if directory in res['Value']['Failed']: gLogger.error('Failed to get directory content', '%s %s' % (directory, res['Value']['Failed'][directory])) return S_ERROR('Failed to get directory content') if directory not in res['Value']['Successful']: return S_ERROR('Directory not existing?') # first, adding the files found in the current directory gLogger.debug("Files in %s: %d" % (directory, len( res['Value']['Successful'][directory]['Files']))) filesInDirectory.update(res['Value']['Successful'][directory]['Files']) # then, looking for subDirectories content if res['Value']['Successful'][directory]['SubDirs']: for l_dir in res['Value']['Successful'][directory]['SubDirs']: # recursion here subDirContent = _getDirectoryContent(l_dir) if not subDirContent['OK']: return subDirContent else: filesInDirectory.update(subDirContent['Value']) return S_OK(filesInDirectory)
def __request_file(self): req = {"selection": {}, "path": "/"} separator = ":::" result = self.fc.getMetadataFields() gLogger.debug("request: %s" % result) if not result["OK"]: gLogger.error("request: %s" % result["Message"]) return req result = result["Value"] if not result.has_key("FileMetaFields"): error = "Service response has no FileMetaFields key. Return empty dict" gLogger.error("request: %s" % error) return req if not result.has_key("DirectoryMetaFields"): error = "Service response has no DirectoryMetaFields key. Return empty dict" gLogger.error("request: %s" % error) return req filemeta = result["FileMetaFields"] dirmeta = result["DirectoryMetaFields"] meta = [] for key, value in dirmeta.items(): meta.append(key) gLogger.always("request: metafields: %s " % meta) selectionElems = self.request.arguments["selection"][0].split("<|>") gLogger.always("request: THISSSS %s " % self.request.arguments["selection"][0]) for param in selectionElems: tmp = str(param).split('|') if len(tmp) != 4: continue name = tmp[0] logic = tmp[1] if not logic in ["in", "nin", "=", "!=", ">=", "<=", ">", "<"]: gLogger.always("Operand '%s' is not supported " % logic) continue if name in meta: #check existence of the 'name' section if not req["selection"].has_key(name): req["selection"][name] = dict() #check existence of the 'sign' section if not req["selection"][name].has_key(logic): if tmp[2] == "v": req["selection"][name][logic] = "" elif tmp[2] == "s": req["selection"][name][logic] = [] if tmp[2] == "v": req["selection"][name][logic] = tmp[3] elif tmp[2] == "s": req["selection"][name][logic] += tmp[3].split(":::") if self.request.arguments.has_key("path"): req["path"] = self.request.arguments["path"][0] gLogger.always("REQ: ", req) return req
def web_getLaunchpadOpts(self): defaultParams = { "JobName": [1, 'DIRAC'], "Executable": [1, "/bin/ls"], "Arguments": [1, "-ltrA"], "OutputSandbox": [1, "std.out, std.err"], "InputData": [0, ""], "OutputData": [0, ""], "OutputSE": [0, "DIRAC-USER"], "OutputPath": [0, ""], "CPUTime": [0, "86400"], "Site": [0, ""], "BannedSite": [0, ""], "Platform": [0, "Linux_x86_64_glibc-2.5"], "Priority": [0, "5"], "StdError": [0, "std.err"], "StdOutput": [0, "std.out"], "Parameters": [0, "0"], "ParameterStart": [0, "0"], "ParameterStep": [0, "1"] } delimiter = gConfig.getValue("/Website/Launchpad/ListSeparator", ',') options = self.__getOptionsFromCS(delimiter=delimiter) # platform = self.__getPlatform() # if platform and options: # if not options.has_key("Platform"): # options[ "Platform" ] = platform # else: # csPlatform = list(options[ "Platform" ]) # allPlatforms = csPlatform + platform # platform = uniqueElements(allPlatforms) # options[ "Platform" ] = platform gLogger.debug("Combined options from CS: %s" % options) override = gConfig.getValue("/Website/Launchpad/OptionsOverride", False) gLogger.info("end __getLaunchpadOpts") # Updating the default values from OptionsOverride configuration branch for key in options: if key not in defaultParams: defaultParams[key] = [0, ""] defaultParams[key][1] = options[key][0] # Reading of the predefined sets of launchpad parameters values obj = Operations() predefinedSets = {} launchpadSections = obj.getSections("Launchpad") import pprint if launchpadSections['OK']: for section in launchpadSections["Value"]: predefinedSets[section] = {} sectionOptions = obj.getOptionsDict("Launchpad/" + section) pprint.pprint(sectionOptions) if sectionOptions['OK']: predefinedSets[section] = sectionOptions["Value"] self.write({ "success": "true", "result": defaultParams, "predefinedSets": predefinedSets })
def web_getLaunchpadSetupWithLFNs(self): #on the fly file catalog for advanced launchpad if not hasattr(self, 'fc'): userData = self.getSessionData() group = str(userData["user"]["group"]) vo = getVOForGroup(group) self.fc = FileCatalog(vo=vo) self.set_header('Content-type', 'text/plain') lfnList = [] arguments = self.request.arguments gLogger.always( "submit: incoming arguments %s to getLaunchpadSetupWithLFNs" % arguments) lfnStr = str(arguments['path'][0]) lfnList = lfnStr.split(',') #checks if the experiments folder in lfn list has a rtg_def.m file at some subfolder gLogger.always("submit: checking if some rtg_def.m" % arguments) processed = [] metaDict = {'type': 'info'} for lfn in lfnStr.split(','): pos_relative = lfn.find("/") pos_relative = lfn.find("/", pos_relative + 1) pos_relative = lfn.find("/", pos_relative + 1) pos_relative = lfn.find("/", pos_relative + 1) pos_relative = lfn.find("/", pos_relative + 1) experiment_lfn = lfn[0:pos_relative] if experiment_lfn in processed: continue processed.append(experiment_lfn) gLogger.always("checking rtg_def.m in %s" % experiment_lfn) result = self.fc.findFilesByMetadata(metaDict, path=str(experiment_lfn)) print "result" print result if not result['OK'] or not result['Value']: gLogger.error("Failed to get type info from $s, %s" % (experiment_lfn, result["Message"])) continue for candidate_lfn in result['Value']: if candidate_lfn.find('rtg_def.m') > 0: lfnList.append(candidate_lfn) totalfn = len(lfnList) ptlfn = '' current = 1 for lfn in lfnList: ptlfn = ptlfn + lfn if current < totalfn: ptlfn = ptlfn + ', ' current = current + 1 defaultParams = { "JobName": [1, 'Eiscat'], "Executable": [1, "/bin/ls"], "Arguments": [1, "-ltrA"], "OutputSandbox": [1, "std.out, std.err"], "InputData": [1, ptlfn], "OutputData": [0, ""], "OutputSE": [1, "EISCAT-disk"], "OutputPath": [0, ""], "CPUTime": [0, "86400"], "Site": [0, ""], "BannedSite": [0, ""], "Platform": [0, "Linux_x86_64_glibc-2.5"], "Priority": [0, "5"], "StdError": [0, "std.err"], "StdOutput": [0, "std.out"], "Parameters": [0, "0"], "ParameterStart": [0, "0"], "ParameterStep": [0, "1"] } delimiter = gConfig.getValue("/Website/Launchpad/ListSeparator", ',') options = self.__getOptionsFromCS(delimiter=delimiter) # platform = self.__getPlatform() # if platform and options: # if not options.has_key("Platform"): # options[ "Platform" ] = platform # else: # csPlatform = list(options[ "Platform" ]) # allPlatforms = csPlatform + platform # platform = uniqueElements(allPlatforms) # options[ "Platform" ] = platform gLogger.debug("Options from CS: %s" % options) override = gConfig.getValue("/Website/Launchpad/OptionsOverride", False) gLogger.info("end __getLaunchpadOpts") # Updating the default values from OptionsOverride configuration branch, for key in options: if key not in defaultParams: defaultParams[key] = [0, ""] defaultParams[key][1] = options[key][0] gLogger.info( "Default params + override from /Website/Launchpad/OptionsOverride -> %s" % defaultParams) # Reading of the predefined sets of launchpad parameters values obj = Operations() predefinedSets = {} launchpadSections = obj.getSections("Launchpad") import pprint if launchpadSections['OK']: for section in launchpadSections["Value"]: predefinedSets[section] = {} sectionOptions = obj.getOptionsDict("Launchpad/" + section) pprint.pprint(sectionOptions) if sectionOptions['OK']: predefinedSets[section] = sectionOptions["Value"] self.write({ "success": "true", "result": defaultParams, "predefinedSets": predefinedSets })
def listDirectory(self, path): """ List the supplied path. First checks whether the path is a directory then gets the contents. """ res = self.__checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] gLogger.debug( "RFIOStorage.listDirectory: Attempting to list %s directories." % len(urls)) res = self.isDirectory(urls) if not res['OK']: return res successful = {} failed = res['Value']['Failed'] directories = [] for url, isDirectory in res['Value']['Successful'].items(): if isDirectory: directories.append(url) else: errStr = "RFIOStorage.listDirectory: Directory does not exist." gLogger.error(errStr, url) failed[url] = errStr for directory in directories: comm = "nsls -l %s" % directory res = shellCall(self.timeout, comm) if res['OK']: returncode, stdout, stderr = res['Value'] if not returncode == 0: errStr = "RFIOStorage.listDirectory: Failed to list directory." gLogger.error(errStr, "%s %s" % (directory, stderr)) failed[directory] = errStr else: subDirs = {} files = {} successful[directory] = {} for line in stdout.splitlines(): permissions, _subdirs, _owner, _group, size, _month, _date, _timeYear, pfn = line.split( ) if not pfn == 'dirac_directory': path = "%s/%s" % (directory, pfn) if permissions[0] == 'd': # If the subpath is a directory subDirs[path] = True elif permissions[0] == 'm': # In the case that the path is a migrated file files[path] = { 'Size': int(size), 'Migrated': 1 } else: # In the case that the path is not migrated file files[path] = { 'Size': int(size), 'Migrated': 0 } successful[directory]['SubDirs'] = subDirs successful[directory]['Files'] = files else: errStr = "RFIOStorage.listDirectory: Completely failed to list directory." gLogger.error(errStr, "%s %s" % (directory, res['Message'])) return S_ERROR(errStr) resDict = {'Failed': failed, 'Successful': successful} return S_OK(resDict)
def __putFile(self, src_file, dest_url, sourceSize): """Put a copy of the local file to the current directory on the physical storage """ # Pre-transfer check res = self.__executeOperation(dest_url, 'exists') if not res['OK']: gLogger.debug( "RFIOStorage.__putFile: Failed to find pre-existance of destination file." ) return res if res['Value']: res = self.__executeOperation(dest_url, 'removeFile') if not res['OK']: gLogger.debug( "RFIOStorage.__putFile: Failed to remove remote file %s." % dest_url) else: gLogger.debug( "RFIOStorage.__putFile: Removed remote file %s." % dest_url) if not os.path.exists(src_file): errStr = "RFIOStorage.__putFile: The source local file does not exist." gLogger.error(errStr, src_file) return S_ERROR(errStr) sourceSize = getSize(src_file) if sourceSize == -1: errStr = "RFIOStorage.__putFile: Failed to get file size." gLogger.error(errStr, src_file) return S_ERROR(errStr) res = self.__getTransportURL(dest_url) if not res['OK']: gLogger.debug( "RFIOStorage.__putFile: Failed to get transport URL for file.") return res turl = res['Value'] MIN_BANDWIDTH = 1024 * 100 # 100 KB/s timeout = sourceSize / MIN_BANDWIDTH + 300 gLogger.debug("RFIOStorage.putFile: Executing transfer of %s to %s" % (src_file, turl)) comm = "rfcp %s '%s'" % (src_file, turl) res = shellCall(timeout, comm) if res['OK']: returncode, _stdout, stderr = res['Value'] if returncode == 0: gLogger.debug( 'RFIOStorage.putFile: Put file to storage, performing post transfer check.' ) res = self.__executeOperation(dest_url, 'getFileSize') if res['OK']: destinationSize = res['Value'] if sourceSize == destinationSize: gLogger.debug( "RFIOStorage.__putFile: Post transfer check successful." ) return S_OK(destinationSize) errorMessage = "RFIOStorage.__putFile: Source and destination file sizes do not match." gLogger.error(errorMessage, dest_url) else: errStr = "RFIOStorage.__putFile: Failed to put file to remote storage." gLogger.error(errStr, stderr) errorMessage = "%s %s" % (errStr, stderr) else: errStr = "RFIOStorage.__putFile: Failed to put file to remote storage." gLogger.error(errStr, res['Message']) errorMessage = "%s %s" % (errStr, res['Message']) res = self.__executeOperation(dest_url, 'removeFile') if res['OK']: gLogger.debug( "RFIOStorage.__putFile: Removed remote file remnant %s." % dest_url) else: gLogger.debug( "RFIOStorage.__putFile: Unable to remove remote file remnant %s." % dest_url) return S_ERROR(errorMessage)
def __componentAction( self , action = None ): """ Actions which should be done on components. The only parameters is an action to perform. Returns standard JSON response structure with with service response or error messages """ DN = getUserDN() group = getSelectedGroup() if ( not action ) or ( not len( action ) > 0 ): error = "Action is not defined or has zero length" gLogger.debug( error ) return { "success" : "false" , "error" : error } if action not in [ "restart" , "start" , "stop" , "uninstall" ]: error = "The request parameters action '%s' is unknown" % action gLogger.debug( error ) return { "success" : "false" , "error" : error } self.action = action result = dict() for i in request.params: if i == "action": continue target = i.split( " @ " , 1 ) if not len( target ) == 2: continue system = request.params[ i ] gLogger.always( "System: %s" % system ) host = target[ 1 ] gLogger.always( "Host: %s" % host ) component = target[ 0 ] gLogger.always( "Component: %s" % component ) if not host in result: result[ host ] = list() result[ host ].append( [ system , component ] ) if not len( result ) > 0: error = "Failed to get component(s) for %s" % action gLogger.debug( error ) return { "success" : "false" , "error" : error } gLogger.always( result ) self.actionSuccess = list() self.actionFailed = list() for hostname in result.keys(): if not len( result[ hostname ] ) > 0: continue client = SystemAdministratorClient( hostname , None , delegatedDN=DN , delegatedGroup=group ) for i in result[ hostname ]: system = i[ 0 ] component = i[ 1 ] try: if action == "restart": result = client.restartComponent( system , component ) elif action == "start": result = client.startComponent( system , component ) elif action == "stop": result = client.stopComponent( system , component ) elif action == "uninstall": result = client.uninstallComponent( system , component ) else: result = list() result[ "Message" ] = "Action %s is not valid" % action except Exception, x: result = list() result[ "Message" ] = "Exception: %s" % str( x ) gLogger.debug( "Result: %s" % result ) if not result[ "OK" ]: error = hostname + ": " + result[ "Message" ] self.actionFailed.append( error ) gLogger.error( "Failure during component %s: %s" % ( action , error ) ) else: gLogger.always( "Successfully %s component %s" % ( action , component ) ) self.actionSuccess.append( component )
def _getCatalogDirectoryContents(self, lfnDirs): """ Obtain the contents of the supplied directory, recursively """ def _getDirectoryContent(directory): """ Inner function: recursively scan a directory, returns list of LFNs """ filesInDirectory = {} gLogger.debug("Examining %s" % directory) res = self.fc.listDirectory(directory) if not res['OK']: gLogger.error('Failed to get directory contents', res['Message']) return res if directory in res['Value']['Failed']: gLogger.error( 'Failed to get directory content', '%s %s' % (directory, res['Value']['Failed'][directory])) return S_ERROR('Failed to get directory content') if directory not in res['Value']['Successful']: return S_ERROR('Directory not existing?') # first, adding the files found in the current directory gLogger.debug( "Files in %s: %d" % (directory, len( res['Value']['Successful'][directory]['Files']))) filesInDirectory.update( res['Value']['Successful'][directory]['Files']) #then, looking for subDirectories content if res['Value']['Successful'][directory]['SubDirs']: for l_dir in res['Value']['Successful'][directory]['SubDirs']: #recursion here subDirContent = _getDirectoryContent(l_dir) if not subDirContent['OK']: return subDirContent else: filesInDirectory.update(subDirContent['Value']) return S_OK(filesInDirectory) gLogger.info('Obtaining the catalog contents for %d directories' % len(lfnDirs)) allFiles = {} for lfnDir in lfnDirs: dirContent = _getDirectoryContent(lfnDir) if not dirContent['OK']: return dirContent else: gLogger.debug("Content of directory %s: %d files" % (lfnDir, len(dirContent['Value']))) allFiles.update(dirContent['Value']) gLogger.debug("Content of directories examined: %d files" % len(allFiles)) replicas = self.fc.getReplicas(list(allFiles)) if not replicas['OK']: return replicas if replicas['Value']['Failed']: return S_ERROR("Failures in replicas discovery") return S_OK({ 'Metadata': allFiles, 'Replicas': replicas['Value']['Successful'] })
def web_getFilesData(self): req = self.__request() gLogger.always(req) gLogger.debug("submit: incoming request %s" % req) result = yield self.threadTask(self.fc.findFilesByMetadataWeb, req["selection"], req["path"], self.S_NUMBER, self.L_NUMBER) gLogger.debug("submit: result of findFilesByMetadataDetailed %s" % result) if not result["OK"]: gLogger.error("submit: %s" % result["Message"]) self.finish({"success": "false", "error": result["Message"]}) return result = result["Value"] if not len(result) > 0: self.finish({ "success": "true", "result": [], "total": 0, "date": "-" }) return total = result["TotalRecords"] result = result["Records"] callback = list() for key, value in result.items(): size = "" if "Size" in value: size = value["Size"] date = "" if "CreationDate" in value: date = str(value["CreationDate"]) meta = "" if "Metadata" in value: m = value["Metadata"] meta = '; '.join(['%s: %s' % (i, j) for (i, j) in m.items()]) dirnameList = key.split("/") dirname = "/".join(dirnameList[:len(dirnameList) - 1]) filename = dirnameList[len(dirnameList) - 1:] callback.append({ "fullfilename": key, "dirname": dirname, "filename": filename, "date": date, "size": size, "metadata": meta }) timestamp = Time.dateTime().strftime("%Y-%m-%d %H:%M [UTC]") self.finish({ "success": "true", "result": callback, "total": total, "date": timestamp })
def setUpClass(cls): cls.failed = False # Add configuration cfg = CFG() cfg.loadFromBuffer(diracTestCACFG) gConfig.loadCFG(cfg) cfg.loadFromBuffer(userCFG) gConfig.loadCFG(cfg) # Prepare CA lines = [] cfgDict = {} cls.caPath = os.path.join(certsPath, "ca") cls.caConfigFile = os.path.join(cls.caPath, "openssl_config_ca.cnf") # Save original configuration file shutil.copyfile(cls.caConfigFile, cls.caConfigFile + "bak") # Parse fields = [ "dir", "database", "serial", "new_certs_dir", "private_key", "certificate" ] with open(cls.caConfigFile, "r") as caCFG: for line in caCFG: if re.findall("=", re.sub(r"#.*", "", line)): field = re.sub(r"#.*", "", line).replace(" ", "").rstrip().split("=")[0] line = "dir = %s #PUT THE RIGHT DIR HERE!\n" % ( cls.caPath) if field == "dir" else line val = re.sub(r"#.*", "", line).replace(" ", "").rstrip().split("=")[1] if field in fields: for i in fields: if cfgDict.get(i): val = val.replace("$%s" % i, cfgDict[i]) cfgDict[field] = val if not cfgDict[field]: cls.failed = "%s have empty value in %s" % ( field, cls.caConfigFile) lines.append(line) with open(cls.caConfigFile, "w") as caCFG: caCFG.writelines(lines) for field in fields: if field not in cfgDict.keys(): cls.failed = "%s value is absent in %s" % (field, cls.caConfigFile) cls.hostCert = os.path.join(certsPath, "host/hostcert.pem") cls.hostKey = os.path.join(certsPath, "host/hostkey.pem") cls.caCert = cfgDict["certificate"] cls.caKey = cfgDict["private_key"] os.chmod(cls.caKey, stat.S_IREAD) # Check directory for new certificates cls.newCertDir = cfgDict["new_certs_dir"] if not os.path.exists(cls.newCertDir): os.makedirs(cls.newCertDir) for f in os.listdir(cls.newCertDir): os.remove(os.path.join(cls.newCertDir, f)) # Empty the certificate database cls.index = cfgDict["database"] with open(cls.index, "w") as indx: indx.write("") # Write down serial cls.serial = cfgDict["serial"] with open(cls.serial, "w") as serialFile: serialFile.write("1000") # Create temporaly directory for users certificates cls.userDir = tempfile.mkdtemp(dir=certsPath) # Create user certificates for userName in ["no_user", "user", "user_1", "user_2", "user_3"]: userConf = """[ req ] default_bits = 4096 encrypt_key = yes distinguished_name = req_dn prompt = no req_extensions = v3_req [ req_dn ] C = CC O = DN 0.O = DIRAC CN = %s [ v3_req ] # Extensions for client certificates (`man x509v3_config`). nsComment = "OpenSSL Generated Client Certificate" keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment extendedKeyUsage = clientAuth """ % (userName) userConfFile = os.path.join(cls.userDir, userName + ".cnf") userReqFile = os.path.join(cls.userDir, userName + ".req") userKeyFile = os.path.join(cls.userDir, userName + ".key.pem") userCertFile = os.path.join(cls.userDir, userName + ".cert.pem") with open(userConfFile, "w") as f: f.write(userConf) status, output = commands.getstatusoutput( "openssl genrsa -out %s" % userKeyFile) if status: gLogger.error(output) exit() gLogger.debug(output) os.chmod(userKeyFile, stat.S_IREAD) status, output = commands.getstatusoutput( "openssl req -config %s -key %s -new -out %s" % (userConfFile, userKeyFile, userReqFile)) if status: gLogger.error(output) exit() gLogger.debug(output) cmd = "openssl ca -config %s -extensions usr_cert -batch -days 375 -in %s -out %s" cmd = cmd % (cls.caConfigFile, userReqFile, userCertFile) status, output = commands.getstatusoutput(cmd) if status: gLogger.error(output) exit() gLogger.debug(output) # Result status, output = commands.getstatusoutput("ls -al %s" % cls.userDir) if status: gLogger.error(output) exit() gLogger.debug("User certificates:\n", output)
def __request(self): req = {"selection": {}, "path": "/"} self.L_NUMBER = 25 if self.request.arguments.has_key("limit") and len( self.request.arguments["limit"][0]) > 0: self.L_NUMBER = int(self.request.arguments["limit"][0]) self.S_NUMBER = 0 if self.request.arguments.has_key("start") and len( self.request.arguments["start"][0]) > 0: self.S_NUMBER = int(self.request.arguments["start"][0]) result = gConfig.getOption("/Website/ListSeparator") if result["OK"]: separator = result["Value"] else: separator = ":::" result = self.fc.getMetadataFields() gLogger.debug("request: %s" % result) if not result["OK"]: gLogger.error("request: %s" % result["Message"]) return req result = result["Value"] if not result.has_key("FileMetaFields"): error = "Service response has no FileMetaFields key. Return empty dict" gLogger.error("request: %s" % error) return req if not result.has_key("DirectoryMetaFields"): error = "Service response has no DirectoryMetaFields key. Return empty dict" gLogger.error("request: %s" % error) return req filemeta = result["FileMetaFields"] dirmeta = result["DirectoryMetaFields"] meta = [] for key, value in dirmeta.items(): meta.append(key) gLogger.always("request: metafields: %s " % meta) for param in self.request.arguments: tmp = str(param).split('.') if len(tmp) != 3: continue name = tmp[1] logic = tmp[2] value = self.request.arguments[param][0].split("|") if not logic in ["in", "nin", "=", "!=", ">=", "<=", ">", "<"]: gLogger.always("Operand '%s' is not supported " % logic) continue if name in meta: #check existence of the 'name' section if not req["selection"].has_key(name): req["selection"][name] = dict() #check existence of the 'sign' section if not req["selection"][name].has_key(logic): if value[0] == "v": req["selection"][name][logic] = "" elif value[0] == "s": req["selection"][name][logic] = [] if value[0] == "v": req["selection"][name][logic] = value[1] elif value[0] == "s": req["selection"][name][logic] += value[1].split(":::") if self.request.arguments.has_key("path"): req["path"] = self.request.arguments["path"][0] gLogger.always("REQ: ", req) return req
def bulk_index(self, indexprefix, doc_type, data, mapping=None, period=None): """ :param str indexPrefix: it is the index name. :param str doc_type: the type of the document :param dict data: contains a list of dictionary :paran dict mapping: the mapping used by elasticsearch :param str period: We can specify, which kind of indexes will be created. Currently only daily and monthly indexes are supported. """ gLogger.info("%d records will be insert to %s" % (len(data), doc_type)) if mapping is None: mapping = {} indexName = generateFullIndexName(indexprefix, period) gLogger.debug("inserting datat to %s index" % indexName) if not self.exists(indexName): retVal = self.createIndex(indexprefix, mapping, period) if not retVal['OK']: return retVal docs = [] for row in data: body = {'_index': indexName, '_type': doc_type, '_source': {}} body['_source'] = row if 'timestamp' not in row: gLogger.warn( "timestamp is not given! Note: the actual time is used!") timestamp = row.get( 'timestamp', int(Time.toEpoch()) ) # if the timestamp is not provided, we use the current utc time. try: if isinstance(timestamp, datetime): body['_source']['timestamp'] = int( timestamp.strftime('%s')) * 1000 elif isinstance(timestamp, basestring): timeobj = datetime.strptime(timestamp, '%Y-%m-%d %H:%M:%S.%f') body['_source']['timestamp'] = int( timeobj.strftime('%s')) * 1000 else: # we assume the timestamp is an unix epoch time (integer). body['_source']['timestamp'] = timestamp * 1000 except (TypeError, ValueError) as e: # in case we are not able to convert the timestamp to epoch time.... gLogger.error("Wrong timestamp", e) body['_source']['timestamp'] = int(Time.toEpoch()) * 1000 docs += [body] try: res = bulk(self.__client, docs, chunk_size=self.__chunk_size) except BulkIndexError as e: return S_ERROR(e) if res[0] == len(docs): # we have inserted all documents... return S_OK(len(docs)) else: return S_ERROR(res) return res
(os.path.join(directory, fname))) tarFile = tarfile.open(os.path.join(directory, fname), 'r') for member in tarFile.getmembers(): tarFile.extract(member, directory) poolCatalogList.append( os.path.join(directory, member.name)) except Exception, x: gLogger.error('Could not untar %s with exception %s' % (fname, str(x))) else: poolCatalogList.append(fname) poolCatalogList = uniqueElements(poolCatalogList) #Now have list of all XML files but some may not be Pool XML catalogs... finalCatList = [] for possibleCat in poolCatalogList: try: cat = PoolXMLCatalog(possibleCat) finalCatList.append(possibleCat) except Exception, x: gLogger.debug('Ignoring non-POOL catalogue file %s' % possibleCat) gLogger.debug('Final list of catalog files are: %s' % string.join(finalCatList, ', ')) return finalCatList #############################################################################
def retrieveAggregatedData( self, typeName, startTime, endTime, interval, selectFields, condDict, grouping, metainfo ): """ Get data from the DB using simple aggregations. Note: this method is equivalent to retrieveBucketedData. The different is the dynamic bucketing. We do not perform dynamic bucketing on the raw data. :param str typeName name of the monitoring type :param int startTime epoch objects. :param int endtime epoch objects. :param dict condDict -> conditions for the query key -> name of the field value -> list of possible values """ # {'query': {'bool': {'filter': [{'bool': {'must': [{'range': {'timestamp': {'gte': 1474271462000, 'lte': 1474357862000}}}]}}]}}, 'aggs': {'end_data': {'date_histogram': {'field': 'timestamp', 'interval': '30m'}, 'aggs': {'tt': {'terms': {'field': 'component', 'size': 0}, 'aggs': {'m1': {'avg': {' # field': 'threads'}}}}}}}} # # query = [Q( 'range',timestamp = {'lte':1474357862000,'gte': 1474271462000} )] # # a = A('terms', field = 'component', size = 0 ) # a.metric('m1', 'avg', field = 'threads' ) # # s = Search(using=cl, index = 'lhcb-certification_componentmonitoring-index-*') # # s = s.filter( 'bool', must = query ) # s = s.aggs.bucket('end_data', 'date_histogram', field='timestamp', interval='30m').metric( 'tt', a ) retVal = self.getIndexName( typeName ) if not retVal['OK']: return retVal # default is average aggregator = metainfo.get( 'metric', 'avg' ) indexName = "%s*" % ( retVal['Value'] ) q = [self._Q( 'range', timestamp = {'lte':endTime * 1000, 'gte': startTime * 1000} )] for cond in condDict: query = None for condValue in condDict[cond]: kwargs = {cond: condValue} if query: query = query | self._Q( 'match', **kwargs ) else: query = self._Q( 'match', **kwargs ) q += [query] a1 = self._A( 'terms', field = grouping, size = 0 ) a1.metric( 'm1', aggregator, field = selectFields[0] ) s = self._Search( indexName ) s = s.filter( 'bool', must = q ) s.aggs.bucket( 'end_data', 'date_histogram', field = 'timestamp', interval = interval ).metric( 'tt', a1 ) #s.fields( ['timestamp'] + selectFields ) s = s.extra( size = 0 ) # do not get the hits! gLogger.debug( 'Query:', s.to_dict() ) retVal = s.execute() result = {} for bucket in retVal.aggregations['end_data'].buckets: # each bucket key is a time (unix epoch and usual datetime bucketTime = bucket.key / 1000 for value in bucket['tt'].buckets: # each bucket contains an agregation called tt which sum/avg of the metric. if value.key not in result: result[value.key] = {bucketTime:value.m1.value if value.m1.value else 0 } #TODO: this is kind of hack. #we can use a default value for pipeline aggregation. EL promised that we can use default value for simple aggregation. Later to be checked. else: result[value.key].update( {bucketTime:value.m1.value if value.m1.value else 0} ) # the result format is { 'grouping':{timestamp:value, timestamp:value} for example : {u'Bookkeeping_BookkeepingManager': {1474300800: 4.0, 1474344000: 4.0, 1474331400: 4.0, 1 # 474302600: 4.0, 1474365600: 4.0, 1474304400: 4.0, 1474320600: 4.0, 1474360200: 4.0, 1474306200: 4.0, 1474356600: 4.0, 1474336800: 4.0, 1474326000: 4.0, 1474315200: 4.0, # 1474281000: 4.0, 1474309800: 4.0, 1474338600: 4.0, 1474311600: 4.0, 1474317000: 4.0, 1474367400: 4.0, 1474333200: 4.0, 1474284600: 4.0, 1474362000: 4.0, # 1474327800: 4.0, 1474345800: 4.0, 1474286400: 4.0, 1474308000: 4.0, 1474322400: 4.0, 1474288200: 4.0, 1474351200: 4.0, 1474282800: 4.0, 1474347600: 4.0, # 1474313400: 4.0, 1474349400: 4.0, 1474297200: 4.0, 1474340400: 4.0, 1474291800: 4.0, 1474335000: 4.0, 1474293600: 4.0, 1474290000: 4.0, 1474363800: 4.0, # 1474329600: 4.0, 1474353000: 4.0, 1474358400: 4.0, 1474324200: 4.0, 1474354800: 4.0, 1474295400: 4.0, 1474318800: 4.0, 1474299000: 4.0, 1474342200: 4.0}, # u'Framework_SystemAdministrator': {1474300800: 8.0, 1474344000: 8.0, 1474331400: 8.0, 1474302600: 8.0, 1474365600: 8.0, 1474304400: 8.0, 1474320600: 8.0, # 1474360200: 8.0, 1474306200: 8.0, 1474356600: 8.0, 1474336800: 8.0, 1474326000: 8.0, 1474315200: 8.0, 1474281000: 8.0, 1474309800: 8.0, 1474338600: 8.0, # 1474311600: 8.0, 1474317000: 8.0, 1474367400: 8.0, 1474333200: 8.0, 1474284600: 8.0, 1474362000: 8.0, 1474327800: 8.0, 1474345800: 8.0, 1474286400: 8.0, # 1474308000: 8.0, 1474322400: 8.0, 1474288200: 8.0, 1474351200: 8.0, 1474282800: 8.0, 1474347600: 8.0, 1474313400: 8.0, 1474349400: 8.0, 1474297200: 8.0, # 1474340400: 8.0, 1474291800: 8.0, 1474335000: 8.0, 1474293600: 8.0, 1474290000: 8.0, 1474363800: 8.0, 1474329600: 8.0, 1474353000: 8.0, 1474358400: 8.0, # 1474324200: 8.0, 1474354800: 8.0, 1474295400: 8.0, 1474318800: 8.0, 1474299000: 8.0, 1474342200: 8.0}} return S_OK( result )
readBanned = [] writeBanned = [] storageCFGBase = "/Resources/StorageElements" for se in ses: res = gConfig.getOptionsDict("%s/%s" % (storageCFGBase, se)) if not res['OK']: gLogger.error("Storage Element %s does not exist" % se) continue existingOptions = res['Value'] if read and existingOptions['ReadAccess'] == "Active": res = csAPI.setOption("%s/%s/ReadAccess" % (storageCFGBase, se), "InActive") if not res['OK']: gLogger.error("Failed to update %s read access to InActive" % se) else: gLogger.debug("Successfully updated %s read access to InActive" % se) readBanned.append(se) if write and existingOptions['WriteAccess'] == "Active": res = csAPI.setOption("%s/%s/WriteAccess" % (storageCFGBase, se), "InActive") if not res['OK']: gLogger.error("Failed to update %s write access to InActive" % se) else: gLogger.debug("Successfully updated %s write access to InActive" % se) writeBanned.append(se) res = csAPI.commitChanges() if not res['OK']: gLogger.error("Failed to commit changes to CS", res['Message']) DIRAC.exit(-1)
def getPoliciesThatApply(decisionParams): """ Method that sanitizes the input parameters and returns the policies that match them. Matches the input dictionary with the policies configuration in the CS. It returns a list of policy dictionaries that matched. """ # InfoGetter is being called from SiteInspector Agent decisionParams = _sanitizedecisionParams(decisionParams) gLogger.debug("Sanitized decisionParams: %s" % str(decisionParams)) policiesThatApply = [] # Get policies configuration metadata from CS. policiesConfig = RssConfiguration.getPolicies() if not policiesConfig['OK']: return policiesConfig policiesConfig = policiesConfig['Value'] gLogger.debug("All policies: %s" % str(policiesConfig)) # Each policy, has the following format # <policyName> # \ # policyType = <policyType> # matchParams # \ # ... # configParams # \ # ... # Get policies that match the given decisionParameters for policyName, policySetup in policiesConfig.items(): # The parameter policyType replaces policyName, so if it is not present, # we pick policyName try: policyType = policySetup['policyType'][0] except KeyError: policyType = policyName # continue # The section matchParams is not mandatory, so we set {} as default. policyMatchParams = policySetup.get('matchParams', {}) gLogger.debug("matchParams of %s: %s" % (policyName, str(policyMatchParams))) # FIXME: make sure the values in the policyConfigParams dictionary are typed !! policyConfigParams = {} # policyConfigParams = policySetup.get( 'configParams', {} ) policyMatch = Utils.configMatch(decisionParams, policyMatchParams) gLogger.debug("PolicyMatch for decisionParams %s: %s" % (decisionParams, str(policyMatch))) # WARNING: we need an additional filtering function when the matching # is not straightforward (e.g. when the policy specify a 'domain', while # the decisionParams has only the name of the element) if policyMatch and _filterPolicies(decisionParams, policyMatchParams): policiesThatApply.append( (policyName, policyType, policyConfigParams)) gLogger.debug("policies that apply (before post-processing): %s" % str(policiesThatApply)) policiesThatApply = postProcessingPolicyList(policiesThatApply) gLogger.debug("policies that apply (after post-processing): %s" % str(policiesThatApply)) policiesToBeLoaded = [] # Gets policies parameters from code. for policyName, policyType, _policyConfigParams in policiesThatApply: try: configModule = Utils.voimport( 'DIRAC.ResourceStatusSystem.Policy.Configurations') policies = copy.deepcopy(configModule.POLICIESMETA) policyMeta = policies[policyType] except KeyError: continue # We are not going to use name / type anymore, but we keep them for debugging # and future usage. policyDict = {'name': policyName, 'type': policyType, 'args': {}} # args is one of the parameters we are going to use on the policies. We copy # the defaults and then we update if with whatever comes from the CS. policyDict.update(policyMeta) policiesToBeLoaded.append(policyDict) return S_OK(policiesToBeLoaded)
def submit(self): pagestart = time() RPC = getRPCClient("ResourceStatus/ResourceStatus") client = ResourceStatusClient(serviceIn=RPC) if not request.params.has_key("mode") or not len( request.params["mode"]) > 0: gLogger.error("The parameter 'mode' is absent") return { "success": "false", "error": "The parameter 'mode' is absent" } mode = request.params["mode"] gLogger.verbose("Requested mode is %s" % mode) if not mode in MODELIST: gLogger.error( "Parameter 'mode': %s is wrong. Should be one of the list %s" % (mode, MODELIST)) return {"success": "false", "error": "Parameter 'mode' is wrong"} if mode in STORELIST: mode = 'StorageElement' gLogger.verbose("Selected mode is %s" % mode) req = self.__request() gLogger.info("getMonitoredsStatusWeb(%s,%s,%s,%s)" % (mode, req, P_NUMBER, R_NUMBER)) result = client.getMonitoredsStatusWeb(mode, req, P_NUMBER, R_NUMBER) gLogger.debug("Call result: %s" % result) if not result["OK"]: error = result["Message"] gLogger.error(error) return {"success": "false", "error": error} result = result["Value"] if not result.has_key( "TotalRecords") or not result["TotalRecords"] > 0: return { "success": "false", "error": "There were no data matching your selection" } if not result.has_key("ParameterNames") or not result.has_key( "Records"): return {"success": "false", "error": "Data structure is corrupted"} if not len(result["ParameterNames"]) > 0: return { "success": "false", "error": "ParameterNames field is undefined" } if not len(result["Records"]) > 0: return { "success": "false", "error": "There are no data to display" } c.result = [] records = result["Records"] head = result["ParameterNames"] headLength = len(head) countryCode = self.__countries() for i in records: tmp = {} for j in range(0, headLength): tmp[head[j]] = i[j] if mode == "Resource": if countryCode.has_key(i[4]): tmp["FullCountry"] = countryCode[i[4]] else: tmp["Country"] = "Unknown" tmp["FullCountry"] = "Unknown" else: if countryCode.has_key(i[3]): tmp["FullCountry"] = countryCode[i[3]] else: tmp["Country"] = "Unknown" tmp["FullCountry"] = "Unknown" c.result.append(tmp) total = result["TotalRecords"] if result.has_key("Extras"): extra = result["Extras"] c.result = { "success": "true", "result": c.result, "total": total, "extra": extra } else: c.result = {"success": "true", "result": c.result, "total": total} return c.result
def _ByJobType( self ): """ By default, all sites are allowed to do every job. The actual rules are freely specified in the Operation JobTypeMapping section. The content of the section may look like this: User { Exclude = PAK Exclude += Ferrara Exclude += Bologna Exclude += Paris Exclude += CERN Exclude += IN2P3 Allow { Paris = IN2P3 CERN = CERN IN2P3 = IN2P3 } } DataReconstruction { Exclude = PAK Exclude += Ferrara Exclude += CERN Exclude += IN2P3 Allow { Ferrara = CERN CERN = CERN IN2P3 = IN2P3 IN2P3 += CERN } } Merge { Exclude = ALL Allow { CERN = CERN IN2P3 = IN2P3 } } The sites in the exclusion list will be removed. The allow section says where each site may help another site """ # 1. get sites list res = getSites() if not res['OK']: gLogger.error( "Could not get the list of sites", res['Message'] ) return res destSites = set( res['Value'] ) # 2. get JobTypeMapping "Exclude" value (and add autoAddedSites) gLogger.debug( "Getting JobTypeMapping 'Exclude' value (and add autoAddedSites)" ) jobType = self.params['JobType'] if not jobType: raise RuntimeError( "No jobType specified" ) excludedSites = self.opsH.getValue( 'JobTypeMapping/%s/Exclude' % jobType, [] ) gLogger.debug( "Explicitly excluded sites for %s task: %s" % ( jobType, ','.join( excludedSites ) ) ) excludedSites += self.opsH.getValue( 'JobTypeMapping/AutoAddedSites', [] ) gLogger.debug( "Full list of excluded sites for %s task: %s" % ( jobType, ','.join( excludedSites ) ) ) # 3. removing sites in Exclude if not excludedSites: pass elif 'ALL' in excludedSites: destSites = set() else: destSites = destSites.difference( set( excludedSites ) ) # 4. get JobTypeMapping "Allow" section res = self.opsH.getOptionsDict( 'JobTypeMapping/%s/Allow' % jobType ) if not res['OK']: gLogger.verbose( res['Message'] ) allowed = {} else: allowed = res['Value'] for site in allowed: allowed[site] = fromChar( allowed[site] ) # 5. add autoAddedSites, if requested autoAddedSites = self.opsH.getValue( 'JobTypeMapping/AutoAddedSites', [] ) if autoAddedSites: for autoAddedSite in autoAddedSites: allowed.setdefault( autoAddedSite, [autoAddedSite] ) if autoAddedSite not in allowed: allowed[autoAddedSite] = [autoAddedSite] else: allowed[autoAddedSite] = [autoAddedSite] + allowed[autoAddedSite] gLogger.debug( "Allowed sites for %s task: %s" % ( jobType, ','.join( allowed ) ) ) # 6. Allowing sites that should be allowed if not self.params['TargetSE'] or self.params['TargetSE'] == 'Unknown': gLogger.warn( "TargetSE is not set: the destination sites list will be incomplete" ) taskSiteDestination = self._BySE() for destSite, fromSites in allowed.iteritems(): for fromSite in fromSites: if taskSiteDestination: if fromSite in taskSiteDestination: destSites.add( destSite ) else: destSites.add( destSite ) gLogger.verbose( "Computed list of destination sites for %s task with TargetSE %s: %s" % ( jobType, self.params['TargetSE'], ','.join( destSites ) ) ) return destSites
def retrieveBucketedData( self, typeName, startTime, endTime, interval, selectFields, condDict, grouping, metainfo ): """ Get data from the DB :param str typeName name of the monitoring type :param int startTime epoch objects. :param int endtime epoch objects. :param dict condDict -> conditions for the query key -> name of the field value -> list of possible values """ retVal = self.getIndexName( typeName ) if not retVal['OK']: return retVal isAvgAgg = False # the data is used to fill the pie charts. This aggregation is used to average the buckets. if metainfo and metainfo.get( 'metric', 'sum' ) == 'avg': isAvgAgg = True indexName = "%s*" % ( retVal['Value'] ) q = [self._Q( 'range', timestamp = {'lte':endTime * 1000, 'gte': startTime * 1000} )] for cond in condDict: query = None for condValue in condDict[cond]: kwargs = {cond: condValue} if query: query = query | self._Q( 'match', **kwargs ) else: query = self._Q( 'match', **kwargs ) q += [query] a1 = self._A( 'terms', field = grouping, size = 0 ) a2 = self._A( 'terms', field = 'timestamp' ) a2.metric( 'total_jobs', 'sum', field = selectFields[0] ) a1.bucket( 'end_data', 'date_histogram', field = 'timestamp', interval = interval ).metric( 'tt', a2 ).pipeline( 'avg_monthly_sales', 'avg_bucket', buckets_path = 'tt>total_jobs', gap_policy = 'insert_zeros' ) if isAvgAgg: a1.pipeline( 'avg_total_jobs', 'avg_bucket', buckets_path = 'end_data>avg_monthly_sales', gap_policy = 'insert_zeros' ) s = self._Search( indexName ) s = s.filter( 'bool', must = q ) s.aggs.bucket( '2', a1 ) #s.fields( ['timestamp'] + selectFields ) gLogger.debug( 'Query:', s.to_dict() ) retVal = s.execute() gLogger.debug( "Query result", len( retVal ) ) result = {} for i in retVal.aggregations['2'].buckets: if isAvgAgg: result[i.key] = i.avg_total_jobs.value else: site = i.key dp = {} for j in i.end_data.buckets: dp[j.key / 1000] = j.avg_monthly_sales.value result[site] = dp # the result format is { 'grouping':{timestamp:value, timestamp:value} for example : # {u'Bookkeeping_BookkeepingManager': {1474300800: 4.0, 1474344000: 4.0, 1474331400: 4.0, 1 # 474302600: 4.0, 1474365600: 4.0, 1474304400: 4.0, 1474320600: 4.0, 1474360200: 4.0, # 1474306200: 4.0, 1474356600: 4.0, 1474336800: 4.0, 1474326000: 4.0, 1474315200: 4.0, # 1474281000: 4.0, 1474309800: 4.0, 1474338600: 4.0, 1474311600: 4.0, 1474317000: 4.0, # 1474367400: 4.0, 1474333200: 4.0, 1474284600: 4.0, 1474362000: 4.0, # 1474327800: 4.0, 1474345800: 4.0, 1474286400: 4.0, 1474308000: 4.0, 1474322400: 4.0, # 1474288200: 4.0, 1474351200: 4.0, 1474282800: 4.0, 1474347600: 4.0, # 1474313400: 4.0, 1474349400: 4.0, 1474297200: 4.0, 1474340400: 4.0, 1474291800: 4.0, # 1474335000: 4.0, 1474293600: 4.0, 1474290000: 4.0, 1474363800: 4.0, # 1474329600: 4.0, 1474353000: 4.0, 1474358400: 4.0, 1474324200: 4.0, 1474354800: 4.0, # 1474295400: 4.0, 1474318800: 4.0, 1474299000: 4.0, 1474342200: 4.0}, # u'Framework_SystemAdministrator': {1474300800: 8.0, 1474344000: 8.0, 1474331400: 8.0, # 1474302600: 8.0, 1474365600: 8.0, 1474304400: 8.0, 1474320600: 8.0, # 1474360200: 8.0, 1474306200: 8.0, 1474356600: 8.0, 1474336800: 8.0, 1474326000: 8.0, # 1474315200: 8.0, 1474281000: 8.0, 1474309800: 8.0, 1474338600: 8.0, # 1474311600: 8.0, 1474317000: 8.0, 1474367400: 8.0, 1474333200: 8.0, 1474284600: 8.0, # 1474362000: 8.0, 1474327800: 8.0, 1474345800: 8.0, 1474286400: 8.0, # 1474308000: 8.0, 1474322400: 8.0, 1474288200: 8.0, 1474351200: 8.0, 1474282800: 8.0, # 1474347600: 8.0, 1474313400: 8.0, 1474349400: 8.0, 1474297200: 8.0, # 1474340400: 8.0, 1474291800: 8.0, 1474335000: 8.0, 1474293600: 8.0, 1474290000: 8.0, # 1474363800: 8.0, 1474329600: 8.0, 1474353000: 8.0, 1474358400: 8.0, # 1474324200: 8.0, 1474354800: 8.0, 1474295400: 8.0, 1474318800: 8.0, 1474299000: 8.0, 1474342200: 8.0}} return S_OK( result )
def __getXRSLExtraString(self): # For the XRSL additional string from configuration - only done at initialisation time # If this string changes, the corresponding (ARC) site directors have to be restarted # # Variable = XRSLExtraString # Default value = '' # If you give a value, I think it should be of the form # (aaa = "xxx") # Otherwise the ARC job description parser will have a fit # Locations searched in order : # Top priority : Resources/Sites/<Grid>/<Site>/CEs/<CE>/XRSLExtraString # Second priority : Resources/Sites/<Grid>/<Site>/XRSLExtraString # Default : Resources/Computing/CEDefaults/XRSLExtraString # self.xrslExtraString = '' # Start with the default value result = getSiteForCE(self.ceHost) self.site = '' if (result['OK']): self.site = result['Value'] else: gLogger.error("Unknown Site ...") return # Now we know the site. Get the grid grid = self.site.split(".")[0] # The different possibilities that we have agreed upon xtraVariable = "XRSLExtraString" firstOption = "Resources/Sites/%s/%s/CEs/%s/%s" % ( grid, self.site, self.ceHost, xtraVariable) secondOption = "Resources/Sites/%s/%s/%s" % (grid, self.site, xtraVariable) defaultOption = "Resources/Computing/CEDefaults/%s" % xtraVariable # Now go about getting the string in the agreed order gLogger.debug("Trying to get xrslExtra string : first option %s" % firstOption) result = gConfig.getValue(firstOption, defaultValue='') if (result != ''): self.xrslExtraString = result gLogger.debug("Found xrslExtra string : %s" % self.xrslExtraString) else: gLogger.debug("Trying to get xrslExtra string : second option %s" % secondOption) result = gConfig.getValue(secondOption, defaultValue='') if (result != ''): self.xrslExtraString = result gLogger.debug("Found xrslExtra string : %s" % self.xrslExtraString) else: gLogger.debug( "Trying to get xrslExtra string : default option %s" % defaultOption) result = gConfig.getValue(defaultOption, defaultValue='') if (result != ''): self.xrslExtraString = result gLogger.debug("Found xrslExtra string : %s" % self.xrslExtraString) if (self.xrslExtraString == ''): gLogger.always( "No XRSLExtra string found in configuration for %s" % self.ceHost) else: gLogger.always("XRSLExtra string : %s" % self.xrslExtraString) gLogger.always(" --- to be added to pilots going to CE : %s" % self.ceHost)
def getSiteSEMapping(self): """ Returns a dictionary of all sites and their localSEs as a list, e.g. {'LCG.CERN.ch':['CERN-RAW','CERN-RDST',...]} """ if self.siteSEMapping: return S_OK(self.siteSEMapping) # Get the list of SEs and keep a mapping of those using an Alias or a # BaseSE storageElements = gConfig.getSections('Resources/StorageElements') if not storageElements['OK']: gLogger.warn('Problem retrieving storage elements', storageElements['Message']) return storageElements storageElements = storageElements['Value'] equivalentSEs = {} for se in storageElements: for option in ('BaseSE', 'Alias'): originalSE = gConfig.getValue( 'Resources/StorageElements/%s/%s' % (se, option)) if originalSE: equivalentSEs.setdefault(originalSE, []).append(se) break siteSEMapping = {} gridTypes = gConfig.getSections('Resources/Sites/') if not gridTypes['OK']: gLogger.warn( 'Problem retrieving sections in /Resources/Sites', gridTypes['Message']) return gridTypes gridTypes = gridTypes['Value'] gLogger.debug('Grid Types are: %s' % (', '.join(gridTypes))) # Get a list of sites and their local SEs siteSet = set() storageElementSet = set() siteSEMapping[LOCAL] = {} for grid in gridTypes: result = gConfig.getSections('/Resources/Sites/%s' % grid) if not result['OK']: gLogger.warn('Problem retrieving /Resources/Sites/%s section' % grid) return result sites = result['Value'] siteSet.update(sites) for site in sites: candidateSEs = gConfig.getValue( '/Resources/Sites/%s/%s/SE' % (grid, site), []) if candidateSEs: candidateSEs += [ eqSE for se in candidateSEs for eqSE in equivalentSEs.get(se, [])] siteSEMapping[LOCAL].setdefault(site, set()).update(candidateSEs) storageElementSet.update(candidateSEs) # Add Sites from the SiteSEMappingByProtocol in the CS siteSEMapping[PROTOCOL] = {} cfgLocalSEPath = cfgPath('SiteSEMappingByProtocol') result = self.__opsHelper.getOptionsDict(cfgLocalSEPath) if result['OK']: sites = result['Value'] for site in sites: candidates = set(self.__opsHelper.getValue( cfgPath(cfgLocalSEPath, site), [])) ses = set(resolveSEGroup(candidates - siteSet) ) | (candidates & siteSet) # If a candidate is a site, then all local SEs are eligible for candidate in ses & siteSet: ses.remove(candidate) ses.update(siteSEMapping[LOCAL][candidate]) siteSEMapping[PROTOCOL].setdefault(site, set()).update(ses) # Add Sites from the SiteSEMappingByDownload in the CS, else # SiteLocalSEMapping (old convention) siteSEMapping[DOWNLOAD] = {} cfgLocalSEPath = cfgPath('SiteSEMappingByDownload') result = self.__opsHelper.getOptionsDict(cfgLocalSEPath) if not result['OK']: cfgLocalSEPath = cfgPath('SiteLocalSEMapping') result = self.__opsHelper.getOptionsDict(cfgLocalSEPath) if result['OK']: sites = result['Value'] for site in sites: candidates = set(self.__opsHelper.getValue( cfgPath(cfgLocalSEPath, site), [])) ses = set(resolveSEGroup(candidates - siteSet) ) | (candidates & siteSet) # If a candidate is a site, then all local SEs are eligible for candidate in ses & siteSet: ses.remove(candidate) ses.update(siteSEMapping[LOCAL][candidate]) siteSEMapping[DOWNLOAD].setdefault(site, set()).update(ses) self.siteSEMapping = siteSEMapping # Add storage elements that may not be associated with a site result = gConfig.getSections('/Resources/StorageElements') if not result['OK']: gLogger.warn( 'Problem retrieving /Resources/StorageElements section', result['Message']) return result self.storageElementSet = storageElementSet | set(result['Value']) self.siteSet = siteSet return S_OK(siteSEMapping)
def getJobOutput(self, jobID, localDir=None): """ Get the specified job standard output and error files. If the localDir is provided, the output is returned as file in this directory. Otherwise, the output is returned as strings. """ result = self._prepareProxy() self.usercfg.ProxyPath(os.environ['X509_USER_PROXY']) if not result['OK']: gLogger.error('ARCComputingElement: failed to set up proxy', result['Message']) return result if jobID.find(':::') != -1: pilotRef, stamp = jobID.split(':::') else: pilotRef = jobID stamp = '' if not stamp: return S_ERROR('Pilot stamp not defined for %s' % pilotRef) job = self.__getARCJob(pilotRef) arcID = os.path.basename(pilotRef) gLogger.debug("Retrieving pilot logs for %s" % pilotRef) if "WorkingDirectory" in self.ceParameters: workingDirectory = os.path.join( self.ceParameters['WorkingDirectory'], arcID) else: workingDirectory = arcID outFileName = os.path.join(workingDirectory, '%s.out' % stamp) errFileName = os.path.join(workingDirectory, '%s.err' % stamp) gLogger.debug("Working directory for pilot output %s" % workingDirectory) isItOkay = job.Retrieve(self.usercfg, arc.URL(workingDirectory), False) if isItOkay: output = None error = None try: with open(outFileName, 'r') as outFile: output = outFile.read() os.unlink(outFileName) with open(errFileName, 'r') as errFile: error = errFile.read() os.unlink(errFileName) except IOError as e: gLogger.error("Error downloading outputs", repr(e).replace(',)', ')')) return S_ERROR("Error downloading outputs") gLogger.debug("Pilot output = %s" % output) gLogger.debug("Pilot error = %s" % error) else: job.Update() arcState = job.State.GetGeneralState() if arcState != "Undefined": return S_ERROR( 'Failed to retrieve output for %s as job is not finished (maybe not started yet)' % jobID) gLogger.debug( "Could not retrieve pilot output for %s - either permission / proxy error or could not connect to CE" % pilotRef) return S_ERROR('Failed to retrieve output for %s' % jobID) return S_OK((output, error))
def __syncComputingElements(self): """ Sync ComputingElements: compares CS with DB and does the necessary modifications. """ res = getCESiteMapping() if not res["OK"]: return res cesCS = list(res["Value"]) gLogger.verbose("%s Computing elements found in CS" % len(cesCS)) cesDB = self.rStatus.selectStatusElement( "Resource", "Status", elementType="ComputingElement", meta={"columns": ["Name"]}) if not cesDB["OK"]: return cesDB cesDB = [ceDB[0] for ceDB in cesDB["Value"]] # ComputingElements that are in DB but not in CS toBeDeleted = list(set(cesDB).difference(set(cesCS))) gLogger.verbose("%s Computing elements to be deleted" % len(toBeDeleted)) # Delete storage elements for ceName in toBeDeleted: deleteQuery = self.rStatus._extermineStatusElement( "Resource", ceName) gLogger.verbose("... %s" % ceName) if not deleteQuery["OK"]: return deleteQuery # statusTypes = RssConfiguration.getValidStatusTypes()[ 'Resource' ] statusTypes = self.rssConfig.getConfigStatusType("ComputingElement") result = self.rStatus.selectStatusElement( "Resource", "Status", elementType="ComputingElement", meta={"columns": ["Name", "StatusType"]}) if not result["OK"]: return result cesTuple = [(x[0], x[1]) for x in result["Value"]] # For each ( se, statusType ) tuple not present in the DB, add it. cesStatusTuples = [(se, statusType) for se in cesCS for statusType in statusTypes] toBeAdded = list(set(cesStatusTuples).difference(set(cesTuple))) gLogger.debug("%s Computing elements entries to be added" % len(toBeAdded)) for ceTuple in toBeAdded: _name = ceTuple[0] _statusType = ceTuple[1] _status = self.defaultStatus _reason = "Synchronized" _elementType = "ComputingElement" query = self.rStatus.addIfNotThereStatusElement( "Resource", "Status", name=_name, statusType=_statusType, status=_status, elementType=_elementType, tokenOwner=self.tokenOwner, reason=_reason, ) if not query["OK"]: return query return S_OK()
def web_ComponentLocation(self): rpcClient = RPCClient("Framework/Monitoring") userData = self.getSessionData() setup = userData['setup'].split('-')[-1] hosts = [] result = Registry.getHosts() if result['OK']: hosts = result['Value'] componentTypes = ['Services', 'Agents'] if "ComponentType" in self.request.arguments: componentTypes = self.request.arguments['ComponentType'] componentNames = [] if "ComponentName" in self.request.arguments: componentNames = list( json.loads(self.request.arguments['ComponentName'][-1])) componentModules = [] if "ComponentModule" in self.request.arguments: componentModules = list( json.loads(self.request.arguments['ComponentModule'][-1])) showAll = 0 if "showAll" in self.request.arguments: showAll = int(self.request.arguments['showAll'][-1]) selectedHosts = [] if "Hosts" in self.request.arguments: # we only use the selected host(s) selectedHosts = list( json.loads(self.request.arguments['Hosts'][-1])) retVal = gConfig.getSections('/Systems') compMatching = {} fullNames = [] if retVal['OK']: systems = retVal['Value'] for i in systems: for compType in componentTypes: compPath = '/Systems/%s/%s/%s' % (i, setup, compType) retVal = gConfig.getSections(compPath) if retVal['OK']: components = retVal['Value'] for j in components: path = '%s/%s' % (i, j) if j in componentNames: fullNames += [path] compMatching[path] = path modulepath = "%s/%s/Module" % (compPath, j) module = gConfig.getValue(modulepath, '') if module != '' and module in componentModules: fullNames += [path] elif module == '' and j in componentModules: fullNames += [path] compMatching[ path] = module if module != '' else path records = [] if fullNames: condDict = {'Setup': userData['setup'], 'ComponentName': fullNames} else: if len(componentTypes) < 2: cType = 'agent' if componentTypes[-1] == 'Agents' else 'service' condDict = {'Setup': userData['setup'], 'Type': cType} else: condDict = {'Setup': userData['setup']} gLogger.debug("condDict" + str(condDict)) retVal = rpcClient.getComponentsStatus(condDict) today = datetime.datetime.today() if retVal['OK']: components = retVal['Value'][0] for setup in components: for cType in components[setup]: for name in components[setup][cType]: for component in components[setup][cType][name]: if selectedHosts and 'Host' in component and component[ 'Host'] not in selectedHosts: continue elif 'Host' in component and component[ 'Host'] not in hosts: continue if 'LastHeartbeat' in component: dateDiff = today - component['LastHeartbeat'] else: dateDiff = today - today if showAll == 0 and dateDiff.days >= 2 and 'Host' in component: continue for conv in component: component[conv] = str(component[conv]) component['ComponentModule'] = compMatching[ component['ComponentName']] if component[ 'ComponentName'] in compMatching else component[ 'ComponentName'] records += [component] result = {"success": "true", "result": records} else: result = {"success": "false", "error": result['Message']} self.finish(result)
def submitJob(self, executableFile, proxy, numberOfJobs=1): """ Method to submit job """ result = self._prepareProxy() self.usercfg.ProxyPath(os.environ['X509_USER_PROXY']) if not result['OK']: gLogger.error('ARCComputingElement: failed to set up proxy', result['Message']) return result gLogger.verbose("Executable file path: %s" % executableFile) if not os.access(executableFile, 5): os.chmod( executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH + stat.S_IXOTH) batchIDList = [] stampDict = {} endpoint = arc.Endpoint(self.ceHost + ":2811/jobs", arc.Endpoint.JOBSUBMIT, "org.nordugrid.gridftpjob") # Submit jobs iteratively for now. Tentatively easier than mucking around with the JobSupervisor class for __i in range(numberOfJobs): # The basic job description jobdescs = arc.JobDescriptionList() # Get the job into the ARC way xrslString, diracStamp = self.__writeXRSL(executableFile) if not arc.JobDescription_Parse(xrslString, jobdescs): gLogger.error("Invalid job description") break # Submit the job jobs = arc.JobList() # filled by the submit process submitter = arc.Submitter(self.usercfg) result = submitter.Submit(endpoint, jobdescs, jobs) # Save info or else ..else. if (result == arc.SubmissionStatus.NONE): # Job successfully submitted pilotJobReference = jobs[0].JobID batchIDList.append(pilotJobReference) stampDict[pilotJobReference] = diracStamp gLogger.debug("Successfully submitted job %s to CE %s" % (pilotJobReference, self.ceHost)) else: message = "Failed to submit job because " if (result.isSet(arc.SubmissionStatus.NOT_IMPLEMENTED)): gLogger.warn( "%s feature not implemented on CE? (weird I know - complain to site admins" % message) if (result.isSet(arc.SubmissionStatus.NO_SERVICES)): gLogger.warn( "%s no services are running on CE? (open GGUS ticket to site admins" % message) if (result.isSet(arc.SubmissionStatus.ENDPOINT_NOT_QUERIED)): gLogger.warn( "%s endpoint was not even queried. (network ..?)" % message) if (result.isSet( arc.SubmissionStatus.BROKER_PLUGIN_NOT_LOADED)): gLogger.warn( "%s BROKER_PLUGIN_NOT_LOADED : ARC library installation problem?" % message) if (result.isSet( arc.SubmissionStatus.DESCRIPTION_NOT_SUBMITTED)): gLogger.warn( "%s no job description was there (Should not happen, but horses can fly (in a plane))" % message) if (result.isSet( arc.SubmissionStatus.SUBMITTER_PLUGIN_NOT_LOADED)): gLogger.warn( "%s SUBMITTER_PLUGIN_NOT_LOADED : ARC library installation problem?" % message) if (result.isSet(arc.SubmissionStatus.AUTHENTICATION_ERROR)): gLogger.warn( "%s authentication error - screwed up / expired proxy? Renew / upload pilot proxy on machine?" % message) if (result.isSet(arc.SubmissionStatus.ERROR_FROM_ENDPOINT)): gLogger.warn( "%s some error from the CE - ask site admins for more information ..." % message) gLogger.warn("%s ... maybe above messages will give a hint." % message) break # Boo hoo *sniff* if batchIDList: result = S_OK(batchIDList) result['PilotStampDict'] = stampDict else: result = S_ERROR( 'No pilot references obtained from the ARC job submission') return result
def web_componentAction(self): """ Actions which should be done on components. The only parameters is an action to perform. Returns standard JSON response structure with with service response or error messages """ userData = self.getSessionData() DN = str(userData["user"]["DN"]) group = str(userData["user"]["group"]) if not (("action" in self.request.arguments) and (len(self.request.arguments["action"][0]) > 0)): self.finish({"success": "false", "error": "No action defined"}) return action = str(self.request.arguments["action"][0]) if action not in ["restart", "start", "stop"]: error = "The request parameters action '%s' is unknown" % action gLogger.debug(error) self.finish({"success": "false", "error": error}) return result = dict() for i in self.request.arguments: if i == "action": continue target = i.split("@") if not len(target) == 2: continue system = self.request.arguments[i][0] gLogger.always("System: %s" % system) host = target[1] gLogger.always("Host: %s" % host) component = target[0] gLogger.always("Component: %s" % component) if not host in result: result[host] = list() result[host].append([system, component]) if not result: error = "Failed to get component(s) for %s" % action gLogger.debug(error) self.finish({"success": "false", "error": error}) gLogger.always(result) actionSuccess = list() actionFailed = list() for hostname in result: if not result[hostname]: continue client = SystemAdministratorClient(hostname, None, delegatedDN=DN, delegatedGroup=group) for i in result[hostname]: system = i[0] component = i[1] try: if action == "restart": result = yield self.threadTask(client.restartComponent, system, component) elif action == "start": result = yield self.threadTask(client.startComponent, system, component) elif action == "stop": result = yield self.threadTask(client.stopComponent, system, component) else: result = list() result["Message"] = "Action %s is not valid" % action except Exception as x: result = list() result["Message"] = "Exception: %s" % str(x) gLogger.debug("Result: %s" % result) if not result["OK"]: error = hostname + ": " + result["Message"] actionFailed.append(error) gLogger.error("Failure during component %s: %s" % (action, error)) else: gLogger.always("Successfully %s component %s" % (action, component)) actionSuccess.append(component) self.finish( self.aftermath(actionSuccess, actionFailed, action, "Component"))
def getCEStatus(self): """ Method to return information on running and pending jobs. We hope to satisfy both instances that use robot proxies and those which use proper configurations. """ result = self._prepareProxy() self.usercfg.ProxyPath(os.environ['X509_USER_PROXY']) if not result['OK']: gLogger.error('ARCComputingElement: failed to set up proxy', result['Message']) return result # Try to find out which VO we are running for. vo = '' res = getVOfromProxyGroup() if res['OK']: vo = res['Value'] result = S_OK() result['SubmittedJobs'] = 0 if not vo: # Presumably the really proper way forward once the infosys-discuss WG comes up with a solution # and it is implemented. Needed for DIRAC instances which use robot certificates for pilots. endpoints = [ arc.Endpoint( "ldap://" + self.ceHost + "/MDS-Vo-name=local,o=grid", arc.Endpoint.COMPUTINGINFO, 'org.nordugrid.ldapng') ] retriever = arc.ComputingServiceRetriever(self.usercfg, endpoints) retriever.wait( ) # Takes a bit of time to get and parse the ldap information targets = retriever.GetExecutionTargets() ceStats = targets[0].ComputingShare gLogger.debug("Running jobs for CE %s : %s" % (self.ceHost, ceStats.RunningJobs)) gLogger.debug("Waiting jobs for CE %s : %s" % (self.ceHost, ceStats.WaitingJobs)) result['RunningJobs'] = ceStats.RunningJobs result['WaitingJobs'] = ceStats.WaitingJobs else: # The system which works properly at present for ARC CEs that are configured correctly. # But for this we need the VO to be known - ask me (Raja) for the whole story if interested. cmd = 'ldapsearch -x -LLL -H ldap://%s:2135 -b mds-vo-name=resource,o=grid "(GlueVOViewLocalID=%s)"' % ( self.ceHost, vo.lower()) res = shellCall(0, cmd) if not res['OK']: gLogger.debug("Could not query CE %s - is it down?" % self.ceHost) return res try: ldapValues = res['Value'][1].split("\n") running = [ lValue for lValue in ldapValues if 'GlueCEStateRunningJobs' in lValue ] waiting = [ lValue for lValue in ldapValues if 'GlueCEStateWaitingJobs' in lValue ] result['RunningJobs'] = int(running[0].split(":")[1]) result['WaitingJobs'] = int(waiting[0].split(":")[1]) except IndexError: res = S_ERROR('Unknown ldap failure for site %s' % self.ceHost) return res return result
def getCAsLocation(): """ Retrieve the CA's files location """ #Grid-Security retVal = gConfig.getOption( '%s/Grid-Security' % g_SecurityConfPath ) if retVal[ 'OK' ]: casPath = "%s/certificates" % retVal[ 'Value' ] gLogger.debug( "Trying %s for CAs" % casPath ) if os.path.isdir( casPath ): gLogger.debug( "Using %s/Grid-Security + /certificates as location for CA's" % g_SecurityConfPath ) return casPath #CAPath retVal = gConfig.getOption( '%s/CALocation' % g_SecurityConfPath ) if retVal[ 'OK' ]: casPath = retVal[ 'Value' ] gLogger.debug( "Trying %s for CAs" % casPath ) if os.path.isdir( casPath ): gLogger.debug( "Using %s/CALocation as location for CA's" % g_SecurityConfPath ) return casPath # Look up the X509_CERT_DIR environment variable if os.environ.has_key( 'X509_CERT_DIR' ): gLogger.debug( "Using X509_CERT_DIR env var as location for CA's" ) casPath = os.environ[ 'X509_CERT_DIR' ] return casPath #rootPath./etc/grid-security/certificates casPath = "%s/etc/grid-security/certificates" % DIRAC.rootPath gLogger.debug( "Trying %s for CAs" % casPath ) if os.path.isdir( casPath ): gLogger.debug( "Using <DIRACRoot>/etc/grid-security/certificates as location for CA's" ) return casPath #/etc/grid-security/certificates casPath = "/etc/grid-security/certificates" gLogger.debug( "Trying %s for CAs" % casPath ) if os.path.isdir( casPath ): gLogger.debug( "Using autodiscovered %s location for CA's" % casPath ) return casPath #No CA's location found return False
def sendMail(self, sendDict=None, title=None, body=None, fromAddress=None): """ Sending an email using sendDict: { e-mail : name } as addressbook title and body is the e-mail's Subject and Body fromAddress is an email address in behalf of whom the message is sent Return success/failure JSON structure """ if not sendDict: result = "" gLogger.debug(result) return {"success": "false", "error": result} if not title: result = "title argument is missing" gLogger.debug(result) return {"success": "false", "error": result} if not body: result = "body argument is missing" gLogger.debug(result) return {"success": "false", "error": result} if not fromAddress: result = "fromAddress argument is missing" gLogger.debug(result) return {"success": "false", "error": result} sentSuccess = list() sentFailed = list() gLogger.debug("Initializing Notification client") ntc = NotificationClient( lambda x, timeout: RPCClient(x, timeout=timeout, static=True)) for email, name in sendDict.iteritems(): result = ntc.sendMail(email, title, body, fromAddress, False) if not result["OK"]: error = name + ": " + result["Message"] sentFailed.append(error) gLogger.error("Sent failure: ", error) else: gLogger.info("Successfully sent to %s" % name) sentSuccess.append(name) success = ", ".join(sentSuccess) failure = "\n".join(sentFailed) if success and failure: result = "Successfully sent e-mail to: " result = result + success + "\n\nFailed to send e-mail to:\n" + failure gLogger.debug(result) return {"success": "true", "result": result} elif success and len(failure) < 1: result = "Successfully sent e-mail to: %s" % success gLogger.debug(result) return {"success": "true", "result": result} elif len(success) < 1 and failure: result = "Failed to sent email to:\n%s" % failure gLogger.debug(result) return {"success": "false", "error": result} result = "No messages were sent due technical failure" gLogger.debug(result) return {"success": "false", "error": result}
def createDataTransformation( flavour, targetSE, sourceSE, metaKey, metaValue, extraData=None, extraname='', groupSize=1, plugin='Broadcast', tGroup=None, tBody=None, enable=False, ): """Creates the replication transformation based on the given parameters. :param str flavour: Flavour of replication to create: Replication or Moving :param targetSE: Destination for files :type targetSE: python:list or str :param sourceSE: Origin of files :type sourceSE: python:list or str :param int metaKey: Meta key to identify input files :param int metaValue: Meta value to identify input files :param dict metaData: Additional meta data to use to identify input files :param str extraname: addition to the transformation name, only needed if the same transformation was already created :param int groupSize: number of files per transformation taks :param str plugin: plugin to use :param str tGroup: transformation group to set :param tBody: transformation body to set :param bool enable: if true submit the transformation, otherwise dry run :returns: S_OK (with the transformation object, if successfully added), S_ERROR """ metadata = {metaKey: metaValue} if isinstance(extraData, dict): metadata.update(extraData) gLogger.debug("Using %r for metadata search" % metadata) if isinstance(targetSE, six.string_types): targetSE = [targetSE] if isinstance(sourceSE, (list, tuple)): sourceSE = '%s' % (",".join(sourceSE)) gLogger.debug('Using plugin: %r' % plugin) if flavour not in ('Replication', 'Moving'): return S_ERROR('Unsupported flavour %s' % flavour) transVerb = {'Replication': 'Replicate', 'Moving': 'Move'}[flavour] transGroup = { 'Replication': 'Replication', 'Moving': 'Moving' }[flavour] if not tGroup else tGroup trans = Transformation() transName = '%s_%s_%s' % (transVerb, str(metaValue), ",".join(targetSE)) if extraname: transName += "_%s" % extraname trans.setTransformationName(transName) description = '%s files for %s %s to %s' % ( transVerb, metaKey, str(metaValue), ",".join(targetSE)) trans.setDescription(description[:255]) trans.setLongDescription(description) trans.setType('Replication') trans.setTransformationGroup(transGroup) trans.setGroupSize(groupSize) trans.setPlugin(plugin) transBody = { 'Moving': [("ReplicateAndRegister", { "SourceSE": sourceSE, "TargetSE": targetSE }), ("RemoveReplica", { "TargetSE": sourceSE })], 'Replication': '', # empty body }[flavour] if tBody is None else tBody trans.setBody(transBody) trans.setInputMetaQuery(metadata) if sourceSE: res = trans.setSourceSE(sourceSE) if not res['OK']: return S_ERROR("SourceSE not valid: %s" % res['Message']) res = trans.setTargetSE(targetSE) if not res['OK']: return S_ERROR("TargetSE not valid: %s" % res['Message']) if not enable: gLogger.always("Dry run, not creating transformation") return S_OK() res = trans.addTransformation() if not res['OK']: return res gLogger.verbose(res) trans.setStatus('Active') trans.setAgentType('Automatic') gLogger.always("Successfully created replication transformation") return S_OK(trans)
def __removeLocal(self, localPath): if os.path.isfile(localPath): gLogger.debug('Remove invalid local file:', localPath) os.remove(localPath)