def _printFormattedDictList( dictList, fields, uniqueField, orderBy ): """ Will print ordered the supplied field of a list of dictionaries """ orderDict = {} fieldWidths = {} dictFields = {} for myDict in dictList: for field in fields: fieldValue = myDict[field] if not fieldWidths.has_key( field ): fieldWidths[field] = len( str( field ) ) if len( str( fieldValue ) ) > fieldWidths[field]: fieldWidths[field] = len( str( fieldValue ) ) orderValue = myDict[orderBy] if not orderDict.has_key( orderValue ): orderDict[orderValue] = [] orderDict[orderValue].append( myDict[uniqueField] ) dictFields[myDict[uniqueField]] = myDict headString = "%s" % fields[0].ljust( fieldWidths[fields[0]] + 5 ) for field in fields[1:]: headString = "%s %s" % ( headString, field.ljust( fieldWidths[field] + 5 ) ) print headString for orderValue in sortList( orderDict.keys() ): uniqueFields = orderDict[orderValue] for uniqueField in sortList( uniqueFields ): myDict = dictFields[uniqueField] outStr = "%s" % str( myDict[fields[0]] ).ljust( fieldWidths[fields[0]] + 5 ) for field in fields[1:]: outStr = "%s %s" % ( outStr, str( myDict[field] ).ljust( fieldWidths[field] + 5 ) ) print outStr
def do_addFile(self, args): """Add new files to transformation DB usage: addFile <lfn> [lfn] """ argss = string.split(args) if not len(argss) > 0: print "no files supplied" return lfnDict = {} for lfn in argss: lfnDict[lfn] = { "PFN": "IGNORED-PFN", "SE": "IGNORED-SE", "Size": 0, "GUID": "IGNORED-GUID", "Checksum": "IGNORED-CHECKSUM", } res = self.server.addFile(lfnDict, force=True) if not res["OK"]: print "failed to add any files: %s" % res["Message"] return for lfn in sortList(res["Value"]["Failed"].keys()): error = res["Value"]["Failed"][lfn] print "failed to add %s: %s" % (lfn, error) for lfn in sortList(res["Value"]["Successful"].keys()): print "added %s" % lfn
def _updateDirectoryUsage(self, directorySEDict, change, connection=False): connection = self._getConnection(connection) for directoryID in sortList(directorySEDict.keys()): result = self.db.dtree.getPathIDsByID(directoryID) if not result["OK"]: return result parentIDs = result["Value"] dirDict = directorySEDict[directoryID] for seID in sortList(dirDict.keys()): seDict = dirDict[seID] files = seDict["Files"] size = seDict["Size"] for dirID in parentIDs: req = ( "UPDATE FC_DirectoryUsage SET SESize=SESize%s%d, SEFiles=SEFiles%s%d, LastUpdate=UTC_TIMESTAMP() " % (change, size, change, files) ) req += "WHERE DirID=%d AND SEID=%d;" % (dirID, seID) res = self.db._update(req) if not res["OK"]: gLogger.warn("Failed to update FC_DirectoryUsage", res["Message"]) if res["Value"]: continue if change != "+": gLogger.warn("Decrement of usage for DirID,SEID that didnt exist", "%d %d" % (dirID, seID)) continue req = "INSERT INTO FC_DirectoryUsage (DirID, SEID, SESize, SEFiles, LastUpdate)" req += " VALUES (%d, %d, %d, %d, UTC_TIMESTAMP());" % (dirID, seID, size, files) res = self.db._update(req) if not res["OK"]: gLogger.warn("Failed to insert FC_DirectoryUsage", res["Message"]) return S_OK()
def test_getTransformations(self): """ Testing the selection of transformations from the database getTransformations This will select all the transformations associated to this test suite and remove them. """ oTrans = Transformation() res = oTrans.getTransformations() self.assert_(res['OK']) parameters = [ 'TransformationID', 'TransformationName', 'Description', 'LongDescription', 'CreationDate', 'LastUpdate', 'AuthorDN', 'AuthorGroup', 'Type', 'Plugin', 'AgentType', 'Status', 'FileMask', 'TransformationGroup', 'GroupSize', 'InheritedFrom', 'Body', 'MaxNumberOfTasks', 'EventsPerTask' ] self.assertEqual(sortList(res['ParameterNames']), sortList(parameters)) self.assertEqual(sortList(res['Value'][0].keys()), sortList(parameters)) self.assertEqual(len(res['Value']), len(res['Records'])) ignore = self.transID for transDict in res['Value']: name = transDict['TransformationName'] if re.search('TestClientTransformation', name): transID = transDict['TransformationID'] if transID != ignore: oTrans = Transformation(transID) res = oTrans.deleteTransformation() self.assert_(res['OK']) self.transID = ignore
def _printFormattedDictList(dictList, fields, uniqueField, orderBy): """ Will print ordered the supplied field of a list of dictionaries """ orderDict = {} fieldWidths = {} dictFields = {} for myDict in dictList: for field in fields: fieldValue = myDict[field] if not fieldWidths.has_key(field): fieldWidths[field] = len(str(field)) if len(str(fieldValue)) > fieldWidths[field]: fieldWidths[field] = len(str(fieldValue)) orderValue = myDict[orderBy] if not orderDict.has_key(orderValue): orderDict[orderValue] = [] orderDict[orderValue].append(myDict[uniqueField]) dictFields[myDict[uniqueField]] = myDict headString = "%s" % fields[0].ljust(fieldWidths[fields[0]] + 5) for field in fields[1:]: headString = "%s %s" % (headString, field.ljust(fieldWidths[field] + 5)) print headString for orderValue in sortList(orderDict.keys()): uniqueFields = orderDict[orderValue] for uniqueField in sortList(uniqueFields): myDict = dictFields[uniqueField] outStr = "%s" % str( myDict[fields[0]]).ljust(fieldWidths[fields[0]] + 5) for field in fields[1:]: outStr = "%s %s" % (outStr, str( myDict[field]).ljust(fieldWidths[field] + 5)) print outStr
def do_addReplica(self, args): ''' Add new replica to the transformation DB usage: addReplica <lfn> <se> ''' argss = args.split() if not len(argss) == 2: print "no file info supplied" return lfn = argss[0] se = argss[1] lfnDict = {} lfnDict[lfn] = { 'PFN': 'IGNORED-PFN', 'SE': se, 'Size': 0, 'GUID': 'IGNORED-GUID', 'Checksum': 'IGNORED-CHECKSUM' } res = self.server.addReplica(lfnDict, force=True) if not res['OK']: print "failed to add replica: %s" % res['Message'] return for lfn in sortList(res['Value']['Failed'].keys()): error = res['Value']['Failed'][lfn] print "failed to add replica: %s" % (error) for lfn in sortList(res['Value']['Successful'].keys()): print "added %s" % lfn
def __globalStatJob(self,siteName): RPC = getRPCClient("WorkloadManagement/JobMonitoring") result = RPC.getStates() stat = [] if result["OK"] and len(result["Value"])>0: for i in result["Value"]: stat.append(str(i)) result = RPC.getJobPageSummaryWeb({'Site': [siteName]},[["JobID","DESC"]],0,1,False) if result["OK"]: result = result["Value"] back = [] if result.has_key("Extras"): extra = result["Extras"] if len(stat) > 0: for i in sortList(stat): if i in sortList(extra.keys()): back.append([i,extra[i]]) else: back.append([i,"-"]) else: for i in sortList(extra.keys()): back.append([i,extra[i]]) c.result = {"success":"true","result":back} c.result = back else: c.result = {"success":"false","error":result["Message"]} gLogger.info("\033[0;31m R E S U L T: \033[0m",c.result) return c.result
def test_ExtendCleanTransformation(self): """ Tests the extension of transformations and the removal of tasks. Also obtain tasks, their status and update their status. extendTransformation() getTransformationTasks() getTransformationTaskStats() deleteTasks() setTaskStatus() cleanTransformation() Tests a transformation can be extended. Tests can obtain the the transformation tasks and their statistics. Tests the removal of already created tasks. Tests can change the status of a task. Tests that Cleaning a transformation removes tasks defined for the transformation. """ oTrans = Transformation(self.transID) nTasks = 100 res = oTrans.extendTransformation(nTasks) self.assert_(res['OK']) taskIDs = res['Value'] self.assertEqual(len(taskIDs),nTasks) res = oTrans.getTransformationTasks() self.assert_(res['OK']) parameters = ['TargetSE', 'TransformationID', 'LastUpdateTime', 'ExternalID', 'CreationTime', 'TaskID', 'ExternalStatus'] self.assertEqual(sortList(res['ParameterNames']),sortList(parameters)) self.assertEqual(sortList(res['Value'][0].keys()),sortList(parameters)) self.assertEqual(res['Value'][0]['TargetSE'],'Unknown') self.assertEqual(res['Value'][0]['TransformationID'],self.transID) self.assertEqual(res['Value'][0]['ExternalID'],'0') self.assertEqual(res['Value'][0]['TaskID'],1) self.assertEqual(res['Value'][0]['ExternalStatus'],'Created') self.assertEqual(res['Records'][0][0],1) self.assertEqual(res['Records'][0][1],self.transID) self.assertEqual(res['Records'][0][2],'Created') self.assertEqual(res['Records'][0][3],'0') self.assertEqual(res['Records'][0][4],'Unknown') res = oTrans.getTransformationTaskStats() self.assert_(res['OK']) self.assertEqual(res['Value']['Created'],100) res = oTrans.deleteTasks(11,100) self.assert_(res['OK']) res = oTrans.getTransformationTaskStats() self.assert_(res['OK']) self.assertEqual(res['Value']['Created'],10) res = oTrans.setTaskStatus(1, 'Done') self.assert_(res['OK']) res = oTrans.getTransformationTaskStats() self.assert_(res['OK']) self.assertEqual(res['Value']['Created'],10) self.assertEqual(res['Value']['Done'],1) res = oTrans.cleanTransformation() self.assert_(res['OK']) res = oTrans.getStatus() self.assert_(res['OK']) self.assertEqual(res['Value'],'Cleaned') res = oTrans.getTransformationTasks() self.assert_(res['OK']) self.assertFalse(res['Value']) self.assertFalse(res['Records'])
def __globalStatJob(self, siteName): RPC = getRPCClient("WorkloadManagement/JobMonitoring") result = RPC.getStates() stat = [] if result["OK"] and len(result["Value"]) > 0: for i in result["Value"]: stat.append(str(i)) result = RPC.getJobPageSummaryWeb({'Site': [siteName]}, [["JobID", "DESC"]], 0, 1, False) if result["OK"]: result = result["Value"] back = [] if result.has_key("Extras"): extra = result["Extras"] if len(stat) > 0: for i in sortList(stat): if i in sortList(extra.keys()): back.append([i, extra[i]]) else: back.append([i, "-"]) else: for i in sortList(extra.keys()): back.append([i, extra[i]]) c.result = {"success": "true", "result": back} c.result = back else: c.result = {"success": "false", "error": result["Message"]} gLogger.info("\033[0;31m R E S U L T: \033[0m", c.result) return c.result
def do_removeReplica(self, args): """Remove replica from the transformation DB usage: removeReplica <lfn> <se> """ argss = string.split(args) if not len(argss) == 2: print "no file info supplied" return lfn = argss[0] se = argss[1] lfnDict = {} lfnDict[lfn] = { "PFN": "IGNORED-PFN", "SE": se, "Size": 0, "GUID": "IGNORED-GUID", "Checksum": "IGNORED-CHECKSUM", } res = self.server.removeReplica(lfnDict) if not res["OK"]: print "failed to remove replica: %s" % res["Message"] return for lfn in sortList(res["Value"]["Failed"].keys()): error = res["Value"]["Failed"][lfn] print "failed to remove replica: %s" % (error) for lfn in sortList(res["Value"]["Successful"].keys()): print "removed %s" % lfn
def dump(self): """ Dump to the logger a summary of the StorageElement items """ gLogger.info( "StorageElement.dump: Preparing dump for StorageElement %s." % self.name) if not self.valid: gLogger.error( "StorageElement.dump: Failed to create StorageElement plugins.", self.errorReason) return i = 1 outStr = "\n\n============ Options ============\n" for key in sortList(self.options.keys()): outStr = "%s%s: %s\n" % (outStr, key.ljust(15), self.options[key]) for storage in self.storages: outStr = "%s============Protocol %s ============\n" % (outStr, i) res = storage.getParameters() storageParameters = res['Value'] for key in sortList(storageParameters.keys()): outStr = "%s%s: %s\n" % (outStr, key.ljust(15), storageParameters[key]) i = i + 1 gLogger.info(outStr)
def do_setReplicaStatus(self, args): """Set replica status, usually used to mark a replica Problematic usage: setReplicaStatus <lfn> <status> <se> """ argss = string.split(args) if not len(argss) > 2: print "no file info supplied" return lfn = argss[0] status = argss[1] se = argss[2] lfnDict = {} lfnDict[lfn] = { "Status": status, "PFN": "IGNORED-PFN", "SE": se, "Size": 0, "GUID": "IGNORED-GUID", "Checksum": "IGNORED-CHECKSUM", } res = self.server.setReplicaStatus(lfnDict) if not res["OK"]: print "failed to set replica status: %s" % res["Message"] return for lfn in sortList(res["Value"]["Failed"].keys()): error = res["Value"]["Failed"][lfn] print "failed to set replica status: %s" % (error) for lfn in sortList(res["Value"]["Successful"].keys()): print "updated replica status %s" % lfn
def _updateDirectoryUsage( self, directorySEDict, change, connection = False ): connection = self._getConnection( connection ) for dirID in sortList( directorySEDict.keys() ): dirDict = directorySEDict[dirID] for seID in sortList( dirDict.keys() ): seDict = dirDict[seID] files = seDict['Files'] size = seDict['Size'] req = "UPDATE FC_DirectoryUsage SET SESize=SESize%s%d, SEFiles=SEFiles%s%d, LastUpdate=UTC_TIMESTAMP() " \ % ( change, size, change, files ) req += "WHERE DirID=%d AND SEID=%d;" % ( dirID, seID ) res = self.db._update( req ) if not res['OK']: gLogger.warn( "Failed to update FC_DirectoryUsage", res['Message'] ) if res['Value']: continue if change != '+': gLogger.warn( "Decrement of usage for DirID,SEID that didnt exist", "%d %d" % ( dirID, seID ) ) continue req = "INSERT INTO FC_DirectoryUsage (DirID, SEID, SESize, SEFiles, LastUpdate)" req += " VALUES (%d, %d, %d, %d, UTC_TIMESTAMP());" % ( dirID, seID, size, files ) res = self.db._update( req ) if not res['OK']: gLogger.warn( "Failed to insert FC_DirectoryUsage", res['Message'] ) return S_OK()
def do_setReplicaStatus(self, args): """Set replica status, usually used to mark a replica Problematic usage: setReplicaStatus <lfn> <status> <se> """ argss = string.split(args) if not len(argss) > 2: print "no file info supplied" return lfn = argss[0] status = argss[1] se = argss[2] lfnDict = {} lfnDict[lfn] = { 'Status': status, 'PFN': 'IGNORED-PFN', 'SE': se, 'Size': 0, 'GUID': 'IGNORED-GUID', 'Checksum': 'IGNORED-CHECKSUM' } res = self.server.setReplicaStatus(lfnDict) if not res['OK']: print "failed to set replica status: %s" % res['Message'] return for lfn in sortList(res['Value']['Failed'].keys()): error = res['Value']['Failed'][lfn] print "failed to set replica status: %s" % (error) for lfn in sortList(res['Value']['Successful'].keys()): print "updated replica status %s" % lfn
def initialize( self ): """Sets defaults """ self.replicaManager = ReplicaManager() self.transClient = TransformationClient() self.wmsClient = WMSClient() self.requestClient = RequestClient() self.metadataClient = FileCatalogClient() self.storageUsageClient = StorageUsageClient() # This sets the Default Proxy to used as that defined under # /Operations/Shifter/DataManager # the shifterProxy option in the Configuration can be used to change this default. self.am_setOption( 'shifterProxy', 'DataManager' ) self.transformationTypes = sortList( self.am_getOption( 'TransformationTypes', ['MCSimulation', 'DataReconstruction', 'DataStripping', 'MCStripping', 'Merge', 'Replication'] ) ) gLogger.info( "Will consider the following transformation types: %s" % str( self.transformationTypes ) ) self.directoryLocations = sortList( self.am_getOption( 'DirectoryLocations', ['TransformationDB', 'StorageUsage', 'MetadataCatalog'] ) ) gLogger.info( "Will search for directories in the following locations: %s" % str( self.directoryLocations ) ) self.transfidmeta = self.am_getOption( 'TransfIDMeta', "TransformationID" ) gLogger.info( "Will use %s as metadata tag name for TransformationID" % self.transfidmeta ) self.archiveAfter = self.am_getOption( 'ArchiveAfter', 7 ) # days gLogger.info( "Will archive Completed transformations after %d days" % self.archiveAfter ) self.activeStorages = sortList( self.am_getOption( 'ActiveSEs', [] ) ) gLogger.info( "Will check the following storage elements: %s" % str( self.activeStorages ) ) self.logSE = self.am_getOption( 'TransformationLogSE', 'LogSE' ) gLogger.info( "Will remove logs found on storage element: %s" % self.logSE ) return S_OK()
def do_removeReplica(self, args): """Remove replica from the transformation DB usage: removeReplica <lfn> <se> """ argss = string.split(args) if not len(argss) == 2: print "no file info supplied" return lfn = argss[0] se = argss[1] lfnDict = {} lfnDict[lfn] = { 'PFN': 'IGNORED-PFN', 'SE': se, 'Size': 0, 'GUID': 'IGNORED-GUID', 'Checksum': 'IGNORED-CHECKSUM' } res = self.server.removeReplica(lfnDict) if not res['OK']: print "failed to remove replica: %s" % res['Message'] return for lfn in sortList(res['Value']['Failed'].keys()): error = res['Value']['Failed'][lfn] print "failed to remove replica: %s" % (error) for lfn in sortList(res['Value']['Successful'].keys()): print "removed %s" % lfn
def do_addFile(self, args): """Add new files to transformation DB usage: addFile <lfn> [lfn] """ argss = string.split(args) if not len(argss) > 0: print "no files supplied" return lfnDict = {} for lfn in argss: lfnDict[lfn] = { 'PFN': 'IGNORED-PFN', 'SE': 'IGNORED-SE', 'Size': 0, 'GUID': 'IGNORED-GUID', 'Checksum': 'IGNORED-CHECKSUM' } res = self.server.addFile(lfnDict, force=True) if not res['OK']: print "failed to add any files: %s" % res['Message'] return for lfn in sortList(res['Value']['Failed'].keys()): error = res['Value']['Failed'][lfn] print "failed to add %s: %s" % (lfn, error) for lfn in sortList(res['Value']['Successful'].keys()): print "added %s" % lfn
def _updateDirectoryUsage(self, directorySEDict, change, connection=False): connection = self._getConnection(connection) for dirID in sortList(directorySEDict.keys()): dirDict = directorySEDict[dirID] for seID in sortList(dirDict.keys()): seDict = dirDict[seID] files = seDict['Files'] size = seDict['Size'] req = "UPDATE FC_DirectoryUsage SET SESize=SESize%s%d, SEFiles=SEFiles%s%d, LastUpdate=UTC_TIMESTAMP() " \ % ( change, size, change, files ) req += "WHERE DirID=%d AND SEID=%d;" % (dirID, seID) res = self.db._update(req) if not res['OK']: gLogger.warn("Failed to update FC_DirectoryUsage", res['Message']) if res['Value']: continue if change != '+': gLogger.warn( "Decrement of usage for DirID,SEID that didnt exist", "%d %d" % (dirID, seID)) continue req = "INSERT INTO FC_DirectoryUsage (DirID, SEID, SESize, SEFiles, LastUpdate)" req += " VALUES (%d, %d, %d, %d, UTC_TIMESTAMP());" % ( dirID, seID, size, files) res = self.db._update(req) if not res['OK']: gLogger.warn("Failed to insert FC_DirectoryUsage", res['Message']) return S_OK()
def __checkPhysicalFiles( self, replicas, catalogMetadata, ses = [] ): """ This obtains the physical file metadata and checks the metadata against the catalog entries """ sePfns = {} pfnLfns = {} for lfn, replicaDict in replicas.items(): for se, pfn in replicaDict.items(): if ( ses ) and ( se not in ses ): continue if not sePfns.has_key( se ): sePfns[se] = [] sePfns[se].append( pfn ) pfnLfns[pfn] = lfn gLogger.info( '%s %s' % ( 'Storage Element'.ljust( 20 ), 'Replicas'.rjust( 20 ) ) ) for site in sortList( sePfns.keys() ): files = len( sePfns[site] ) gLogger.info( '%s %s' % ( site.ljust( 20 ), str( files ).rjust( 20 ) ) ) for se in sortList( sePfns.keys() ): pfns = sePfns[se] pfnDict = {} for pfn in pfns: pfnDict[pfn] = pfnLfns[pfn] sizeMismatch = [] res = self.__checkPhysicalFileMetadata( pfnDict, se ) if not res['OK']: gLogger.error( 'Failed to get physical file metadata.', res['Message'] ) return res for pfn, metadata in res['Value'].items(): if catalogMetadata.has_key( pfnLfns[pfn] ): if ( metadata['Size'] != catalogMetadata[pfnLfns[pfn]]['Size'] ) and ( metadata['Size'] != 0 ): sizeMismatch.append( ( pfnLfns[pfn], pfn, se, 'CatalogPFNSizeMismatch' ) ) if sizeMismatch: self.__reportProblematicReplicas( sizeMismatch, se, 'CatalogPFNSizeMismatch' ) return S_OK()
def test_AddFilesGetFilesSetFileStatus(self): """ Testing adding, getting and setting file status. addFilesToTransformation() getTransformationFiles() getTransformationStats() setFileStatusForTransformation() addTaskForTransformation() Test adding and files to transformation. Test selecting the files for the transformation. Test getting the status count of the transformation files. Test setting the file status for transformation. Test creating a task for the added files and ensure the status is updated correctly. """ oTrans = Transformation(self.transID) lfns = ['/test/lfn/file1', '/test/lfn/file2'] res = oTrans.addFilesToTransformation(lfns) self.assert_(res['OK']) res = oTrans.getTransformationFiles() self.assert_(res['OK']) self.assertEqual(sortList(lfns), res['LFNs']) self.assertEqual(len(lfns), len(res['Records'])) self.assertEqual(len(lfns), len(res['Value'])) fileParams = sortList([ 'LFN', 'TransformationID', 'FileID', 'Status', 'TaskID', 'TargetSE', 'UsedSE', 'ErrorCount', 'LastUpdate', 'InsertedTime' ]) self.assertEqual(fileParams, sortList(res['ParameterNames'])) self.assertEqual(res['Records'][0][0], lfns[0]) self.assertEqual(res['Value'][0]['LFN'], lfns[0]) self.assertEqual(res['Records'][0][1], self.transID) self.assertEqual(res['Value'][0]['TransformationID'], self.transID) self.assertEqual(res['Records'][0][3], 'Unused') self.assertEqual(res['Value'][0]['Status'], 'Unused') res = oTrans.getTransformationStats() self.assert_(res['OK']) self.assertEqual(res['Value']['Total'], 2) self.assertEqual(res['Value']['Unused'], 2) res = oTrans.setFileStatusForTransformation('Processed', [lfns[0]]) self.assert_(res['OK']) res = oTrans.getTransformationStats() self.assert_(res['OK']) self.assertEqual(res['Value']['Total'], 2) self.assertEqual(res['Value']['Unused'], 1) self.assertEqual(res['Value']['Processed'], 1) res = oTrans.setFileStatusForTransformation('Unused', [lfns[0]]) self.assert_(res['OK']) self.assert_(res['Value']['Failed'].has_key(lfns[0])) self.assertEqual(res['Value']['Failed'][lfns[0]], 'Can not change Processed status') res = oTrans.addTaskForTransformation(lfns=[lfns[1]], se='Test') self.assert_(res['OK']) res = oTrans.getTransformationStats() self.assert_(res['OK']) self.assertEqual(res['Value']['Total'], 2) self.assertEqual(res['Value']['Assigned'], 1) self.assertEqual(res['Value']['Processed'], 1)
def _ByShare(self, shareType='CPU'): """ first get the shares from the CS, and then makes the grouping looking at the history """ res = self._getShares(shareType, normalise=True) if not res['OK']: return res cpuShares = res['Value'] gLogger.info("Obtained the following target shares (%):") for site in sortList(cpuShares.keys()): gLogger.info("%s: %.1f" % (site.ljust(15), cpuShares[site])) # Get the existing destinations from the transformationDB res = self._getExistingCounters(requestedSites=cpuShares.keys()) if not res['OK']: gLogger.error("Failed to get existing file share", res['Message']) return res existingCount = res['Value'] if existingCount: gLogger.info("Existing site utilization (%):") normalisedExistingCount = self._normaliseShares( existingCount.copy()) for se in sortList(normalisedExistingCount.keys()): gLogger.info("%s: %.1f" % (se.ljust(15), normalisedExistingCount[se])) # Group the input files by their existing replicas res = self._groupByReplicas() if not res['OK']: return res replicaGroups = res['Value'] tasks = [] # For the replica groups for replicaSE, lfns in replicaGroups: possibleSEs = replicaSE.split(',') # Determine the next site based on requested shares, existing usage and candidate sites res = self._getNextSite( existingCount, cpuShares, candidates=self._getSitesForSEs(possibleSEs)) if not res['OK']: gLogger.error("Failed to get next destination SE", res['Message']) continue targetSite = res['Value'] # Resolve the ses for the target site res = getSEsForSite(targetSite) if not res['OK']: continue ses = res['Value'] # Determine the selected SE and create the task for chosenSE in ses: if chosenSE in possibleSEs: tasks.append((chosenSE, lfns)) if not existingCount.has_key(targetSite): existingCount[targetSite] = 0 existingCount[targetSite] += len(lfns) return S_OK(tasks)
def test_AddFilesGetFilesSetFileStatus(self): """ Testing adding, getting and setting file status. addFilesToTransformation() getTransformationFiles() getTransformationStats() setFileStatusForTransformation() addTaskForTransformation() Test adding and files to transformation. Test selecting the files for the transformation. Test getting the status count of the transformation files. Test setting the file status for transformation. Test creating a task for the added files and ensure the status is updated correctly. """ oTrans = Transformation(self.transID) lfns = ['/test/lfn/file1','/test/lfn/file2'] res = oTrans.addFilesToTransformation(lfns) self.assert_(res['OK']) res = oTrans.getTransformationFiles() self.assert_(res['OK']) self.assertEqual(sortList(lfns),res['LFNs']) self.assertEqual(len(lfns),len(res['Records'])) self.assertEqual(len(lfns),len(res['Value'])) fileParams = sortList(['LFN', 'TransformationID', 'FileID', 'Status', 'TaskID', 'TargetSE', 'UsedSE', 'ErrorCount', 'LastUpdate', 'InsertedTime']) self.assertEqual(fileParams,sortList(res['ParameterNames'])) self.assertEqual(res['Records'][0][0], lfns[0]) self.assertEqual(res['Value'][0]['LFN'],lfns[0]) self.assertEqual(res['Records'][0][1], self.transID) self.assertEqual(res['Value'][0]['TransformationID'], self.transID) self.assertEqual(res['Records'][0][3],'Unused') self.assertEqual(res['Value'][0]['Status'],'Unused') res = oTrans.getTransformationStats() self.assert_(res['OK']) self.assertEqual(res['Value']['Total'],2) self.assertEqual(res['Value']['Unused'],2) res = oTrans.setFileStatusForTransformation('Processed',[lfns[0]]) self.assert_(res['OK']) res = oTrans.getTransformationStats() self.assert_(res['OK']) self.assertEqual(res['Value']['Total'],2) self.assertEqual(res['Value']['Unused'],1) self.assertEqual(res['Value']['Processed'],1) res = oTrans.setFileStatusForTransformation('Unused',[lfns[0]]) self.assert_(res['OK']) self.assert_(res['Value']['Failed'].has_key(lfns[0])) self.assertEqual(res['Value']['Failed'][lfns[0]],'Can not change Processed status') res = oTrans.addTaskForTransformation(lfns=[lfns[1]],se='Test') self.assert_(res['OK']) res = oTrans.getTransformationStats() self.assert_(res['OK']) self.assertEqual(res['Value']['Total'],2) self.assertEqual(res['Value']['Assigned'],1) self.assertEqual(res['Value']['Processed'],1)
def __verifyPfns( self, pfnSizes, storageElements ): gLogger.info( 'Checking %s storage files exist in the catalog' % len( pfnSizes ) ) pfnsToRemove = [] incorrectlyRegistered = [] allDone = True # First get all the PFNs as they should be registered in the catalog for pfns in breakListIntoChunks( sortList( pfnSizes.keys() ), 100 ): res = self.replicaManager.getPfnForProtocol( pfns, storageElements[0], withPort = False ) if not res['OK']: allDone = False continue for pfn, error in res['Value']['Failed'].items(): gLogger.error( 'Failed to obtain registered PFN for physical file', '%s %s' % ( pfn, error ) ) if res['Value']['Failed']: allDone = False catalogStoragePfns = res['Value']['Successful'] # Determine whether these PFNs are registered and if so obtain the LFN res = self.replicaManager.getCatalogLFNForPFN( catalogStoragePfns.values() ) if not res['OK']: allDone = False continue for surl in sortList( res['Value']['Failed'].keys() ): if res['Value']['Failed'][surl] == 'No such file or directory': #pfnsToRemove.append(surl) print surl else: gLogger.error( 'Failed to get LFN for PFN', '%s %s' % ( surl, res['Value']['Failed'][surl] ) ) existingLFNs = res['Value']['Successful'].values() if existingLFNs: res = self.replicaManager.getCatalogReplicas( existingLFNs ) if not res['OK']: allDone = False continue for lfn, error in res['Value']['Failed'].items(): gLogger.error( 'Failed to obtain registered replicas for LFN', '%s %s' % ( lfn, error ) ) if res['Value']['Failed']: allDone = False for lfn, replicas in res['Value']['Successful'].items(): match = False for storageElement in storageElements: if storageElement in replicas.keys(): match = True if not match: pass#incorrectlyRegistered.append(lfn) #print lfn gLogger.info( "Verification of PFNs complete" ) if incorrectlyRegistered: gLogger.info( "Found %d files incorrectly registered" % len( incorrectlyRegistered ) ) if pfnsToRemove: gLogger.info( "Found %d files to be removed" % len( pfnsToRemove ) ) resDict = {'Remove':pfnsToRemove, 'ReRegister':incorrectlyRegistered, 'AllDone':allDone} return S_OK( resDict )
def _ByShare( self, shareType = 'CPU' ): """ first get the shares from the CS, and then makes the grouping looking at the history """ res = self._getShares( shareType, normalise = True ) if not res['OK']: return res cpuShares = res['Value'] gLogger.info( "Obtained the following target shares (%):" ) for site in sortList( cpuShares.keys() ): gLogger.info( "%s: %.1f" % ( site.ljust( 15 ), cpuShares[site] ) ) # Get the existing destinations from the transformationDB res = self._getExistingCounters( requestedSites = cpuShares.keys() ) if not res['OK']: gLogger.error( "Failed to get existing file share", res['Message'] ) return res existingCount = res['Value'] if existingCount: gLogger.info( "Existing site utilization (%):" ) normalisedExistingCount = self._normaliseShares( existingCount.copy() ) for se in sortList( normalisedExistingCount.keys() ): gLogger.info( "%s: %.1f" % ( se.ljust( 15 ), normalisedExistingCount[se] ) ) # Group the input files by their existing replicas res = self._groupByReplicas() if not res['OK']: return res replicaGroups = res['Value'] tasks = [] # For the replica groups for replicaSE, lfns in replicaGroups: possibleSEs = replicaSE.split( ',' ) # Determine the next site based on requested shares, existing usage and candidate sites res = self._getNextSite( existingCount, cpuShares, candidates = self._getSitesForSEs( possibleSEs ) ) if not res['OK']: gLogger.error( "Failed to get next destination SE", res['Message'] ) continue targetSite = res['Value'] # Resolve the ses for the target site res = getSEsForSite( targetSite ) if not res['OK']: continue ses = res['Value'] # Determine the selected SE and create the task for chosenSE in ses: if chosenSE in possibleSEs: tasks.append( ( chosenSE, lfns ) ) if not existingCount.has_key( targetSite ): existingCount[targetSite] = 0 existingCount[targetSite] += len( lfns ) return S_OK( tasks )
def checkTransformationIntegrity(self, transID): """ This method contains the real work """ gLogger.info("-" * 40) gLogger.info("Checking the integrity of transformation %s" % transID) gLogger.info("-" * 40) res = self.getTransformationDirectories(transID) if not res['OK']: return res directories = res['Value'] if not directories: return S_OK() ###################################################### # # This check performs Catalog->SE for possible output directories # res = self.replicaManager.getCatalogExists(directories) if not res['OK']: gLogger.error(res['Message']) return res for directory, error in res['Value']['Failed']: gLogger.error('Failed to determine existance of directory', '%s %s' % (directory, error)) if res['Value']['Failed']: return S_ERROR("Failed to determine the existance of directories") directoryExists = res['Value']['Successful'] for directory in sortList(directoryExists.keys()): if not directoryExists[directory]: continue iRes = self.integrityClient.catalogDirectoryToSE(directory) if not iRes['OK']: gLogger.error(iRes['Message']) return iRes ###################################################### # # This check performs SE->Catalog for possible output directories # for storageElementName in sortList(self.activeStorages): res = self.integrityClient.storageDirectoryToCatalog( directories, storageElementName) if not res['OK']: gLogger.error(res['Message']) return res gLogger.info("-" * 40) gLogger.info("Completed integrity check for transformation %s" % transID) return S_OK()
def initializeFileCatalogHandler(serviceInfo): global fcDB serviceCS = serviceInfo['serviceSectionPath'] # Instantiate the requested database dbLocation = gConfig.getValue('%s/Database' % serviceCS, 'DataManagement/FileCatalogDB') fcDB = FileCatalogDB(dbLocation) databaseConfig = {} # Obtain the plugins to be used for DB interaction gLogger.info("Initializing with FileCatalog with following managers:") defaultManagers = { 'UserGroupManager': 'UserAndGroupManagerDB', 'SEManager': 'SEManagerDB', 'SecurityManager': 'NoSecurityManager', 'DirectoryManager': 'DirectoryLevelTree', 'FileManager': 'FileManager', 'DirectoryMetadata': 'DirectoryMetadata', 'FileMetadata': 'FileMetadata' } for configKey in sortList(defaultManagers.keys()): defaultValue = defaultManagers[configKey] configValue = gConfig.getValue('%s/%s' % (serviceCS, configKey), defaultValue) gLogger.info("%s : %s" % (str(configKey).ljust(20), str(configValue).ljust(20))) databaseConfig[configKey] = configValue # Obtain some general configuration of the database gLogger.info( "Initializing the FileCatalog with the following configuration:") defaultConfig = { 'UniqueGUID': False, 'GlobalReadAccess': True, 'LFNPFNConvention': True, 'ResolvePFN': True, 'DefaultUmask': 0775, 'VisibleStatus': ['AprioriGood'] } for configKey in sortList(defaultConfig.keys()): defaultValue = defaultConfig[configKey] configValue = gConfig.getValue('%s/%s' % (serviceCS, configKey), defaultValue) gLogger.info("%s : %s" % (str(configKey).ljust(20), str(configValue).ljust(20))) databaseConfig[configKey] = configValue res = fcDB.setConfig(databaseConfig) return res
def initializeFileCatalogHandler(serviceInfo): """ handler initialisation """ global gFileCatalogDB dbLocation = getServiceOption(serviceInfo, 'Database', 'DataManagement/FileCatalogDB') gFileCatalogDB = FileCatalogDB(dbLocation) databaseConfig = {} # Obtain the plugins to be used for DB interaction gLogger.info("Initializing with FileCatalog with following managers:") defaultManagers = { 'UserGroupManager': 'UserAndGroupManagerDB', 'SEManager': 'SEManagerDB', 'SecurityManager': 'NoSecurityManager', 'DirectoryManager': 'DirectoryLevelTree', 'FileManager': 'FileManager', 'DirectoryMetadata': 'DirectoryMetadata', 'FileMetadata': 'FileMetadata', 'DatasetManager': 'DatasetManager' } for configKey in sortList(defaultManagers.keys()): defaultValue = defaultManagers[configKey] configValue = getServiceOption(serviceInfo, configKey, defaultValue) gLogger.info("%-20s : %-20s" % (str(configKey), str(configValue))) databaseConfig[configKey] = configValue # Obtain some general configuration of the database gLogger.info( "Initializing the FileCatalog with the following configuration:") defaultConfig = { 'UniqueGUID': False, 'GlobalReadAccess': True, 'LFNPFNConvention': 'Strong', 'ResolvePFN': True, 'DefaultUmask': 0775, 'ValidFileStatus': ['AprioriGood', 'Trash', 'Removing', 'Probing'], 'ValidReplicaStatus': ['AprioriGood', 'Trash', 'Removing', 'Probing'], 'VisibleFileStatus': ['AprioriGood'], 'VisibleReplicaStatus': ['AprioriGood'] } for configKey in sortList(defaultConfig.keys()): defaultValue = defaultConfig[configKey] configValue = getServiceOption(serviceInfo, configKey, defaultValue) gLogger.info("%-20s : %-20s" % (str(configKey), str(configValue))) databaseConfig[configKey] = configValue res = gFileCatalogDB.setConfig(databaseConfig) return res
def initialize(self): """ agent initialisation reading and setting confing opts :param self: self reference """ # # shifter proxy self.am_setOption('shifterProxy', 'DataManager') # # transformations types self.dataProcTTypes = Operations().getValue( 'Transformations/DataProcessing', ['MCSimulation', 'Merge']) self.dataManipTTypes = Operations().getValue( 'Transformations/DataManipulation', ['Replication', 'Removal']) agentTSTypes = self.am_getOption('TransformationTypes', []) if agentTSTypes: self.transformationTypes = sortList(agentTSTypes) else: self.transformationTypes = sortList(self.dataProcTTypes + self.dataManipTTypes) self.log.info("Will consider the following transformation types: %s" % str(self.transformationTypes)) # # directory locations self.directoryLocations = sortList( self.am_getOption('DirectoryLocations', ['TransformationDB', 'MetadataCatalog'])) self.log.info( "Will search for directories in the following locations: %s" % str(self.directoryLocations)) # # transformation metadata self.transfidmeta = self.am_getOption('TransfIDMeta', "TransformationID") self.log.info("Will use %s as metadata tag name for TransformationID" % self.transfidmeta) # # archive periof in days self.archiveAfter = self.am_getOption('ArchiveAfter', 7) # days self.log.info("Will archive Completed transformations after %d days" % self.archiveAfter) # # active SEs self.activeStorages = sortList(self.am_getOption('ActiveSEs', [])) self.log.info("Will check the following storage elements: %s" % str(self.activeStorages)) # # transformation log SEs self.logSE = self.am_getOption('TransformationLogSE', 'LogSE') self.log.info("Will remove logs found on storage element: %s" % self.logSE) # # enable/disable execution, should be using CS option Status?? with default value as 'Active'?? self.enableFlag = self.am_getOption('EnableFlag', 'True') return S_OK()
def __init__( self, agentName, loadName, baseAgentName, properties = {} ): ''' c'tor ''' AgentModule.__init__( self, agentName, loadName, baseAgentName, properties ) self.transClient = TransformationClient() agentTSTypes = self.am_getOption( 'TransformationTypes', [] ) if agentTSTypes: self.transformationTypes = sortList( agentTSTypes ) else: self.transformationTypes = sortList( Operations().getValue( 'Transformations/ExtendableTransfTypes', ['MCSimulation', 'Simulation'] ) ) self.maxIterationTasks = self.am_getOption( 'TasksPerIteration', 50 ) self.maxFailRate = self.am_getOption( 'MaxFailureRate', 30 ) self.maxWaitingJobs = self.am_getOption( 'MaxWaitingJobs', 1000 )
def __init__(self, *args, **kwargs): """ c'tor """ AgentModule.__init__(self, *args, **kwargs) TransformationAgentsUtilities.__init__(self) #few parameters self.pluginLocation = self.am_getOption( 'PluginLocation', 'DIRAC.TransformationSystem.Agent.TransformationPlugin') self.transformationStatus = self.am_getOption( 'transformationStatus', ['Active', 'Completing', 'Flush']) self.maxFiles = self.am_getOption('MaxFiles', 5000) agentTSTypes = self.am_getOption('TransformationTypes', []) if agentTSTypes: self.transformationTypes = sortList(agentTSTypes) else: dataProc = Operations().getValue('Transformations/DataProcessing', ['MCSimulation', 'Merge']) dataManip = Operations().getValue( 'Transformations/DataManipulation', ['Replication', 'Removal']) self.transformationTypes = sortList(dataProc + dataManip) #clients self.transfClient = TransformationClient() #for the threading self.transQueue = Queue.Queue() self.transInQueue = [] #for caching using a pickle file self.workDirectory = self.am_getWorkDirectory() self.cacheFile = os.path.join(self.workDirectory, 'ReplicaCache.pkl') self.dateWriteCache = datetime.datetime.utcnow() # Validity of the cache self.replicaCache = None self.replicaCacheValidity = self.am_getOption('ReplicaCacheValidity', 2) self.writingCache = False self.noUnusedDelay = self.am_getOption('NoUnusedDelay', 6) self.unusedFiles = {} self.unusedTimeStamp = {} self.debug = False self.transInThread = {}
def checkTransformationIntegrity( self, transID ): """ This method contains the real work """ gLogger.info( "-" * 40 ) gLogger.info( "Checking the integrity of transformation %s" % transID ) gLogger.info( "-" * 40 ) res = self.getTransformationDirectories( transID ) if not res['OK']: return res directories = res['Value'] if not directories: return S_OK() ###################################################### # # This check performs Catalog->SE for possible output directories # res = self.fc.exists( directories ) if not res['OK']: gLogger.error( res['Message'] ) return res for directory, error in res['Value']['Failed']: gLogger.error( 'Failed to determine existance of directory', '%s %s' % ( directory, error ) ) if res['Value']['Failed']: return S_ERROR( "Failed to determine the existance of directories" ) directoryExists = res['Value']['Successful'] for directory in sortList( directoryExists.keys() ): if not directoryExists[directory]: continue iRes = self.integrityClient.catalogDirectoryToSE( directory ) if not iRes['OK']: gLogger.error( iRes['Message'] ) return iRes ###################################################### # # This check performs SE->Catalog for possible output directories # for storageElementName in sortList( self.activeStorages ): res = self.integrityClient.storageDirectoryToCatalog( directories, storageElementName ) if not res['OK']: gLogger.error( res['Message'] ) return res gLogger.info( "-" * 40 ) gLogger.info( "Completed integrity check for transformation %s" % transID ) return S_OK()
def retrieveRepositoryOutputDataLFNs(self, requestedStates = ['Done']): """Helper function Get the list of uploaded output data for a set of jobs in a repository @param requestedStates: List of states requested for filtering the list @type requestedStates: list of strings @return: list """ llist = [] if not self.jobRepo: gLogger.warn( "No repository is initialised" ) return S_OK() jobs = self.jobRepo.readRepository()['Value'] for jobID in sortList( jobs.keys() ): jobDict = jobs[jobID] if jobDict.has_key( 'State' ) and ( jobDict['State'] in requestedStates ): if ( jobDict.has_key( 'UserOutputData' ) and ( not int( jobDict['UserOutputData'] ) ) ) or \ ( not jobDict.has_key( 'UserOutputData' ) ): params = self.parameters(int(jobID)) if params['OK']: if params['Value'].has_key('UploadedOutputData'): lfn = params['Value']['UploadedOutputData'] llist.append(lfn) return llist
def resolveLFNZeroReplicas( self, problematicDict ): """ This takes the problematic dictionary returned by the integrity DB and resolves the LFNZeroReplicas prognosis """ lfn = problematicDict['LFN'] fileID = problematicDict['FileID'] res = self.rm.getCatalogReplicas( lfn, allStatus = True, singleFile = True ) if res['OK'] and res['Value']: gLogger.info( "LFNZeroReplicas file (%d) found to have replicas" % fileID ) else: gLogger.info( "LFNZeroReplicas file (%d) does not have replicas. Checking storage..." % fileID ) pfnsFound = False for storageElementName in sortList( gConfig.getValue( 'Resources/StorageElementGroups/Tier1_MC_M-DST', [] ) ): res = self.__getStoragePathExists( [lfn], storageElementName ) if res['Value'].has_key( lfn ): gLogger.info( "LFNZeroReplicas file (%d) found storage file at %s" % ( fileID, storageElementName ) ) pfn = res['Value'][lfn] self.__reportProblematicReplicas( [( lfn, pfn, storageElementName, 'PFNNotRegistered' )], storageElementName, 'PFNNotRegistered' ) pfnsFound = True if not pfnsFound: gLogger.info( "LFNZeroReplicas file (%d) did not have storage files. Removing..." % fileID ) res = self.rm.removeCatalogFile( lfn, singleFile = True ) if not res['OK']: gLogger.error( res['Message'] ) # Increment the number of retries for this file self.server.incrementProblematicRetry( fileID ) return res gLogger.info( "LFNZeroReplicas file (%d) removed from catalog" % fileID ) # If we get here the problem is solved so we can update the integrityDB return self.__updateCompletedFiles( 'LFNZeroReplicas', fileID )
def submitTransformationTasks(self, taskDict): submitted = 0 failed = 0 startTime = time.time() for taskID in sortList(taskDict.keys()): if not taskDict[taskID]['TaskObject']: taskDict[taskID]['Success'] = False failed += 1 continue res = self.submitTaskToExternal(taskDict[taskID]['TaskObject']) if res['OK']: taskDict[taskID]['ExternalID'] = res['Value'] taskDict[taskID]['Success'] = True submitted += 1 else: self.log.error("Failed to submit task to WMS", res['Message']) taskDict[taskID]['Success'] = False failed += 1 self.log.info( 'submitTransformationTasks: Submitted %d tasks to WMS in %.1f seconds' % (submitted, time.time() - startTime)) if failed: self.log.info( 'submitTransformationTasks: Failed to submit %d tasks to WMS.' % (failed)) return S_OK(taskDict)
def prepareTransformationTasks(self, transBody, taskDict, owner='', ownerGroup=''): requestType = 'transfer' requestOperation = 'replicateAndRegister' try: requestType, requestOperation = transBody.split(';') except: pass for taskID in sortList(taskDict.keys()): paramDict = taskDict[taskID] transID = paramDict['TransformationID'] oRequest = RequestContainer(init=False) subRequestIndex = oRequest.initiateSubRequest(requestType)['Value'] attributeDict = { 'Operation': requestOperation, 'TargetSE': paramDict['TargetSE'] } oRequest.setSubRequestAttributes(subRequestIndex, requestType, attributeDict) files = [] for lfn in paramDict['InputData'].split(';'): files.append({'LFN': lfn}) oRequest.setSubRequestFiles(subRequestIndex, requestType, files) requestName = str(transID).zfill(8) + '_' + str(taskID).zfill(8) oRequest.setRequestAttributes({'RequestName': requestName}) taskDict[taskID]['TaskObject'] = oRequest.toXML()['Value'] return S_OK(taskDict)
def getTransformationDirectories( self, transID ): ''' get the directories for the supplied transformation from the transformation system :param self: self reference :param int transID: transformation ID ''' directories = [] if 'TransformationDB' in self.directoryLocations: res = self.transClient.getTransformationParameters( transID, ['OutputDirectories'] ) if not res['OK']: self.log.error( "Failed to obtain transformation directories", res['Message'] ) return res transDirectories = res['Value'].splitlines() directories = self._addDirs( transID, transDirectories, directories ) if 'MetadataCatalog' in self.directoryLocations: res = self.metadataClient.findDirectoriesByMetadata( {self.transfidmeta:transID} ) if not res['OK']: self.log.error( "Failed to obtain metadata catalog directories", res['Message'] ) return res transDirectories = res['Value'] directories = self._addDirs( transID, transDirectories, directories ) if not directories: self.log.info( "No output directories found" ) directories = sortList( directories ) return S_OK( directories )
def getSubmittedFileStatus(self, fileDicts): taskFiles = {} for fileDict in fileDicts: transID = fileDict['TransformationID'] taskID = fileDict['TaskID'] taskName = str(transID).zfill(8) + '_' + str(taskID).zfill(8) if not taskFiles.has_key(taskName): taskFiles[taskName] = {} taskFiles[taskName][fileDict['LFN']] = fileDict['Status'] updateDict = {} for taskName in sortList(taskFiles.keys()): lfnDict = taskFiles[taskName] res = self.requestClient.getRequestFileStatus( taskName, lfnDict.keys(), 'RequestManagement/centralURL') if not res['OK']: self.log.warn( "getSubmittedFileStatus: Failed to get files status for request", res['Message']) continue for lfn, newStatus in res['Value'].items(): if newStatus == lfnDict[lfn]: pass elif newStatus == 'Done': updateDict[lfn] = 'Processed' elif newStatus == 'Failed': updateDict[lfn] = 'Problematic' return S_OK(updateDict)
def extendTransformation( self, transID, maxTasks ): gLogger.info( "Considering extension of transformation %d" % transID ) # Get the current count of tasks submitted for this transformation res = self.transClient.getTransformationTaskStats( transID ) if not res['OK']: if res['Message'] != 'No records found': gLogger.error( "Failed to get task statistics", "%s %s" % ( transID, res['Message'] ) ) return res else: statusDict = {} else: statusDict = res['Value'] gLogger.verbose( "Current task count for transformation %d" % transID ) for status in sortList( statusDict.keys() ): statusCount = statusDict[status] gLogger.verbose( "%s : %s" % ( status.ljust( 20 ), str( statusCount ).rjust( 8 ) ) ) # Determine the number of tasks to be created numberOfTasks = self.calculateTaskNumber( maxTasks, statusDict ) if not numberOfTasks: gLogger.info( "No tasks required for transformation %d" % transID ) return S_OK() # Extend the transformation by the determined number of tasks res = self.transClient.extendTransformation( transID, numberOfTasks ) if not res['OK']: gLogger.error( "Failed to extend transformation", "%s %s" % ( transID, res['Message'] ) ) return res gLogger.info( "Successfully extended transformation %d by %d tasks" % ( transID, numberOfTasks ) ) return S_OK()
def __checkPhysicalFiles( self, replicas, catalogMetadata, ses = [] ): """ This obtains the physical file metadata and checks the metadata against the catalog entries """ seLfns = {} for lfn, replicaDict in replicas.items(): for se, _url in replicaDict.items(): if ( ses ) and ( se not in ses ): continue seLfns.setdefault( se, [] ).append( lfn ) gLogger.info( '%s %s' % ( 'Storage Element'.ljust( 20 ), 'Replicas'.rjust( 20 ) ) ) for se in sortList( seLfns ): files = len( seLfns[se] ) gLogger.info( '%s %s' % ( se.ljust( 20 ), str( files ).rjust( 20 ) ) ) lfns = seLfns[se] sizeMismatch = [] res = self.__checkPhysicalFileMetadata( lfns, se ) if not res['OK']: gLogger.error( 'Failed to get physical file metadata.', res['Message'] ) return res for lfn, metadata in res['Value'].items(): if lfn in catalogMetadata: if ( metadata['Size'] != catalogMetadata[lfn]['Size'] ) and ( metadata['Size'] != 0 ): sizeMismatch.append( ( lfn, 'deprecatedUrl', se, 'CatalogPFNSizeMismatch' ) ) if sizeMismatch: self.__reportProblematicReplicas( sizeMismatch, se, 'CatalogPFNSizeMismatch' ) return S_OK()
def getStorageUsage(self): """ Fill the current Status of the SE Caches from the DB """ self.storageElementCache = {} res = self.storageDB.getSubmittedStagePins() if not res['OK']: gLogger.fatal( "StageRequest.getStorageUsage: Failed to obtain submitted requests from StorageManagementDB.", res['Message']) return res self.storageElementUsage = res['Value'] if self.storageElementUsage: gLogger.info( "StageRequest.getStorageUsage: Active stage/pin requests found at the following sites:" ) for storageElement in sortList(self.storageElementUsage.keys()): seDict = self.storageElementUsage[storageElement] # Convert to GB for printout seDict['TotalSize'] = seDict['TotalSize'] / (1000 * 1000 * 1000.0) gLogger.info( "StageRequest.getStorageUsage: %s: %s replicas with a size of %.3f GB." % (storageElement.ljust(15), str( seDict['Replicas']).rjust(6), seDict['TotalSize'])) if not self.storageElementUsage: gLogger.info( "StageRequest.getStorageUsage: No active stage/pin requests found." ) return S_OK()
def getTransformationDirectories( self, transID ): """ Get the directories for the supplied transformation from the transformation system """ directories = [] if 'TransformationDB' in self.directoryLocations: res = self.transClient.getTransformationParameters( transID, ['OutputDirectories'] ) if not res['OK']: gLogger.error( "Failed to obtain transformation directories", res['Message'] ) return res transDirectories = res['Value'].splitlines() directories = self.__addDirs( transID, transDirectories, directories ) if 'StorageUsage' in self.directoryLocations: res = self.storageUsageClient.getStorageDirectories( '', '', transID, [] ) if not res['OK']: gLogger.error( "Failed to obtain storage usage directories", res['Message'] ) return res transDirectories = res['Value'] directories = self.__addDirs( transID, transDirectories, directories ) if 'MetadataCatalog' in self.directoryLocations: res = self.metadataClient.findDirectoriesByMetadata( {self.transfidmeta:transID} ) if not res['OK']: gLogger.error( "Failed to obtain metadata catalog directories", res['Message'] ) return res transDirectories = res['Value'] directories = self.__addDirs( transID, transDirectories, directories ) if not directories: gLogger.info( "No output directories found" ) directories = sortList( directories ) return S_OK( directories )
def __fileRetry(self, prodid, mode): id = int(prodid) RPC = getRPCClient('Transformation/TransformationManager') if mode == "proc": res = RPC.getTransformationFilesCount(prodid, "ErrorCount", {'Status': 'Processed'}) elif mode == "not": res = RPC.getTransformationFilesCount( prodid, "ErrorCount", {'Status': ['Unused', 'Assigned', 'Failed']}) elif mode == "all": res = RPC.getTransformationFilesCount(prodid, "ErrorCount") else: return {"success": "false", "error": res["Message"]} if not res['OK']: c.result = {"success": "false", "error": res["Message"]} else: resList = [] total = res['Value'].pop('Total') if total == 0: c.result = {"success": "false", "error": "No files found"} else: for status in sortList(res['Value'].keys()): count = res['Value'][status] percent = "%.1f" % ((count * 100.0) / total) resList.append((status, str(count), percent)) resList.append(('Total', total, '-')) c.result = {"success": "true", "result": resList} gLogger.info("#######", res) return c.result
def __fileRetry(self,prodid,mode): id = int(prodid) RPC = getRPCClient('Transformation/TransformationManager') if mode == "proc": res = RPC.getTransformationFilesCount(prodid,"ErrorCount",{'Status':'Processed'}) elif mode == "not": res = RPC.getTransformationFilesCount(prodid,"ErrorCount",{'Status':['Unused','Assigned','Failed']}) elif mode == "all": res = RPC.getTransformationFilesCount(prodid,"ErrorCount") else: return {"success":"false","error":res["Message"]} if not res['OK']: c.result = {"success":"false","error":res["Message"]} else: resList = [] total = res['Value'].pop('Total') if total == 0: c.result = {"success":"false","error":"No files found"} else: for status in sortList(res['Value'].keys()): count = res['Value'][status] percent = "%.1f" % ((count*100.0)/total) resList.append((status,str(count),percent)) resList.append(('Total',total,'-')) c.result = {"success":"true","result":resList} gLogger.info("#######",res) return c.result
def retrieveRepositoryOutputDataLFNs(self, requestedStates=['Done']): """Helper function Get the list of uploaded output data for a set of jobs in a repository @param requestedStates: List of states requested for filtering the list @type requestedStates: list of strings @return: list """ llist = [] if not self.jobRepo: gLogger.warn("No repository is initialized") return S_OK() jobs = self.jobRepo.readRepository()['Value'] for jobID in sortList(jobs.keys()): jobDict = jobs[jobID] if jobDict.has_key('State') and (jobDict['State'] in requestedStates): if ( jobDict.has_key( 'UserOutputData' ) and ( not int( jobDict['UserOutputData'] ) ) ) or \ ( not jobDict.has_key( 'UserOutputData' ) ): params = self.parameters(int(jobID)) if params['OK']: if params['Value'].has_key('UploadedOutputData'): lfn = params['Value']['UploadedOutputData'] llist.append(lfn) return llist
def getTransformationDirectories(self, transID): """ Get the directories for the supplied transformation from the transformation system """ directories = [] if 'TransformationDB' in self.directoryLocations: res = self.transClient.getTransformationParameters( transID, ['OutputDirectories']) if not res['OK']: gLogger.error("Failed to obtain transformation directories", res['Message']) return res transDirectories = res['Value'].splitlines() directories = self._addDirs(transID, transDirectories, directories) if 'MetadataCatalog' in self.directoryLocations: res = self.fileCatalogClient.findDirectoriesByMetadata( {self.transfidmeta: transID}) if not res['OK']: gLogger.error("Failed to obtain metadata catalog directories", res['Message']) return res transDirectories = res['Value'] directories = self._addDirs(transID, transDirectories, directories) if not directories: gLogger.info("No output directories found") directories = sortList(directories) return S_OK(directories)
def __fileRetry(self, prodid, mode): id = int(prodid) RPC = getRPCClient("Transformation/TransformationManager") if mode == "proc": res = RPC.getTransformationFilesCount(prodid, "ErrorCount", {"Status": "Processed"}) elif mode == "not": res = RPC.getTransformationFilesCount(prodid, "ErrorCount", {"Status": ["Unused", "Assigned", "Failed"]}) elif mode == "all": res = RPC.getTransformationFilesCount(prodid, "ErrorCount") else: return {"success": "false", "error": res["Message"]} if not res["OK"]: c.result = {"success": "false", "error": res["Message"]} else: resList = [] total = res["Value"].pop("Total") if total == 0: c.result = {"success": "false", "error": "No files found"} else: for status in sortList(res["Value"].keys()): count = res["Value"][status] percent = "%.1f" % ((count * 100.0) / total) resList.append((status, str(count), percent)) resList.append(("Total", total, "-")) c.result = {"success": "true", "result": resList} gLogger.info("#######", res) return c.result
def __getFullOutput( self ): comm = ['glite-transfer-status', '-s', self.ftsServer, '-l', self.ftsGUID] res = executeGridCommand( '', comm, self.gridEnv ) if not res['OK']: return res returnCode, output, errStr = res['Value'] # Returns a non zero status if error if not returnCode == 0: return S_ERROR( errStr ) statusExp = re.compile( "^(\S+)" ) self.requestStatus = re.search( statusExp, output ).group( 1 ) output = output.replace( "%s\n" % self.requestStatus, "", 1 ) toRemove = ["'", "<", ">"] for char in toRemove: output = output.replace( char, '' ) regExp = re.compile( "[ ]+Source:[ ]+(\S+)\n[ ]+Destination:[ ]+(\S+)\n[ ]+State:[ ]+(\S+)\n[ ]+Retries:[ ]+(\d+)\n[ ]+Reason:[ ]+([\S ]+).+?[ ]+Duration:[ ]+(\d+)", re.S ) fileInfo = re.findall( regExp, output ) for source, target, status, retries, reason, duration in fileInfo: lfn = '' for candidate in sortList( self.fileDict.keys() ): if re.search( candidate, source ): lfn = candidate if not lfn: continue self.__setFileParameter( lfn, 'Source', source ) self.__setFileParameter( lfn, 'Target', target ) self.__setFileParameter( lfn, 'Status', status ) if reason == '(null)': reason = '' self.__setFileParameter( lfn, 'Reason', reason.replace( "\n", " " ) ) self.__setFileParameter( lfn, 'Duration', int( duration ) ) return S_OK()
def getSubmittedFileStatus( self, fileDicts ): taskFiles = {} for fileDict in fileDicts: transID = fileDict['TransformationID'] taskID = fileDict['TaskID'] taskName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 ) if not taskFiles.has_key( taskName ): taskFiles[taskName] = {} taskFiles[taskName][fileDict['LFN']] = fileDict['Status'] updateDict = {} for taskName in sortList( taskFiles.keys() ): lfnDict = taskFiles[taskName] res = self.requestClient.getRequestFileStatus( taskName, lfnDict.keys() ) if not res['OK']: self.log.warn( "getSubmittedFileStatus: Failed to get files status for request", res['Message'] ) continue for lfn, newStatus in res['Value'].items(): if newStatus == lfnDict[lfn]: pass elif newStatus == 'Done': updateDict[lfn] = 'Processed' elif newStatus == 'Failed': updateDict[lfn] = 'Problematic' return S_OK( updateDict )
def _getConfigStorageProtocols(self, storageName): """ Protocol specific information is present as sections in the Storage configuration """ result = getSiteForResource('Storage', storageName) if not result['OK']: return result site = result['Value'] result = self.resourcesHelper.getEligibleNodes('AccessProtocol', { 'Site': site, 'Resource': storageName }) if not result['OK']: return result nodesDict = result['Value'] protocols = [] for site in nodesDict: for se in nodesDict[site]: protocols.extend(nodesDict[site][se]) sortedProtocols = sortList(protocols) protocolDetails = [] for protocol in sortedProtocols: result = self._getConfigStorageProtocolDetails( storageName, protocol) if not result['OK']: return result protocolDetails.append(result['Value']) self.protocols = self.localProtocols + self.remoteProtocols return S_OK(protocolDetails)
def initialize(self): """Sets defaults """ self.transClient = TransformationClient() # This sets the Default Proxy to used as that defined under # /Operations/Shifter/DataManager # the shifterProxy option in the Configuration can be used to change this default. self.am_setOption('shifterProxy', 'DataManager') self.transformationTypes = sortList( self.am_getOption('TransformationTypes', ['MCSimulation', 'Simulation'])) gLogger.info("Will consider the following transformation types: %s" % str(self.transformationTypes)) self.maxIterationTasks = self.am_getOption('TasksPerIteration', 50) gLogger.info("Will create a maximum of %s tasks per iteration" % self.maxIterationTasks) self.maxFailRate = self.am_getOption('MaxFailureRate', 30) gLogger.info( "Will not submit tasks for transformations with failure rate greater than %s%s" % (self.maxFailRate, '%')) self.maxWaitingJobs = self.am_getOption('MaxWaitingJobs', 1000) gLogger.info( "Will not submit tasks for transformations with more than %d waiting jobs" % self.maxWaitingJobs) return S_OK()
def __checkPhysicalFiles(self, replicas, catalogMetadata, ses=[]): """ This obtains the physical file metadata and checks the metadata against the catalog entries """ seLfns = {} for lfn, replicaDict in replicas.items(): for se, _url in replicaDict.items(): if (ses) and (se not in ses): continue seLfns.setdefault(se, []).append(lfn) gLogger.info('%s %s' % ('Storage Element'.ljust(20), 'Replicas'.rjust(20))) for se in sortList(seLfns): files = len(seLfns[se]) gLogger.info('%s %s' % (se.ljust(20), str(files).rjust(20))) lfns = seLfns[se] sizeMismatch = [] res = self.__checkPhysicalFileMetadata(lfns, se) if not res['OK']: gLogger.error('Failed to get physical file metadata.', res['Message']) return res for lfn, metadata in res['Value'].items(): if lfn in catalogMetadata: if (metadata['Size'] != catalogMetadata[lfn]['Size']) and ( metadata['Size'] != 0): sizeMismatch.append((lfn, 'deprecatedUrl', se, 'CatalogPFNSizeMismatch')) if sizeMismatch: self.__reportProblematicReplicas(sizeMismatch, se, 'CatalogPFNSizeMismatch') return S_OK()
def resolveTransforamtionProblematics( transID ): gLogger.notice("Obtaining problematic files for transformation %d" % transID) res = integrityClient.getTransformationProblematics(transID) if not res['OK']: gLogger.error("Failed to get transformation problematic files", res['Message']) return S_ERROR() problematicFiles = res['Value'] if not problematicFiles: gLogger.notice("No problematic files found for transformation") return S_OK() for lfn in sortList(problematicFiles.keys()): prognosis = problematicFiles[lfn]['Prognosis'] problematicDict = problematicFiles[lfn] gLogger.notice("Prognosis is %s" % prognosis ) if not hasattr( integrityClient, methodToCall ): gLogger.notice( "DataIntegrityClient hasn't got '%s' member" % methodToCall ) continue fcn = getattr( integrityClient, methodToCall ) if not callable( fcn ): gLogger.notice( "DataIntegrityClient member '%s' isn't a method" % methodToCall ) continue ## results not checked??? Where is The Food? res = fcn( problematicDict ) gLogger.notice("Problematic files resolved for transformation %d" % transID) return S_OK()
def updateTaskStatus(self): gLogger.info("updateTaskStatus: Updating the Status of tasks") # Get the transformations to be updated status = self.am_getOption('UpdateTasksStatus',['Active','Completing','Stopped']) res = self._selectTransformations(transType=self.transType,status=status,agentType=[]) if not res['OK']: return res for transformation in res['Value']: transID = transformation['TransformationID'] # Get the tasks which are in a UPDATE state updateStatus = self.am_getOption('TaskUpdateStatus',['Checking','Deleted','Killed','Staging','Stalled','Matched','Rescheduled','Completed','Submitted','Received','Waiting','Running']) condDict = {"TransformationID":transID,"ExternalStatus":updateStatus} timeStamp = str(datetime.datetime.utcnow() - datetime.timedelta(minutes=10)) res = self.transClient.getTransformationTasks(condDict=condDict,older=timeStamp, timeStamp='LastUpdateTime') if not res['OK']: gLogger.error("updateTaskStatus: Failed to get tasks to update for transformation", "%s %s" % (transID,res['Message'])) continue if not res['Value']: gLogger.verbose("updateTaskStatus: No tasks found to update for transformation %s" % transID) continue res = self.getSubmittedTaskStatus(res['Value']) if not res['OK']: gLogger.error("updateTaskStatus: Failed to get updated task statuses for transformation", "%s %s" % (transID,res['Message'])) continue statusDict = res['Value'] for status in sortList(statusDict.keys()): taskIDs = statusDict[status] gLogger.info("updateTaskStatus: Updating %d task(s) from transformation %d to %s" % (len(taskIDs),transID,status)) res = self.transClient.setTaskStatus(transID,taskIDs,status) if not res['OK']: gLogger.error("updateTaskStatus: Failed to update task status for transformation", "%s %s" % (transID,res['Message'])) gLogger.info("updateTaskStatus: Transformation task status update complete") return S_OK()
def execute(self): # Get the current submitted stage space and the amount of pinned space for each storage element res = self.storageDB.getSubmittedStagePins() if not res['OK']: gLogger.fatal( "StageRequest.submitStageRequests: Failed to obtain submitted requests from StorageManagementDB.", res['Message']) return res self.storageElementUsage = res['Value'] if self.storageElementUsage: gLogger.info( "StageRequest.execute: Active stage/pin requests found at the following sites:" ) for storageElement in sortList(self.storageElementUsage.keys()): seDict = self.storageElementUsage[storageElement] # Daniela: fishy? Changed it to GB and division by 1024 instead of 1000 gLogger.info( "StageRequest.execute: %s: %s replicas with a size of %.3f GB." % (storageElement.ljust(15), str( seDict['Replicas']).rjust(6), seDict['TotalSize'] / (1024 * 1024 * 1024.0))) if not self.storageElementUsage: gLogger.info( "StageRequest.execute: No active stage/pin requests found.") res = self.submitStageRequests() return res
def extendTransformation(self, transID, maxTasks): gLogger.info("Considering extension of transformation %d" % transID) # Get the current count of tasks submitted for this transformation res = self.transClient.getTransformationTaskStats(transID) if not res['OK']: if res['Message'] != 'No records found': gLogger.error("Failed to get task statistics", "%s %s" % (transID, res['Message'])) return res else: statusDict = {} else: statusDict = res['Value'] gLogger.verbose("Current task count for transformation %d" % transID) for status in sortList(statusDict.keys()): statusCount = statusDict[status] gLogger.verbose("%s : %s" % (status.ljust(20), str(statusCount).rjust(8))) # Determine the number of tasks to be created numberOfTasks = self._calculateTaskNumber(maxTasks, statusDict) if not numberOfTasks: gLogger.info("No tasks required for transformation %d" % transID) return S_OK() # Extend the transformation by the determined number of tasks res = self.transClient.extendTransformation(transID, numberOfTasks) if not res['OK']: gLogger.error("Failed to extend transformation", "%s %s" % (transID, res['Message'])) return res gLogger.info("Successfully extended transformation %d by %d tasks" % (transID, numberOfTasks)) return S_OK()
def _getFileGroups( self, fileReplicas ): fileGroups = {} for lfn, replicas in fileReplicas.items(): replicaSEs = str.join( ',', sortList( uniqueElements( replicas.keys() ) ) ) if not fileGroups.has_key( replicaSEs ): fileGroups[replicaSEs] = [] fileGroups[replicaSEs].append( lfn ) return fileGroups
def do_removeFile( self, args ): """Remove file from transformation DB usage: removeFile <lfn> [lfn] """ argss = string.split( args ) if not len( argss ) > 0: print "no files supplied" return res = self.server.removeFile( argss ) if not res['OK']: print "failed to remove any files: %s" % res['Message'] return for lfn in sortList( res['Value']['Failed'].keys() ): error = res['Value']['Failed'][lfn] print "failed to remove %s: %s" % ( lfn, error ) for lfn in sortList( res['Value']['Successful'].keys() ): print "removed %s" % lfn
def dumpSummary( self, printOutput = False ): outStr = '' for status in sortList( self.statusSummary.keys() ): if self.statusSummary[status]: outStr = '%s\t%s : %s\n' % ( outStr, status.ljust( 10 ), str( self.statusSummary[status] ).ljust( 10 ) ) outStr = outStr.rstrip( '\n' ) if printOutput: print outStr return S_OK( outStr )