def __removeStorageDirectory(self, directory, storageElement): """ wipe out all contents from :directory: at :storageElement: :param self: self reference :param str directory: path :param str storageElement: SE name """ self.log.info('Removing the contents of %s at %s' % (directory, storageElement)) se = StorageElement(storageElement) res = returnSingleResult(se.exists(directory)) if not res['OK']: self.log.error("Failed to obtain existance of directory", res['Message']) return res exists = res['Value'] if not exists: self.log.info("The directory %s does not exist at %s " % (directory, storageElement)) return S_OK() res = returnSingleResult(se.removeDirectory(directory, recursive=True)) if not res['OK']: self.log.error("Failed to remove storage directory", res['Message']) return res self.log.info( "Successfully removed %d files from %s at %s" % (res['Value']['FilesRemoved'], directory, storageElement)) return S_OK()
def resolveLFNZeroReplicas( self, problematicDict ): """ This takes the problematic dictionary returned by the integrity DB and resolves the LFNZeroReplicas prognosis """ lfn = problematicDict['LFN'] fileID = problematicDict['FileID'] res = returnSingleResult( self.fc.getReplicas( lfn, allStatus = True ) ) if res['OK'] and res['Value']: gLogger.info( "LFNZeroReplicas file (%d) found to have replicas" % fileID ) else: gLogger.info( "LFNZeroReplicas file (%d) does not have replicas. Checking storage..." % fileID ) pfnsFound = False for storageElementName in sortList( gConfig.getValue( 'Resources/StorageElementGroups/Tier1_MC_M-DST', [] ) ): res = self.__getStoragePathExists( [lfn], storageElementName ) if res['Value'].has_key( lfn ): gLogger.info( "LFNZeroReplicas file (%d) found storage file at %s" % ( fileID, storageElementName ) ) pfn = res['Value'][lfn] self.__reportProblematicReplicas( [( lfn, pfn, storageElementName, 'PFNNotRegistered' )], storageElementName, 'PFNNotRegistered' ) pfnsFound = True if not pfnsFound: gLogger.info( "LFNZeroReplicas file (%d) did not have storage files. Removing..." % fileID ) res = returnSingleResult( self.fc.removeFile( lfn ) ) if not res['OK']: gLogger.error( res['Message'] ) # Increment the number of retries for this file self.server.incrementProblematicRetry( fileID ) return res gLogger.info( "LFNZeroReplicas file (%d) removed from catalog" % fileID ) # If we get here the problem is solved so we can update the integrityDB return self.__updateCompletedFiles( 'LFNZeroReplicas', fileID )
def resolveBKReplicaNo(self, problematicDict): """ This takes the problematic dictionary returned by the integrity DB and resolved the BKReplicaNo prognosis """ lfn = problematicDict['LFN'] fileID = problematicDict['FileID'] res = returnSingleResult(self.fc.exists(lfn)) if not res['OK']: return self.__returnProblematicError(fileID, res) # If the file exists in the catalog if not res['Value']: return self.__updateCompletedFiles('BKReplicaNo', fileID) gLogger.info("BKReplicaNo file (%d) found to exist in the catalog" % fileID) # and has available replicas res = returnSingleResult(self.fc.getCatalogReplicas(lfn)) if not res['OK']: return self.__returnProblematicError(fileID, res) if not res['Value']: gLogger.info("BKReplicaNo file (%d) found to have no replicas" % fileID) return self.changeProblematicPrognosis(fileID, 'LFNZeroReplicas') gLogger.info("BKReplicaNo file (%d) found to have replicas" % fileID) res = returnSingleResult( FileCatalog(catalogs=['BookkeepingDB']).addFile(lfn)) if not res['OK']: return self.__returnProblematicError(fileID, res) return self.__updateCompletedFiles('BKReplicaNo', fileID)
def test_getFile(self): print('\n\n#########################################################' '################\n\n\t\t\tGet file test\n') destinationFilePath = '%s/testFile.%s' % (self.destDirectory, time.time()) # pfnForLfnRes = returnSingleResult( self.storageElement.getURL( destinationFilePath ) ) # destinationPfn = pfnForLfnRes['Value'] fileDict = {destinationFilePath: self.localSourceFile} putFileRes = returnSingleResult(self.storageElement.putFile(fileDict)) # Now get a local copy of the file getFileRes = returnSingleResult( self.storageElement.getFile(destinationFilePath)) # Now remove the destination file removeFileRes = returnSingleResult( self.storageElement.removeFile(destinationFilePath)) # Clean up the local mess os.remove(os.path.basename(destinationFilePath)) # Check that the put was done correctly self.assertTrue(putFileRes['OK']) self.assertTrue(putFileRes['Value']) self.assertEqual(putFileRes['Value'], self.localFileSize) # Check that we got the file correctly self.assertTrue(getFileRes['OK']) self.assertEqual(getFileRes['Value'], self.localFileSize) # Check that the removal was done correctly self.assertTrue(removeFileRes['OK']) self.assertTrue(removeFileRes['Value'])
def test_getURL( self ): print '\n\n#########################################################################\n\n\t\tGet access url test\n' destinationFilePath = '%s/testFile.%s' % ( self.destDirectory, time.time() ) # pfnForLfnRes = returnSingleResult( self.storageElement.getURL( destinationFilePath ) ) #destinationPfn = pfnForLfnRes['Value'] fileDict = {destinationFilePath:self.localSourceFile} putFileRes = returnSingleResult( self.storageElement.putFile( fileDict ) ) # Get a transfer url for the file getTurlRes = self.storageElement.getURL( destinationFilePath, protocol = 'dips' ) # Remove the destination file removeFileRes = returnSingleResult( self.storageElement.removeFile( destinationFilePath ) ) # Get missing turl res getMissingTurlRes = self.storageElement.getURL( destinationFilePath, protocol = 'dips' ) # Check that the put was done correctly self.assert_( putFileRes['OK'] ) self.assert_( putFileRes['Value'] ) self.assertEqual( putFileRes['Value'], self.localFileSize ) # Check that we can get the tURL properly self.assert_( getTurlRes['OK'] ) self.assert_( getTurlRes['Value'] ) self.assert_( type( getTurlRes['Value'] ) == types.DictType ) self.assert_( type( getTurlRes['Value']['Successful'][destinationFilePath] ) in types.StringTypes ) # Check that the removal was done correctly self.assert_( removeFileRes['OK'] ) self.assert_( removeFileRes['Value'] )
def test_getURL(self): print('\n\n#########################################################' '################\n\n\t\tGet access url test\n') destinationFilePath = '%s/testFile.%s' % (self.destDirectory, time.time()) # pfnForLfnRes = returnSingleResult( self.storageElement.getURL( destinationFilePath ) ) # destinationPfn = pfnForLfnRes['Value'] fileDict = {destinationFilePath: self.localSourceFile} putFileRes = returnSingleResult(self.storageElement.putFile(fileDict)) # Get a transfer url for the file getTurlRes = self.storageElement.getURL(destinationFilePath, protocol='dips') # Remove the destination file removeFileRes = returnSingleResult( self.storageElement.removeFile(destinationFilePath)) # Get missing turl res getMissingTurlRes = self.storageElement.getURL(destinationFilePath, protocol='dips') # Check that the put was done correctly self.assertTrue(putFileRes['OK']) self.assertTrue(putFileRes['Value']) self.assertEqual(putFileRes['Value'], self.localFileSize) # Check that we can get the tURL properly self.assertTrue(getTurlRes['OK']) self.assertTrue(getTurlRes['Value']) self.assertTrue(isinstance(getTurlRes['Value'], dict)) self.assertTrue( type(getTurlRes['Value']['Successful'][destinationFilePath]) in six.string_types) # Check that the removal was done correctly self.assertTrue(removeFileRes['OK']) self.assertTrue(removeFileRes['Value'])
def __removeStorageDirectory( self, directory, storageElement ): """ wipe out all contents from :directory: at :storageElement: :param self: self reference :param str directory: path :param str storageElement: SE name """ self.log.info( 'Removing the contents of %s at %s' % ( directory, storageElement ) ) se = StorageElement( storageElement ) res = returnSingleResult( se.exists( directory ) ) if not res['OK']: self.log.error( "Failed to obtain existance of directory", res['Message'] ) return res exists = res['Value'] if not exists: self.log.info( "The directory %s does not exist at %s " % ( directory, storageElement ) ) return S_OK() res = returnSingleResult( se.removeDirectory( directory, recursive = True ) ) if not res['OK']: self.log.error( "Failed to remove storage directory", res['Message'] ) return res self.log.info( "Successfully removed %d files from %s at %s" % ( res['Value']['FilesRemoved'], directory, storageElement ) ) return S_OK()
def test_getFileSize( self ): print '\n\n#########################################################################\n\n\t\t\tGet file size test\n' destinationFilePath = '%s/testFile.%s' % ( self.destDirectory, time.time() ) # pfnForLfnRes = returnSingleResult( self.storageElement.getURL( destinationFilePath ) ) #destinationPfn = pfnForLfnRes['Value'] fileDict = {destinationFilePath:self.localSourceFile} putFileRes = returnSingleResult( self.storageElement.putFile( fileDict ) ) # Get the file metadata getFileSizeRes = returnSingleResult( self.storageElement.getFileSize( destinationFilePath ) ) # Now remove the destination file removeFileRes = returnSingleResult( self.storageElement.removeFile( destinationFilePath ) ) # Get metadata for a removed file getMissingFileSizeRes = returnSingleResult( self.storageElement.getFileSize( destinationFilePath ) ) # Check directories are handled properly destinationDir = os.path.dirname( destinationFilePath ) directorySizeRes = returnSingleResult( self.storageElement.getFileSize( destinationDir ) ) # Check that the put was done correctly self.assert_( putFileRes['OK'] ) self.assert_( putFileRes['Value'] ) self.assertEqual( putFileRes['Value'], self.localFileSize ) # Check that the metadata was done correctly self.assert_( getFileSizeRes['OK'] ) self.assertEqual( getFileSizeRes['Value'], self.localFileSize ) # Check that the removal was done correctly self.assert_( removeFileRes['OK'] ) self.assert_( removeFileRes['Value'] ) # Check the get metadata for non existant file self.assertFalse( getMissingFileSizeRes['OK'] ) expectedError = "File does not exist" self.assert_( expectedError in getMissingFileSizeRes['Message'] ) # Check that metadata operation with a directory self.assertFalse( directorySizeRes['OK'] ) expectedError = "Supplied path is not a file" self.assert_( expectedError in directorySizeRes['Message'] )
def test_exists( self ): print '\n\n#########################################################################\n\n\t\t\tExists test\n' destinationFilePath = '%s/testFile.%s' % ( self.destDirectory, time.time() ) # pfnForLfnRes = self.storageElement.getURL( destinationFilePath ) #destinationPfn = pfnForLfnRes['Value']['Successful'].values()[0] fileDict = {destinationFilePath:self.localSourceFile} putFileRes = returnSingleResult( self.storageElement.putFile( fileDict ) ) # File exists existsRes = returnSingleResult( self.storageElement.exists( destinationFilePath ) ) # Now remove the destination file removeFileRes = returnSingleResult( self.storageElement.removeFile( destinationFilePath ) ) # Check removed file missingExistsRes = returnSingleResult( self.storageElement.exists( destinationFilePath ) ) # Check directories are handled properly destinationDir = os.path.dirname( destinationFilePath ) directoryExistsRes = returnSingleResult( self.storageElement.exists( destinationDir ) ) # Check that the put was done correctly self.assert_( putFileRes['OK'] ) self.assert_( putFileRes['Value'] ) self.assertEqual( putFileRes['Value'], self.localFileSize ) # Check that we checked the file correctly self.assert_( existsRes['OK'] ) self.assert_( existsRes['Value'] ) # Check that the removal was done correctly self.assert_( removeFileRes['OK'] ) self.assert_( removeFileRes['Value'] ) # Check the exists for non existant file self.assert_( missingExistsRes['OK'] ) self.assertFalse( missingExistsRes['Value'] ) # Check that directories exist self.assert_( directoryExistsRes['OK'] ) self.assert_( directoryExistsRes['Value'] )
def __getFilePermission(self, path, credDict, noExistStrategy=None): """Checks POSIX permission for a file using the VOMS roles. That is, if the owner group of the file shares the same vomsRole as the requesting user, we check the permission as if the request was done with the real owner group. :param path : file path (string) :param credDict : credential of the user :param noExistStrategy : If the directory does not exist, we can * True : allow the access * False : forbid the access * None : return the error as is :returns S_OK structure with a dictionary ( Read/Write/Execute : True/False) """ if not path: return S_ERROR("Empty path") # We check what is the group stored in the DB for the given path res = returnSingleResult(self.db.fileManager.getFileMetadata([path])) if not res["OK"]: # If the error is not due to the directory not existing, we return if not self.__isNotExistError(res["Message"]): return res # From now on, we know that the error is due to the file not existing # If we have no strategy regarding non existing files, then just return the error if noExistStrategy is None: return res # Finally, follow the strategy return S_OK( dict.fromkeys(["Read", "Write", "Execute"], noExistStrategy)) # =========================================================================== # # This does not seem necessary since we add the OwnerGroup in the query behind the scene # origGrp = 'unknown' # res = self.db.ugManager.getGroupName( res['Value']['GID'] ) # if res['OK']: # origGrp = res['Value'] # =========================================================================== origGrp = res["Value"].get("OwnerGroup", "unknown") # If the two group share the same voms role, we do the query like if we were # the group stored in the DB # useless in eiscat # if self.__shareVomsRole( credDict.get( 'group', 'anon' ), origGrp ): # credDict = { 'username' : credDict.get( 'username', 'anon' ), 'group' : origGrp} # eiscat rules, credDict = self.__eiscatFilesMatching(path, credDict, origGrp, isDir=False) return returnSingleResult( self.db.fileManager.getPathPermissions([path], credDict))
def resolvePFNMissing(self, problematicDict): """ This takes the problematic dictionary returned by the integrity DB and resolved the PFNMissing prognosis """ pfn = problematicDict['PFN'] se = problematicDict['SE'] lfn = problematicDict['LFN'] fileID = problematicDict['FileID'] res = returnSingleResult(self.fc.exists(lfn)) if not res['OK']: return self.__returnProblematicError(fileID, res) if not res['Value']: gLogger.info("PFNMissing file (%d) no longer exists in catalog" % fileID) return self.__updateCompletedFiles('PFNMissing', fileID) res = returnSingleResult(StorageElement(se).exists(pfn)) if not res['OK']: return self.__returnProblematicError(fileID, res) if res['Value']: gLogger.info("PFNMissing replica (%d) is no longer missing" % fileID) return self.__updateReplicaToChecked(problematicDict) gLogger.info("PFNMissing replica (%d) does not exist" % fileID) res = returnSingleResult(self.fc.getReplicas(lfn, allStatus=True)) if not res['OK']: return self.__returnProblematicError(fileID, res) replicas = res['Value'] seSite = se.split('_')[0].split('-')[0] found = False print replicas for replicaSE in replicas.keys(): if re.search(seSite, replicaSE): found = True problematicDict['SE'] = replicaSE se = replicaSE if not found: gLogger.info( "PFNMissing replica (%d) is no longer registered at SE. Resolved." % fileID) return self.__updateCompletedFiles('PFNMissing', fileID) gLogger.info( "PFNMissing replica (%d) does not exist. Removing from catalog..." % fileID) res = returnSingleResult(self.fc.removeReplica({lfn: problematicDict})) if not res['OK']: return self.__returnProblematicError(fileID, res) if len(replicas) == 1: gLogger.info( "PFNMissing replica (%d) had a single replica. Updating prognosis" % fileID) return self.changeProblematicPrognosis(fileID, 'LFNZeroReplicas') res = self.dm.replicateAndRegister(problematicDict['LFN'], se) if not res['OK']: return self.__returnProblematicError(fileID, res) # If we get here the problem is solved so we can update the integrityDB return self.__updateCompletedFiles('PFNMissing', fileID)
def checkArchive(self, archiveLFN): """Check that archiveLFN does not exist yet.""" LOG.notice('Using Tarball: %s' % archiveLFN) exists = returnSingleResult(self.fcClient.isFile(archiveLFN)) LOG.debug('Checking for Tarball existence %r' % exists) if exists['OK'] and exists['Value']: raise RuntimeError('Tarball %r already exists' % archiveLFN) LOG.debug('Checking permissions for %r' % archiveLFN) hasAccess = returnSingleResult(self.fcClient.hasAccess(archiveLFN, 'addFile')) if not archiveLFN or not hasAccess['OK'] or not hasAccess['Value']: LOG.error('Error checking tarball location: %r' % hasAccess) raise ValueError('%s is not a valid path, parameter "Name" must be correct' % archiveLFN)
def __getFilePermission( self, path, credDict, noExistStrategy = None ): """ Checks POSIX permission for a file using the VOMS roles. That is, if the owner group of the file shares the same vomsRole as the requesting user, we check the permission as if the request was done with the real owner group. :param path : file path (string) :param credDict : credential of the user :param noExistStrategy : If the directory does not exist, we can * True : allow the access * False : forbid the access * None : return the error as is :returns S_OK structure with a dictionary ( Read/Write/Execute : True/False) """ if not path: return S_ERROR( 'Empty path' ) # We check what is the group stored in the DB for the given path res = returnSingleResult( self.db.fileManager.getFileMetadata( [path] ) ) if not res['OK']: # If the error is not due to the directory not existing, we return if not self.__isNotExistError( res['Message'] ): return res # From now on, we know that the error is due to the file not existing # If we have no strategy regarding non existing files, then just return the error if noExistStrategy is None: return res # Finally, follow the strategy return S_OK( dict.fromkeys( ['Read', 'Write', 'Execute'], noExistStrategy ) ) #=========================================================================== # # This does not seem necessary since we add the OwnerGroup in the query behind the scene # origGrp = 'unknown' # res = self.db.ugManager.getGroupName( res['Value']['GID'] ) # if res['OK']: # origGrp = res['Value'] #=========================================================================== origGrp = res['Value'].get( 'OwnerGroup', 'unknown' ) # If the two group share the same voms role, we do the query like if we were # the group stored in the DB if self.__shareVomsRole( credDict.get( 'group', 'anon' ), origGrp ): credDict = { 'username' : credDict.get( 'username', 'anon' ), 'group' : origGrp} return returnSingleResult( self.db.fileManager.getPathPermissions( [path], credDict ) )
def resolvePFNMissing( self, problematicDict ): """ This takes the problematic dictionary returned by the integrity DB and resolved the PFNMissing prognosis """ pfn = problematicDict['PFN'] se = problematicDict['SE'] lfn = problematicDict['LFN'] fileID = problematicDict['FileID'] res = returnSingleResult( self.fc.exists( lfn ) ) if not res['OK']: return self.__returnProblematicError( fileID, res ) if not res['Value']: gLogger.info( "PFNMissing file (%d) no longer exists in catalog" % fileID ) return self.__updateCompletedFiles( 'PFNMissing', fileID ) res = returnSingleResult( StorageElement( se ).exists( pfn ) ) if not res['OK']: return self.__returnProblematicError( fileID, res ) if res['Value']: gLogger.info( "PFNMissing replica (%d) is no longer missing" % fileID ) return self.__updateReplicaToChecked( problematicDict ) gLogger.info( "PFNMissing replica (%d) does not exist" % fileID ) res = returnSingleResult( self.fc.getReplicas( lfn, allStatus = True ) ) if not res['OK']: return self.__returnProblematicError( fileID, res ) replicas = res['Value'] seSite = se.split( '_' )[0].split( '-' )[0] found = False print replicas for replicaSE in replicas.keys(): if re.search( seSite, replicaSE ): found = True problematicDict['SE'] = replicaSE se = replicaSE if not found: gLogger.info( "PFNMissing replica (%d) is no longer registered at SE. Resolved." % fileID ) return self.__updateCompletedFiles( 'PFNMissing', fileID ) gLogger.info( "PFNMissing replica (%d) does not exist. Removing from catalog..." % fileID ) res = returnSingleResult( self.fc.removeReplica( {lfn:problematicDict} ) ) if not res['OK']: return self.__returnProblematicError( fileID, res ) if len( replicas ) == 1: gLogger.info( "PFNMissing replica (%d) had a single replica. Updating prognosis" % fileID ) return self.changeProblematicPrognosis( fileID, 'LFNZeroReplicas' ) res = self.dm.replicateAndRegister( problematicDict['LFN'], se ) if not res['OK']: return self.__returnProblematicError( fileID, res ) # If we get here the problem is solved so we can update the integrityDB return self.__updateCompletedFiles( 'PFNMissing', fileID )
def _run(self): """Check for migration bit, set file done when migrated.""" self.waitingFiles = self.getWaitingFilesList() self.log.notice("Waiting files:", len(self.waitingFiles)) targetSESet = set(self.operation.targetSEList) self.log.notice("Target SEs:", ",".join(targetSESet)) migrated = True for opFile in self.waitingFiles: self.log.notice("Checking:", opFile.LFN) for targetSE in targetSESet: se = StorageElement(targetSE) if not se.status()["TapeSE"]: migrated = True and migrated continue metaData = returnSingleResult(se.getFileMetadata(opFile.LFN)) self.log.debug("MetaData: %s" % pformat(metaData)) if not metaData["OK"]: self.log.error("Failed to get metadata:", "%s: %s" % (opFile.LFN, metaData["Message"])) migrated = False continue migrated = metaData["Value"].get("Migrated", 0) == 1 and migrated if migrated: self.log.notice("File has been migrated:", opFile.LFN) opFile.Status = "Done" else: self.log.notice("File has NOT been migrated:", opFile.LFN) now = datetime.datetime.utcnow().replace(microsecond=0) extraDelay = datetime.timedelta(minutes=20) self.request.NotBefore = now + extraDelay
def _registerDescendent(self): """Register the tarball as a descendent of the archived LFNs. Actually registers all LFNs as an ancestor to the Tarball. """ registerDescendents = self.parameterDict.get("RegisterDescendent", None) if not registerDescendents: self.log.verbose( "Will not register tarball as descendent to the Archived LFNs." ) return self.log.info( "Will register tarball as descendent to the Archived LFNs.") tarballLFN = self.parameterDict["ArchiveLFN"] ancestorDict = {tarballLFN: {"Ancestors": self.lfns}} for _trial in range(3): resAncestors = returnSingleResult( self.fc.addFileAncestors(ancestorDict)) if resAncestors["OK"]: break else: self.log.error("Failed to register ancestors", resAncestors["Message"]) raise RuntimeError("Failed to register ancestors") self.log.info("Successfully registered ancestors")
def downloadFile(dm, lfn, localFolder): res = returnSingleResult(dm.getFile(lfn, localFolder)) if not res or not res['OK']: print "Error with file %s"%lfn print res else: print "Downloaded %s"%lfn
def _checkArchiveLFN(self): """Make sure the archive LFN does not exist yet.""" archiveLFN = self.parameterDict["ArchiveLFN"] exists = returnSingleResult(self.fc.isFile(archiveLFN)) self.log.debug("Checking for Tarball existence %r" % exists) if exists["OK"] and exists["Value"]: raise RuntimeError("Tarball %r already exists" % archiveLFN)
def _run(self): """Check for migration bit, set file done when migrated.""" self.waitingFiles = self.getWaitingFilesList() self.log.notice('Waiting files:', len(self.waitingFiles)) targetSESet = set(self.operation.targetSEList) self.log.notice('Target SEs:', ','.join(targetSESet)) migrated = True for opFile in self.waitingFiles: self.log.notice('Checking:', opFile.LFN) for targetSE in targetSESet: se = StorageElement(targetSE) if not se.status()['TapeSE']: migrated = True and migrated continue metaData = returnSingleResult(se.getFileMetadata(opFile.LFN)) self.log.debug('MetaData: %s' % pformat(metaData)) if not metaData['OK']: self.log.error( 'Failed to get metadata:', '%s: %s' % (opFile.LFN, metaData['Message'])) migrated = False continue migrated = metaData['Value'].get('Migrated', 0) == 1 and migrated if migrated: self.log.notice('File has been migrated:', opFile.LFN) opFile.Status = 'Done' else: self.log.notice('File has NOT been migrated:', opFile.LFN) now = datetime.datetime.utcnow().replace(microsecond=0) extraDelay = datetime.timedelta(minutes=20) self.request.NotBefore = now + extraDelay
def test_isFile( self ): print '\n\n#########################################################################\n\n\t\t\tIs file size test\n' destinationFilePath = '%s/testFile.%s' % ( self.destDirectory, time.time() ) pfnForLfnRes = returnSingleResult( self.storageElement.getPfnForLfn( destinationFilePath ) ) destinationPfn = pfnForLfnRes['Value'] fileDict = {destinationPfn:self.localSourceFile} putFileRes = self.storageElement.putFile( fileDict, singleFile = True ) # Is a file isFileRes = self.storageElement.isFile( destinationPfn, singleFile = True ) # Now remove the destination file removeFileRes = self.storageElement.removeFile( destinationPfn, singleFile = True ) # Get metadata for a removed file missingIsFileRes = self.storageElement.isFile( destinationPfn, singleFile = True ) # Check directories are handled properly destinationDir = os.path.dirname( destinationPfn ) directoryIsFileRes = self.storageElement.isFile( destinationDir, singleFile = True ) # Check that the put was done correctly self.assert_( putFileRes['OK'] ) self.assert_( putFileRes['Value'] ) self.assertEqual( putFileRes['Value'], self.localFileSize ) # Check that we checked the file correctly self.assert_( isFileRes['OK'] ) self.assert_( isFileRes['Value'] ) # Check that the removal was done correctly self.assert_( removeFileRes['OK'] ) self.assert_( removeFileRes['Value'] ) # Check the is file for non existant file self.assertFalse( missingIsFileRes['OK'] ) expectedError = "File does not exist" self.assert_( expectedError in missingIsFileRes['Message'] ) # Check that is file operation with a directory self.assert_( directoryIsFileRes['OK'] ) self.assertFalse( directoryIsFileRes['Value'] )
def __insertRegisterOperation( self, request, operation, toRegister ): """ add RegisterReplica operation :param Request request: request instance :param Operation transferOp: 'ReplicateAndRegister' operation for this FTSJob :param list toRegister: [ FTSDB.FTSFile, ... ] - files that failed to register """ log = self.log.getSubLogger( "req_%s/%s/registerFiles" % ( request.RequestID, request.RequestName ) ) byTarget = {} for ftsFile in toRegister: if ftsFile.TargetSE not in byTarget: byTarget.setdefault( ftsFile.TargetSE, [] ) byTarget[ftsFile.TargetSE].append( ftsFile ) log.info( "will create %s 'RegisterReplica' operations" % len( byTarget ) ) for target, ftsFileList in byTarget.iteritems(): log.info( "creating 'RegisterReplica' operation for targetSE %s with %s files..." % ( target, len( ftsFileList ) ) ) registerOperation = Operation() registerOperation.Type = "RegisterReplica" registerOperation.Status = "Waiting" registerOperation.TargetSE = target targetSE = StorageElement( target ) for ftsFile in ftsFileList: opFile = File() opFile.LFN = ftsFile.LFN pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = self.registrationProtocols ) ) if not pfn["OK"]: continue opFile.PFN = pfn["Value"] registerOperation.addFile( opFile ) request.insertBefore( registerOperation, operation ) return S_OK()
def __uploadFile(self, se, pfn): """proxied upload file""" res = self.__prepareSecurityDetails() if not res["OK"]: return res # Put file to the SE try: storageElement = StorageElement(se) except AttributeError as x: errStr = "__uploadFile: Exception while instantiating the Storage Element." gLogger.exception(errStr, se, str(x)) return S_ERROR(errStr) putFileDir = "%s/putFile" % BASE_PATH localFileName = "%s/%s" % (putFileDir, os.path.basename(pfn)) res = returnSingleResult(storageElement.putFile({pfn: localFileName})) if not res["OK"]: gLogger.error("prepareFile: Failed to put local file to storage.", res["Message"]) # Clear the local cache try: gLogger.debug("Removing temporary file", localFileName) os.remove(localFileName) except Exception as x: gLogger.exception("Failed to remove local file", localFileName, x) return res
def finalize( self ): """ register successfully transferred files """ if self.Status not in FTSJob.FINALSTATES: return S_OK() startTime = time.time() targetSE = StorageElement( self.TargetSE ) toRegister = [ ftsFile for ftsFile in self if ftsFile.Status == "Finished" ] toRegisterDict = {} for ftsFile in toRegister: pfn = returnSingleResult( targetSE.getPfnForProtocol( ftsFile.TargetSURL, protocol = "SRM2", withPort = False ) ) if not pfn["OK"]: continue pfn = pfn["Value"] toRegisterDict[ ftsFile.LFN ] = { "PFN": pfn, "SE": self.TargetSE } if toRegisterDict: self._regTotal += len( toRegisterDict ) register = self._fc.addReplica( toRegisterDict ) self._regTime += time.time() - startTime if not register["OK"]: for ftsFile in toRegister: ftsFile.Error = "AddCatalogReplicaFailed" return register register = register["Value"] self._regSuccess += len( register.get( 'Successful', {} ) ) failedFiles = register.get( "Failed", {} ) for ftsFile in toRegister: if ftsFile.LFN in failedFiles: ftsFile.Error = "AddCatalogReplicaFailed" return S_OK()
def downloadFile(dm, lfn, localFolder): res = returnSingleResult(dm.getFile(lfn, localFolder)) if not res or not res['OK']: print "Error with file %s" % lfn print res else: print "Downloaded %s" % lfn
def test_getAccessUrl(self): print '\n\n#########################################################################\n\n\t\tGet access url test\n' destinationFilePath = '%s/testFile.%s' % (self.destDirectory, time.time()) pfnForLfnRes = returnSingleResult( self.storageElement.getPfnForLfn(destinationFilePath)) destinationPfn = pfnForLfnRes['Value'] fileDict = {destinationPfn: self.localSourceFile} putFileRes = self.storageElement.putFile(fileDict, singleFile=True) # Get a transfer url for the file getTurlRes = self.storageElement.getAccessUrl(destinationPfn, singleFile=True) # Remove the destination file removeFileRes = self.storageElement.removeFile(destinationPfn, singleFile=True) # Get missing turl res getMissingTurlRes = self.storageElement.getAccessUrl(destinationPfn, singleFile=True) # Check that the put was done correctly self.assert_(putFileRes['OK']) self.assert_(putFileRes['Value']) self.assertEqual(putFileRes['Value'], self.localFileSize) # Check that we can get the tURL properly self.assert_(getTurlRes['OK']) self.assert_(getTurlRes['Value']) self.assert_(type(getTurlRes['Value']) in types.StringTypes) # Check that the removal was done correctly self.assert_(removeFileRes['OK']) self.assert_(removeFileRes['Value'])
def __getCatalogDirectoryContents(self, directories): """ get catalog contents under paths :directories: :param self: self reference :param list directories: list of paths in catalog """ self.log.info('Obtaining the catalog contents for %d directories:' % len(directories)) for directory in directories: self.log.info(directory) activeDirs = directories allFiles = {} fc = FileCatalog() while activeDirs: currentDir = activeDirs[0] res = returnSingleResult(fc.listDirectory(currentDir)) activeDirs.remove(currentDir) if not res['OK'] and 'Directory does not exist' in res['Message']: # FIXME: DFC should return errno self.log.info("The supplied directory %s does not exist" % currentDir) elif not res['OK']: if "No such file or directory" in res['Message']: self.log.info("%s: %s" % (currentDir, res['Message'])) else: self.log.error("Failed to get directory %s content: %s" % (currentDir, res['Message'])) else: dirContents = res['Value'] activeDirs.extend(dirContents['SubDirs']) allFiles.update(dirContents['Files']) self.log.info("Found %d files" % len(allFiles)) return S_OK(allFiles.keys())
def checkArchive(self, archiveLFN): """Check that archiveLFN does not exist yet.""" sLog.notice("Using Tarball: %s" % archiveLFN) exists = returnSingleResult(self.fcClient.isFile(archiveLFN)) sLog.debug("Checking for Tarball existence %r" % exists) if exists["OK"] and exists["Value"]: raise RuntimeError("Tarball %r already exists" % archiveLFN) sLog.debug("Checking permissions for %r" % archiveLFN) hasAccess = returnSingleResult( self.fcClient.hasAccess(archiveLFN, "addFile")) if not archiveLFN or not hasAccess["OK"] or not hasAccess["Value"]: sLog.error("Error checking tarball location: %r" % hasAccess) raise ValueError( '%s is not a valid path, parameter "Name" must be correct' % archiveLFN)
def _registerDescendent(self): """Register the tarball as a descendent of the archived LFNs. Actually registers all LFNs as an ancestor to the Tarball. """ registerDescendents = self.parameterDict.get('RegisterDescendent', None) if not registerDescendents: self.log.verbose( 'Will not register tarball as descendent to the Archived LFNs.' ) return self.log.info( 'Will register tarball as descendent to the Archived LFNs.') tarballLFN = self.parameterDict['ArchiveLFN'] ancestorDict = {tarballLFN: {'Ancestors': self.lfns}} for _trial in range(3): resAncestors = returnSingleResult( self.fc.addFileAncestors(ancestorDict)) if resAncestors['OK']: break else: self.log.error('Failed to register ancestors', resAncestors['Message']) raise RuntimeError('Failed to register ancestors') self.log.info('Successfully registered ancestors')
def __registerSuccessful( self, transLFNs ): """ register successfully transferred files to the catalogs, fill failedRegistrations dict for files that failed to register :param self: self reference :param list transLFNs: LFNs in FTS job """ self.failedRegistrations = {} toRegister = {} for lfn in transLFNs: res = returnSingleResult( self.oTargetSE.getPfnForProtocol( self.fileDict[lfn].get( 'Target' ), protocol = 'SRM2', withPort = False ) ) if not res['OK']: self.__setFileParameter( lfn, 'Reason', res['Message'] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) else: toRegister[lfn] = { 'PFN' : res['Value'], 'SE' : self.targetSE } if not toRegister: return S_OK( ( 0, 0 ) ) res = self.__getCatalogObject() if not res['OK']: for lfn in toRegister: self.failedRegistrations = toRegister self.log.error( 'Failed to get Catalog Object', res['Message'] ) return S_OK( ( 0, len( toRegister ) ) ) res = self.oCatalog.addReplica( toRegister ) if not res['OK']: self.failedRegistrations = toRegister self.log.error( 'Failed to get Catalog Object', res['Message'] ) return S_OK( ( 0, len( toRegister ) ) ) for lfn, error in res['Value']['Failed'].items(): self.failedRegistrations[lfn] = toRegister[lfn] self.log.error( 'Registration of Replica failed', '%s : %s' % ( lfn, str( error ) ) ) return S_OK( ( len( res['Value']['Successful'] ), len( toRegister ) ) )
def _downloadFiles(self): """Download the files.""" self._checkFilePermissions() for index, opFile in enumerate(self.waitingFiles): lfn = opFile.LFN self.log.info('Processing file (%d/%d) %r' % (index, len(self.waitingFiles), lfn)) sourceSE = self.parameterDict['SourceSE'] attempts = 0 destFolder = os.path.join(self.cacheFolder, os.path.dirname(lfn)[1:]) self.log.debug('Local Cache Folder: %s' % destFolder) if not os.path.exists(destFolder): os.makedirs(destFolder) while True: attempts += 1 download = returnSingleResult(self.dm.getFile(lfn, destinationDir=destFolder, sourceSE=sourceSE)) if download['OK']: self.log.info('Downloaded file %r to %r' % (lfn, destFolder)) break errorString = download['Message'] self.log.error('Failed to download file:', errorString) opFile.Error = errorString opFile.Attempt += 1 self.operation.Error = opFile.Error if 'No such file or directory' in opFile.Error: # The File does not exist, we just ignore this and continue, otherwise we never archive the other files opFile.Status = 'Done' download = S_OK() break if attempts > 10: self.log.error('Completely failed to download file:', errorString) raise RuntimeError('Completely failed to download file: %s' % errorString) return
def _checkArchiveLFN(self): """Make sure the archive LFN does not exist yet.""" archiveLFN = self.parameterDict['ArchiveLFN'] exists = returnSingleResult(self.fc.isFile(archiveLFN)) self.log.debug('Checking for Tarball existence %r' % exists) if exists['OK'] and exists['Value']: raise RuntimeError('Tarball %r already exists' % archiveLFN)
def __getCatalogDirectoryContents(self, directories): """ get catalog contents under paths :directories: :param self: self reference :param list directories: list of paths in catalog """ self.log.info('Obtaining the catalog contents for %d directories:' % len(directories)) for directory in directories: self.log.info(directory) activeDirs = directories allFiles = {} fc = FileCatalog() while activeDirs: currentDir = activeDirs[0] res = returnSingleResult(fc.listDirectory(currentDir)) activeDirs.remove(currentDir) if not res['OK'] and 'Directory does not exist' in res[ 'Message']: # FIXME: DFC should return errno self.log.info("The supplied directory %s does not exist" % currentDir) elif not res['OK']: if "No such file or directory" in res['Message']: self.log.info("%s: %s" % (currentDir, res['Message'])) else: self.log.error("Failed to get directory %s content: %s" % (currentDir, res['Message'])) else: dirContents = res['Value'] activeDirs.extend(dirContents['SubDirs']) allFiles.update(dirContents['Files']) self.log.info("Found %d files" % len(allFiles)) return S_OK(allFiles.keys())
def tearDown(self): destinationDir = returnSingleResult( self.storageElement.getPfnForLfn(self.destDirectory)) res = self.storageElement.removeDirectory(destinationDir, recursive=True, singleDirectory=True) self.assert_(res['OK'])
def __registerSuccessful( self, transLFNs ): """ register successfully transferred files to the catalogs, fill failedRegistrations dict for files that failed to register :param self: self reference :param list transLFNs: LFNs in FTS job """ self.failedRegistrations = {} toRegister = {} for lfn in transLFNs: res = returnSingleResult( self.oTargetSE.getURL( self.fileDict[lfn].get( 'Target' ), protocol = 'srm' ) ) if not res['OK']: self.__setFileParameter( lfn, 'Reason', res['Message'] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) else: toRegister[lfn] = { 'PFN' : res['Value'], 'SE' : self.targetSE } if not toRegister: return S_OK( ( 0, 0 ) ) res = self.__getCatalogObject() if not res['OK']: for lfn in toRegister: self.failedRegistrations = toRegister self.log.error( 'Failed to get Catalog Object', res['Message'] ) return S_OK( ( 0, len( toRegister ) ) ) res = self.oCatalog.addReplica( toRegister ) if not res['OK']: self.failedRegistrations = toRegister self.log.error( 'Failed to get Catalog Object', res['Message'] ) return S_OK( ( 0, len( toRegister ) ) ) for lfn, error in res['Value']['Failed'].items(): self.failedRegistrations[lfn] = toRegister[lfn] self.log.error( 'Registration of Replica failed', '%s : %s' % ( lfn, str( error ) ) ) return S_OK( ( len( res['Value']['Successful'] ), len( toRegister ) ) )
def resolvePFNUnavailable(self, problematicDict): """ This takes the problematic dictionary returned by the integrity DB and resolved the PFNUnavailable prognosis """ lfn = problematicDict['LFN'] se = problematicDict['SE'] fileID = problematicDict['FileID'] res = returnSingleResult(StorageElement(se).getFileMetadata(lfn)) if (not res['OK']) and (re.search('File does not exist', res['Message'])): # The file is no longer Unavailable but has now dissapeared completely gLogger.info( "PFNUnavailable replica (%d) found to be missing. Updating prognosis" % fileID) return self.changeProblematicPrognosis(fileID, 'PFNMissing') if (not res['OK']) or res['Value']['Unavailable']: gLogger.info( "PFNUnavailable replica (%d) found to still be Unavailable" % fileID) return self.incrementProblematicRetry(fileID) if res['Value']['Lost']: gLogger.info( "PFNUnavailable replica (%d) is now found to be Lost. Updating prognosis" % fileID) return self.changeProblematicPrognosis(fileID, 'PFNLost') gLogger.info("PFNUnavailable replica (%d) is no longer Unavailable" % fileID) # Need to make the replica okay in the Catalog return self.__updateReplicaToChecked(problematicDict)
def __prepareFile(self, se, pfn): """proxied prepare file""" res = self.__prepareSecurityDetails() if not res["OK"]: return res # Clear the local cache getFileDir = "%s/getFile" % BASE_PATH if not os.path.exists(getFileDir): os.mkdir(getFileDir) # Get the file to the cache try: storageElement = StorageElement(se) except AttributeError as x: errStr = "prepareFile: Exception while instantiating the Storage Element." gLogger.exception(errStr, se, str(x)) return S_ERROR(errStr) res = returnSingleResult( storageElement.getFile(pfn, localPath="%s/getFile" % BASE_PATH)) if not res["OK"]: gLogger.error("prepareFile: Failed to get local copy of file.", res["Message"]) return res return S_OK()
def resolveCatalogPFNSizeMismatch( self, problematicDict ): """ This takes the problematic dictionary returned by the integrity DB and resolved the CatalogPFNSizeMismatch prognosis """ lfn = problematicDict['LFN'] pfn = problematicDict['PFN'] se = problematicDict['SE'] fileID = problematicDict['FileID'] res = returnSingleResult( self.fc.getFileSize( lfn ) ) if not res['OK']: return self.__returnProblematicError( fileID, res ) catalogSize = res['Value'] res = returnSingleResult( StorageElement( se ).getFileSize( pfn ) ) if not res['OK']: return self.__returnProblematicError( fileID, res ) storageSize = res['Value'] bkKCatalog = FileCatalog( ['BookkeepingDB'] ) res = returnSingleResult( bkKCatalog.getFileSize( lfn ) ) if not res['OK']: return self.__returnProblematicError( fileID, res ) bookkeepingSize = res['Value'] if bookkeepingSize == catalogSize == storageSize: gLogger.info( "CatalogPFNSizeMismatch replica (%d) matched all registered sizes." % fileID ) return self.__updateReplicaToChecked( problematicDict ) if ( catalogSize == bookkeepingSize ): gLogger.info( "CatalogPFNSizeMismatch replica (%d) found to mismatch the bookkeeping also" % fileID ) res = returnSingleResult( self.fc.getReplicas( lfn ) ) if not res['OK']: return self.__returnProblematicError( fileID, res ) if len( res['Value'] ) <= 1: gLogger.info( "CatalogPFNSizeMismatch replica (%d) has no other replicas." % fileID ) return S_ERROR( "Not removing catalog file mismatch since the only replica" ) else: gLogger.info( "CatalogPFNSizeMismatch replica (%d) has other replicas. Removing..." % fileID ) res = self.dm.removeReplica( se, lfn ) if not res['OK']: return self.__returnProblematicError( fileID, res ) return self.__updateCompletedFiles( 'CatalogPFNSizeMismatch', fileID ) if ( catalogSize != bookkeepingSize ) and ( bookkeepingSize == storageSize ): gLogger.info( "CatalogPFNSizeMismatch replica (%d) found to match the bookkeeping size" % fileID ) res = self.__updateReplicaToChecked( problematicDict ) if not res['OK']: return self.__returnProblematicError( fileID, res ) return self.changeProblematicPrognosis( fileID, 'BKCatalogSizeMismatch' ) gLogger.info( "CatalogPFNSizeMismatch replica (%d) all sizes found mismatch. Updating retry count" % fileID ) return self.incrementProblematicRetry( fileID )
def resolvePFNZeroSize(self, problematicDict): """ This takes the problematic dictionary returned by the integrity DB and resolves the PFNZeroSize prognosis """ lfn = problematicDict['LFN'] seName = problematicDict['SE'] fileID = problematicDict['FileID'] se = StorageElement(seName) res = returnSingleResult(se.getFileSize(lfn)) if (not res['OK']) and (re.search('File does not exist', res['Message'])): gLogger.info( "PFNZeroSize replica (%d) found to be missing. Updating prognosis" % problematicDict['FileID']) return self.changeProblematicPrognosis(fileID, 'PFNMissing') storageSize = res['Value'] if storageSize == 0: res = returnSingleResult(se.removeFile(lfn)) if not res['OK']: return self.__returnProblematicError(fileID, res) gLogger.info( "PFNZeroSize replica (%d) removed. Updating prognosis" % problematicDict['FileID']) return self.changeProblematicPrognosis(fileID, 'PFNMissing') res = returnSingleResult(self.fc.getReplicas(lfn)) if not res['OK']: return self.__returnProblematicError(fileID, res) if seName not in res['Value']: gLogger.info( "PFNZeroSize replica (%d) not registered in catalog. Updating prognosis" % problematicDict['FileID']) return self.changeProblematicPrognosis(fileID, 'PFNNotRegistered') res = returnSingleResult(self.fc.getFileMetadata(lfn)) if not res['OK']: return self.__returnProblematicError(fileID, res) catalogSize = res['Value']['Size'] if catalogSize != storageSize: gLogger.info( "PFNZeroSize replica (%d) size found to differ from registered metadata. Updating prognosis" % problematicDict['FileID']) return self.changeProblematicPrognosis(fileID, 'CatalogPFNSizeMismatch') return self.__updateCompletedFiles('PFNZeroSize', fileID)
def _hasAccess(self, lfn, s3_method): """Check if we have permission to execute given operation on the given file (if exists) or its directory""" opType = self._s3ToFC_methods.get(s3_method) if not opType: return S_ERROR(errno.EINVAL, "Unknown S3 method %s" % s3_method) return returnSingleResult(self._fc.hasAccess(lfn, opType))
def test_putFile( self ): print '\n\n#########################################################################\n\n\t\t\tPut file test\n' destinationFilePath = '%s/testFile.%s' % ( self.destDirectory, time.time() ) # pfnForLfnRes = returnSingleResult( self.storageElement.getURL( destinationFilePath ) ) #destinationPfn = pfnForLfnRes['Value'] fileDict = {destinationFilePath:self.localSourceFile} putFileRes = returnSingleResult( self.storageElement.putFile( fileDict ) ) # Now remove the destination file removeFileRes = returnSingleResult( self.storageElement.removeFile( destinationFilePath ) ) # Check that the put was done correctly self.assert_( putFileRes['OK'] ) self.assert_( putFileRes['Value'] ) self.assertEqual( putFileRes['Value'], self.localFileSize ) # Check that the removal was done correctly self.assert_( removeFileRes['OK'] ) self.assert_( removeFileRes['Value'] )
def resolveLFNCatalogMissing( self, problematicDict ): """ This takes the problematic dictionary returned by the integrity DB and resolved the LFNCatalogMissing prognosis """ lfn = problematicDict['LFN'] fileID = problematicDict['FileID'] res = returnSingleResult( self.fc.exists( lfn ) ) if not res['OK']: return self.__returnProblematicError( fileID, res ) if res['Value']: return self.__updateCompletedFiles( 'LFNCatalogMissing', fileID ) # Remove the file from all catalogs # RF_NOTE : here I can do it because it's a single file, but otherwise I would need to sort the path res = returnSingleResult( self.fc.removeFile( lfn ) ) if not res['OK']: return self.__returnProblematicError( fileID, res ) return self.__updateCompletedFiles( 'LFNCatalogMissing', fileID )
def resolveLFNCatalogMissing(self, problematicDict): """ This takes the problematic dictionary returned by the integrity DB and resolved the LFNCatalogMissing prognosis """ lfn = problematicDict['LFN'] fileID = problematicDict['FileID'] res = returnSingleResult(self.fc.exists(lfn)) if not res['OK']: return self.__returnProblematicError(fileID, res) if res['Value']: return self.__updateCompletedFiles('LFNCatalogMissing', fileID) # Remove the file from all catalogs # RF_NOTE : here I can do it because it's a single file, but otherwise I would need to sort the path res = returnSingleResult(self.fc.removeFile(lfn)) if not res['OK']: return self.__returnProblematicError(fileID, res) return self.__updateCompletedFiles('LFNCatalogMissing', fileID)
def setUp( self ): self.numberOfFiles = 1 self.storageElement = StorageElement( storageElementToTest ) self.localSourceFile = "/etc/group" self.localFileSize = getSize( self.localSourceFile ) self.destDirectory = "/lhcb/test/unit-test/TestStorageElement" destinationDir = returnSingleResult( self.storageElement.getPfnForLfn( self.destDirectory ) ) res = self.storageElement.createDirectory( destinationDir, singleDirectory = True ) self.assert_( res['OK'] )
def resolveTarget( self ): """ find target SE eligible for submission :param self: self reference """ toResolve = [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( 'Status' ) not in self.noSubmitStatus ] if not toResolve: return S_OK() res = self.__updateReplicaCache( toResolve ) if not res['OK']: return res for lfn in toResolve: res = returnSingleResult( self.oTargetSE.getURL( lfn, protocol = 'srm' ) ) if not res['OK']: reason = res.get( 'Message', res['Message'] ) gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, reason ) ) self.__setFileParameter( lfn, 'Reason', reason ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue res = self.setTargetSURL( lfn, res['Value'] ) if not res['OK']: gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, res["Message"] ) ) self.__setFileParameter( lfn, 'Reason', res['Message'] ) self.__setFileParameter( lfn, 'Status', 'Failed' ) continue toResolve = [] for lfn in self.fileDict: if "Target" in self.fileDict[lfn]: toResolve.append( lfn ) if not toResolve: return S_ERROR( "No eligible Target files" ) res = self.oTargetSE.exists( toResolve ) if not res['OK']: return S_ERROR( "Failed to check target existence" ) for lfn, error in res['Value']['Failed'].items(): self.__setFileParameter( lfn, 'Reason', error ) self.__setFileParameter( lfn, 'Status', 'Failed' ) toRemove = [] for lfn, exists in res['Value']['Successful'].items(): if exists: res = self.getSourceSURL( lfn ) if not res['OK']: gLogger.warn( "resolveTarget: skipping %s - target exists" % lfn ) self.__setFileParameter( lfn, 'Reason', "Target exists" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) elif res['Value'] == self.fileDict[lfn]['Target']: gLogger.warn( "resolveTarget: skipping %s - source and target pfns are the same" % lfn ) self.__setFileParameter( lfn, 'Reason', "Source and Target the same" ) self.__setFileParameter( lfn, 'Status', 'Failed' ) else: toRemove.append( lfn ) if toRemove: self.oTargetSE.removeFile( toRemove ) return S_OK()
def removeStorageDirectoryFromSE( directory, storageElement ): """ Delete directory on selected storage element """ se = StorageElement( storageElement, False ) res = returnSingleResult( se.exists( directory ) ) if not res['OK']: return S_ERROR( "Failed to obtain existence of directory" + res['Message'] ) exists = res['Value'] if not exists: return S_OK( "The directory %s does not exist at %s " % ( directory, storageElement ) ) res = returnSingleResult( se.removeDirectory( directory, recursive = True ) ) if not res['OK']: return S_ERROR( "Failed to remove storage directory" + res['Message'] ) return S_OK()
def __updateReplicaToChecked( self, problematicDict ): lfn = problematicDict['LFN'] fileID = problematicDict['FileID'] prognosis = problematicDict['Prognosis'] problematicDict['Status'] = 'Checked' res = returnSingleResult( self.fc.setReplicaStatus( {lfn:problematicDict} ) ) if not res['OK']: return self.__returnProblematicError( fileID, res ) gLogger.info( "%s replica (%d) is updated to Checked status" % ( prognosis, fileID ) ) return self.__updateCompletedFiles( prognosis, fileID )
def cleanTransformationLogFiles( self, directory ): """ clean up transformation logs from directory :directory: :param self: self reference :param str directory: folder name """ self.log.verbose( "Removing log files found in the directory %s" % directory ) res = returnSingleResult( StorageElement( self.logSE ).removeDirectory( directory ) ) if not res['OK']: self.log.error( "Failed to remove log files", res['Message'] ) return res self.log.info( "Successfully removed transformation log directory" ) return S_OK()
def __generateURLDict(self, lfns, storage, replicaDict=None): """ Generates a dictionary (url : lfn ), where the url are constructed from the lfn using the constructURLFromLFN method of the storage plugins. :param: lfns : dictionary {lfn:whatever} :returns dictionary {constructed url : lfn} """ log = self.log.getSubLogger("__generateURLDict") log.verbose("generating url dict for %s lfn in %s." % (len(lfns), self.name)) if not replicaDict: replicaDict = {} urlDict = {} # url : lfn failed = {} # lfn : string with errors for lfn in lfns: if self.useCatalogURL: # Is this self.name alias proof? url = replicaDict.get(lfn, {}).get(self.name, '') if url: urlDict[url] = lfn continue else: fc = self.__getFileCatalog() result = fc.getReplicas() if not result['OK']: failed[lfn] = result['Message'] url = result['Value']['Successful'].get(lfn, {}).get(self.name, '') if not url: failed[lfn] = 'Failed to get catalog replica' else: # Update the URL according to the current SE description result = returnSingleResult(storage.updateURL(url)) if not result['OK']: failed[lfn] = result['Message'] else: urlDict[result['Value']] = lfn else: result = storage.constructURLFromLFN(lfn, withWSUrl=True) if not result['OK']: errStr = result['Message'] log.debug(errStr, 'for %s' % (lfn)) failed[lfn] = "%s %s" % (failed[lfn], errStr) if lfn in failed else errStr else: urlDict[result['Value']] = lfn res = S_OK({'Successful': urlDict, 'Failed': failed}) # res['Failed'] = failed return res
def resolvePFNZeroSize( self, problematicDict ): """ This takes the problematic dictionary returned by the integrity DB and resolves the PFNZeroSize prognosis """ lfn = problematicDict['LFN'] seName = problematicDict['SE'] fileID = problematicDict['FileID'] se = StorageElement( seName ) res = returnSingleResult( se.getFileSize( lfn ) ) if ( not res['OK'] ) and ( re.search( 'File does not exist', res['Message'] ) ): gLogger.info( "PFNZeroSize replica (%d) found to be missing. Updating prognosis" % problematicDict['FileID'] ) return self.changeProblematicPrognosis( fileID, 'PFNMissing' ) storageSize = res['Value'] if storageSize == 0: res = returnSingleResult( se.removeFile( lfn ) ) if not res['OK']: return self.__returnProblematicError( fileID, res ) gLogger.info( "PFNZeroSize replica (%d) removed. Updating prognosis" % problematicDict['FileID'] ) return self.changeProblematicPrognosis( fileID, 'PFNMissing' ) res = returnSingleResult( self.fc.getReplicas( lfn ) ) if not res['OK']: return self.__returnProblematicError( fileID, res ) if seName not in res['Value']: gLogger.info( "PFNZeroSize replica (%d) not registered in catalog. Updating prognosis" % problematicDict['FileID'] ) return self.changeProblematicPrognosis( fileID, 'PFNNotRegistered' ) res = returnSingleResult( self.fc.getFileMetadata( lfn ) ) if not res['OK']: return self.__returnProblematicError( fileID, res ) catalogSize = res['Value']['Size'] if catalogSize != storageSize: gLogger.info( "PFNZeroSize replica (%d) size found to differ from registered metadata. Updating prognosis" % problematicDict['FileID'] ) return self.changeProblematicPrognosis( fileID, 'CatalogPFNSizeMismatch' ) return self.__updateCompletedFiles( 'PFNZeroSize', fileID )
def __getRegisteredPFNLFN( self, pfn, storageElement ): res = StorageElement( storageElement ).getPfnForProtocol( pfn, withPort = False ) if not res['OK']: gLogger.error( "Failed to get registered PFN for physical files", res['Message'] ) return res for pfn, error in res['Value']['Failed'].items(): gLogger.error( 'Failed to obtain registered PFN for physical file', '%s %s' % ( pfn, error ) ) return S_ERROR( 'Failed to obtain registered PFNs from physical file' ) registeredPFN = res['Value']['Successful'][pfn] res = returnSingleResult( self.fc.getLFNForPFN( registeredPFN ) ) if ( not res['OK'] ) and re.search( 'No such file or directory', res['Message'] ): return S_OK( False ) return S_OK( res['Value'] )
def finalize( self ): """ register successfully transferred files """ if self.Status not in FTSJob.FINALSTATES: return S_OK() if not len( self ): return S_ERROR( "Empty job in finalize" ) startTime = time.time() targetSE = StorageElement( self.TargetSE ) toRegister = [ ftsFile for ftsFile in self if ftsFile.Status == "Finished" ] toRegisterDict = {} for ftsFile in toRegister: pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = 'srm' ) ) if pfn["OK"]: pfn = pfn["Value"] toRegisterDict[ ftsFile.LFN ] = { "PFN": pfn, "SE": self.TargetSE } else: self._log.error( "Error getting SRM URL", pfn['Message'] ) if toRegisterDict: self._regTotal += len( toRegisterDict ) register = self._fc.addReplica( toRegisterDict ) self._regTime += time.time() - startTime if not register["OK"]: self._log.error( 'Error registering replica', register['Message'] ) for ftsFile in toRegister: ftsFile.Error = "AddCatalogReplicaFailed" return register register = register["Value"] self._regSuccess += len( register.get( 'Successful', {} ) ) if self._regSuccess: self._log.info( 'Successfully registered %d replicas' % self._regSuccess ) failedFiles = register.get( "Failed", {} ) errorReason = {} for lfn, reason in failedFiles.items(): errorReason.setdefault( str( reason ), [] ).append( lfn ) for reason in errorReason: self._log.error( 'Error registering %d replicas' % len( errorReason[reason] ), reason ) for ftsFile in toRegister: if ftsFile.LFN in failedFiles: ftsFile.Error = "AddCatalogReplicaFailed" else: statuses = set( [ftsFile.Status for ftsFile in self] ) self._log.warn( "No replicas to register for FTSJob (%s) - Files status: '%s'" % \ ( self.Status, ','.join( sorted( statuses ) ) ) ) return S_OK()
def test_getDirectory( self ): print '\n\n#########################################################################\n\n\t\t\tGet directory test\n' directory = "%s/%s" % ( self.destDirectory, 'getDirectoryTest' ) destDirectory = returnSingleResult( self.storageElement.getPfnForLfn( directory ) )['Value'] # Create a local directory to upload localDir = '/tmp/unit-test' srcFile = '/etc/group' sizeOfLocalFile = getSize( srcFile ) if not os.path.exists( localDir ): os.mkdir( localDir ) for i in range( self.numberOfFiles ): shutil.copy( srcFile, '%s/testFile.%s' % ( localDir, time.time() ) ) time.sleep( 1 ) # Check that we can successfully upload the directory to the storage element dirDict = {destDirectory:localDir} putDirRes = self.storageElement.putDirectory( dirDict, singleDirectory = True ) # Get the directory metadata #Clean up the locally created directory shutil.rmtree( localDir ) getDirRes = self.storageElement.getDirectory( destDirectory, localPath = localDir, singleDirectory = True ) # Now remove the remove directory removeDirRes = self.storageElement.removeDirectory( destDirectory, recursive = True, singleDirectory = True ) #Clean up the locally created directory shutil.rmtree( localDir ) # Perform the checks for the put dir operation self.assert_( putDirRes['OK'] ) self.assert_( putDirRes['Value'] ) if putDirRes['Value']['Files']: self.assertEqual( putDirRes['Value']['Files'], self.numberOfFiles ) self.assertEqual( putDirRes['Value']['Size'], self.numberOfFiles * sizeOfLocalFile ) self.assert_( type( putDirRes['Value']['Files'] ) in [types.IntType, types.LongType] ) self.assert_( type( putDirRes['Value']['Size'] ) in [types.IntType, types.LongType] ) # Perform the checks for the get directory operation self.assert_( getDirRes['OK'] ) self.assert_( getDirRes['Value'] ) if getDirRes['Value']['Files']: self.assertEqual( getDirRes['Value']['Files'], self.numberOfFiles ) self.assertEqual( getDirRes['Value']['Size'], self.numberOfFiles * sizeOfLocalFile ) self.assert_( type( getDirRes['Value']['Files'] ) in [types.IntType, types.LongType] ) self.assert_( type( getDirRes['Value']['Size'] ) in [types.IntType, types.LongType] ) # Perform the checks for the remove directory operation self.assert_( removeDirRes['OK'] ) self.assert_( removeDirRes['Value'] ) if removeDirRes['Value']['FilesRemoved']: self.assertEqual( removeDirRes['Value']['FilesRemoved'], self.numberOfFiles ) self.assertEqual( removeDirRes['Value']['SizeRemoved'], self.numberOfFiles * sizeOfLocalFile ) self.assert_( type( removeDirRes['Value']['FilesRemoved'] ) in [types.IntType, types.LongType] ) self.assert_( type( removeDirRes['Value']['SizeRemoved'] ) in [types.IntType, types.LongType] )
def test_createDirectory( self ): print '\n\n#########################################################################\n\n\t\t\tCreate directory test\n' directory = "%s/%s" % ( self.destDirectory, 'createDirectoryTest' ) pfnForLfnRes = returnSingleResult( self.storageElement.getPfnForLfn( directory ) ) directoryPfn = pfnForLfnRes['Value'] createDirRes = self.storageElement.createDirectory( directoryPfn, singleDirectory = True ) # Remove the target dir removeDirRes = self.storageElement.removeDirectory( directoryPfn, recursive = True, singleDirectory = True ) # Check that the creation was done correctly self.assert_( createDirRes['OK'] ) self.assert_( createDirRes['Value'] ) # Remove the directory self.assert_( removeDirRes['OK'] ) self.assert_( removeDirRes['Value'] )
def test_getFile( self ): print '\n\n#########################################################################\n\n\t\t\tGet file test\n' destinationFilePath = '%s/testFile.%s' % ( self.destDirectory, time.time() ) # pfnForLfnRes = returnSingleResult( self.storageElement.getURL( destinationFilePath ) ) #destinationPfn = pfnForLfnRes['Value'] fileDict = {destinationFilePath:self.localSourceFile} putFileRes = returnSingleResult( self.storageElement.putFile( fileDict ) ) # Now get a local copy of the file getFileRes = returnSingleResult( self.storageElement.getFile( destinationFilePath ) ) # Now remove the destination file removeFileRes = returnSingleResult( self.storageElement.removeFile( destinationFilePath ) ) # Clean up the local mess os.remove( os.path.basename( destinationFilePath ) ) # Check that the put was done correctly self.assertTrue( putFileRes['OK'] ) self.assertTrue( putFileRes['Value'] ) self.assertEqual( putFileRes['Value'], self.localFileSize ) # Check that we got the file correctly self.assertTrue( getFileRes['OK'] ) self.assertEqual( getFileRes['Value'], self.localFileSize ) # Check that the removal was done correctly self.assertTrue( removeFileRes['OK'] ) self.assertTrue( removeFileRes['Value'] )
def _createDirectory(self, path, failed, mode=0o775): """Create the directory at path if it does not exist. :param str path: path to check :param list failed: list of failed paths :param int mode: mode to set for directory """ exists = returnSingleResult(self.fc.isDirectory(path)) if exists['OK'] and exists['Value']: LOG.verbose('Directory already exists:', path) return S_OK() result = returnSingleResult(self.fc.createDirectory(path)) if not result['OK']: LOG.error('Failed to create directory:', '%s: %s' % (path, result['Message'])) failed[path].append(result['Message']) return S_ERROR() LOG.verbose('Successfully created directory:', path) res = self.fc.changePathMode({path: mode}, False) if not res['OK']: LOG.error(res['Message']) failed[path].append(res['Message']) return S_ERROR() LOG.verbose('Successfully changed mode:', path) return S_OK()
def removeRemoteDirectory(fc,lfn): """ Remove file from the catalog """ storageElements = gConfig.getValue( 'Resources/StorageElementGroups/SE_Cleaning_List', [] ) for storageElement in sorted( storageElements ): res = removeStorageDirectoryFromSE( lfn, storageElement ) if not res['OK']: return S_ERROR( "Failed to clean storage directory at all SE:" + res['Message'] ) res = returnSingleResult( fc.removeDirectory( lfn, recursive = True ) ) if not res['OK']: return S_ERROR( "Failed to clean storage directory at all SE:" + res['Message'] ) return S_OK("Successfully removed directory")