def execute(self): """ Run this. """ if not self.workflowStatus['OK'] or not self.stepStatus['OK']: LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('Workflow status is not OK') result = self.resolveInputVariables() if not result['OK']: LOG.error("Failed to resolve input parameters:", result["Message"]) return result if not self.srmfiles: LOG.error('Files txt was not found correctly: %s' % self.srmfiles) return S_ERROR('Files txt was not found correctly') if not isinstance( self.files[0], dict ): LOG.error('Files were not found correctly: %s' % self.files) return S_ERROR('Files were not found correctly') ##Now need to check that there are not that many concurrent jobs getting the overlay at the same time max_concurrent_running = self.ops.getValue('/GetSRM/MaxConcurrentRunning', 100) error_count = 0 while 1: if error_count > 10 : LOG.error('JobDB Content does not return expected dictionary') return S_ERROR('Failed to get number of concurrent overlay jobs') jobMonitor = JobMonitoringClient() res = jobMonitor.getCurrentJobCounters({'ApplicationStatus':'Downloading SRM files'}) if not res['OK']: error_count += 1 time.sleep(60) continue running = 0 if 'Running' in res['Value']: running = res['Value']['Running'] if running < max_concurrent_running: break else: time.sleep(60) self.setApplicationStatus('Downloading SRM files') for filed in self.files: if 'file' not in filed or 'site' not in filed: LOG.error('Dictionnary does not contain correct keys') return S_ERROR('Dictionnary does not contain correct keys') start = os.getcwd() downloadDir = tempfile.mkdtemp(prefix = 'InputData_%s' % (self.counter), dir = start) os.chdir(downloadDir) storageElement = StorageElement( filed['site'] ) result = storageElement.getFile( filed['file'] ) if result['Value']['Failed']: result = storageElement.getFile( filed['file'] ) os.chdir(start) if result['Value']['Failed']: LOG.error("Failed to get the file from storage:", result['Value']['Failed']) return result self.counter += 1 return S_OK()
def setUp( self, mk_getConfigStorageName, mk_getConfigStorageOptions, mk_getConfigStorageProtocols, mk_isLocalSE, mk_addAccountingOperation, ): self.storage = StorageElementItem("FAKE") self.storage.vo = "test"
def setUp( self, _mk_getConfigStorageName, _mk_getConfigStorageOptions, _mk_getConfigStorageProtocols, _mk_generateStorage, _mk_isLocalSE, _mk_addAccountingOperation, _mk_dmsHelpers ): self.seA = StorageElementItem( 'StorageA' ) self.seA.vo = 'lhcb' self.seB = StorageElementItem( 'StorageB' ) self.seB.vo = 'lhcb' self.seC = StorageElementItem( 'StorageC' ) self.seC.vo = 'lhcb' self.seD = StorageElementItem( 'StorageD' ) self.seD.vo = 'lhcb' self.seE = StorageElementItem( 'StorageE' ) self.seE.vo = 'lhcb'
def setUp( self, mk_getConfigStorageName, mk_getConfigStorageOptions, mk_getConfigStorageProtocols, mk_isLocalSE, mk_addAccountingOperation, ): self.se = StorageElementItem("FAKE") self.se.vo = "test" self.basePath = tempfile.mkdtemp(dir="/tmp") # Update the basePath of the plugin self.se.storages[0].basePath = self.basePath self.srcPath = tempfile.mkdtemp(dir="/tmp") self.destPath = tempfile.mkdtemp(dir="/tmp") self.existingFile = "/test/file.txt" self.existingFileSize = 0 self.nonExistingFile = "/test/nonExistingFile.txt" self.subDir = "/test/subDir" self.subFile = os.path.join(self.subDir, "subFile.txt") self.subFileSize = 0 self.FILES = [self.existingFile, self.nonExistingFile, self.subFile] self.DIRECTORIES = [self.subDir] self.ALL = self.FILES + self.DIRECTORIES with open( os.path.join(self.srcPath, self.existingFile.replace("/test/", "")), "w") as f: f.write("I put something in the file so that it has a size\n") self.existingFileSize = os.path.getsize( os.path.join(self.srcPath, self.existingFile.replace("/test/", ""))) assert self.existingFileSize os.mkdir(os.path.join(self.srcPath, os.path.basename(self.subDir))) with open( os.path.join(self.srcPath, self.subFile.replace("/test/", "")), "w") as f: f.write("This one should have a size as well\n") self.subFileSize = os.path.getsize( os.path.join(self.srcPath, self.subFile.replace("/test/", ""))) assert self.subFileSize
class StoragePlugInTestCase( unittest.TestCase ): """ Base test class. Defines all the method to test """ @mock.patch( 'DIRAC.Resources.Storage.StorageFactory.StorageFactory._getConfigStorageName', side_effect = mock_StorageFactory_getConfigStorageName ) @mock.patch( 'DIRAC.Resources.Storage.StorageFactory.StorageFactory._getConfigStorageOptions', side_effect = mock_StorageFactory_getConfigStorageOptions ) @mock.patch( 'DIRAC.Resources.Storage.StorageFactory.StorageFactory._getConfigStorageProtocols', side_effect = mock_StorageFactory_getConfigStorageProtocols ) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value = S_OK( True ) ) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value = None ) # Don't send accounting # @mock.patch( 'DIRAC.Resources.Storage.StorageFactory.StorageFactory._getCurrentURL', # side_effect = mock_StorageFactory_getCurrentURL_getCurrentURL ) def setUp( self, mk_getConfigStorageName, mk_getConfigStorageOptions, mk_getConfigStorageProtocols, mk_isLocalSE, mk_addAccountingOperation ): self.storage = StorageElementItem( 'FAKE' ) self.storage.vo = 'test' # self.storage = storageDetails['StorageObjects'][0] # self.storage.changeDirectory( 'lhcb/test/unit-test/Storage/RFIOStorage' ) def test_createUnitTestDir( self ): print('\n\n#########################################################' '################\n\n\t\t\tCreate Directory test\n') # destDir = self.storage.getCurrentURL( '' )['Value'] destDir = '/bla/' res = self.storage.createDirectory( destDir ) print(res) self.assertTrue(res['OK']) self.assertTrue( destDir in res['Value']['Successful'] ) self.assertTrue( res['Value']['Successful'][destDir] )
class StoragePlugInTestCase( unittest.TestCase ): """ Base test class. Defines all the method to test """ @mock.patch( 'DIRAC.Resources.Storage.StorageFactory.StorageFactory._getConfigStorageName', side_effect = mock_StorageFactory_getConfigStorageName ) @mock.patch( 'DIRAC.Resources.Storage.StorageFactory.StorageFactory._getConfigStorageOptions', side_effect = mock_StorageFactory_getConfigStorageOptions ) @mock.patch( 'DIRAC.Resources.Storage.StorageFactory.StorageFactory._getConfigStorageProtocols', side_effect = mock_StorageFactory_getConfigStorageProtocols ) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value = S_OK( True ) ) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value = None ) # Don't send accounting # @mock.patch( 'DIRAC.Resources.Storage.StorageFactory.StorageFactory._getCurrentURL', # side_effect = mock_StorageFactory_getCurrentURL_getCurrentURL ) def setUp( self, mk_getConfigStorageName, mk_getConfigStorageOptions, mk_getConfigStorageProtocols, mk_isLocalSE, mk_addAccountingOperation ): self.storage = StorageElementItem( 'FAKE' ) self.storage.vo = 'test' # self.storage = storageDetails['StorageObjects'][0] # self.storage.changeDirectory( 'lhcb/test/unit-test/Storage/RFIOStorage' ) def test_createUnitTestDir( self ): print '\n\n#########################################################################\n\n\t\t\tCreate Directory test\n' # destDir = self.storage.getCurrentURL( '' )['Value'] destDir = '/bla/' res = self.storage.createDirectory( destDir ) print res self.assertTrue(res['OK']) self.assertTrue( res['Value']['Successful'].has_key( destDir ) ) self.assertTrue( res['Value']['Successful'][destDir] )
def setUp(self, mk_getConfigStorageName, mk_getConfigStorageOptions, mk_getConfigStorageProtocols, mk_isLocalSE): self.se = StorageElementItem('FAKE') self.basePath = tempfile.mkdtemp(dir='/tmp') # Update the basePath of the plugin self.se.storages[0].basePath = self.basePath self.srcPath = tempfile.mkdtemp(dir='/tmp') self.destPath = tempfile.mkdtemp(dir='/tmp') self.existingFile = '/lhcb/file.txt' self.existingFileSize = 0 self.nonExistingFile = '/lhcb/nonExistingFile.txt' self.subDir = '/lhcb/subDir' self.subFile = os.path.join(self.subDir, 'subFile.txt') self.subFileSize = 0 self.FILES = [self.existingFile, self.nonExistingFile, self.subFile] self.DIRECTORIES = [self.subDir] self.ALL = self.FILES + self.DIRECTORIES with open( os.path.join(self.srcPath, self.existingFile.replace('/lhcb/', '')), 'w') as f: f.write("I put something in the file so that it has a size\n") self.existingFileSize = os.path.getsize( os.path.join(self.srcPath, self.existingFile.replace('/lhcb/', ''))) assert self.existingFileSize os.mkdir(os.path.join(self.srcPath, os.path.basename(self.subDir))) with open( os.path.join(self.srcPath, self.subFile.replace('/lhcb/', '')), 'w') as f: f.write("This one should have a size as well\n") self.subFileSize = os.path.getsize( os.path.join(self.srcPath, self.subFile.replace('/lhcb/', ''))) assert self.subFileSize
def _getProdLogs(): """get production log files from LogSE""" clip = _Params() clip.registerSwitch() Script.parseCommandLine() if not ( clip.logF or clip.logD or clip.prodid ): Script.showHelp() dexit(1) from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations ops = Operations() storageElementName = ops.getValue('/LogStorage/LogSE', 'LogSE') from DIRAC.Resources.Storage.StorageElement import StorageElementItem as StorageElement logSE = StorageElement(storageElementName) if clip.prodid and not ( clip.logD or clip.logF ): result = _getLogFolderFromID( clip ) if not result['OK']: gLogger.error( result['Message'] ) dexit(1) if clip.logD: if not clip.noPromptBeforeDL: res = promptUser('Are you sure you want to get ALL the files in this directory?') if not res['OK']: dexit() choice = res['Value'] if choice.lower()=='n': dexit(0) if isinstance(clip.logD, str): res = logSE.getDirectory(clip.logD, localPath=clip.outputdir) _printErrorReport(res) elif isinstance(clip.logD, list): for logdir in clip.logD: gLogger.notice('Getting log files from '+str(logdir)) res = logSE.getDirectory(logdir, localPath=clip.outputdir) _printErrorReport(res) if clip.logF: res = logSE.getFile(clip.logF, localPath = clip.outputdir) _printErrorReport(res)
def __init__(self): """Module initialization. """ super(UploadLogFile, self).__init__() self.version = __RCSID__ self.productionID = None self.jobID = None self.workflow_commons = None self.logFilePath = "" self.logLFNPath = "" self.logdir = '' self.logSE = StorageElement( self.ops.getValue('/LogStorage/LogSE', 'LogSE') ) self.root = gConfig.getValue('/LocalSite/Root', os.getcwd()) self.logSizeLimit = self.ops.getValue('/LogFiles/SizeLimit', 20 * 1024 * 1024) self.logExtensions = [] self.failoverSEs = gConfig.getValue('/Resources/StorageElementGroups/Tier1-Failover', []) self.experiment = 'CLIC' self.enable = True self.failoverTest = False #flag to put log files to failover by default self.jobID = ''
def setUp( self, mk_getConfigStorageName, mk_getConfigStorageOptions, mk_getConfigStorageProtocols, mk_isLocalSE, mk_addAccountingOperation ): self.se = StorageElementItem( 'FAKE' ) self.se.vo = 'test' self.basePath = tempfile.mkdtemp( dir = '/tmp' ) # Update the basePath of the plugin self.se.storages[0].basePath = self.basePath self.srcPath = tempfile.mkdtemp( dir = '/tmp' ) self.destPath = tempfile.mkdtemp( dir = '/tmp' ) self.existingFile = '/test/file.txt' self.existingFileSize = 0 self.nonExistingFile = '/test/nonExistingFile.txt' self.subDir = '/test/subDir' self.subFile = os.path.join( self.subDir, 'subFile.txt' ) self.subFileSize = 0 self.FILES = [self.existingFile, self.nonExistingFile, self.subFile] self.DIRECTORIES = [self.subDir] self.ALL = self.FILES + self.DIRECTORIES with open( os.path.join( self.srcPath, self.existingFile.replace( '/test/', '' ) ), 'w' ) as f: f.write( "I put something in the file so that it has a size\n" ) self.existingFileSize = os.path.getsize( os.path.join( self.srcPath, self.existingFile.replace( '/test/', '' ) ) ) assert self.existingFileSize os.mkdir( os.path.join( self.srcPath, os.path.basename( self.subDir ) ) ) with open( os.path.join( self.srcPath, self.subFile.replace( '/test/', '' ) ), 'w' ) as f: f.write( "This one should have a size as well\n" ) self.subFileSize = os.path.getsize( os.path.join( self.srcPath, self.subFile.replace( '/test/', '' ) ) ) assert self.subFileSize
def __init__(self): """Module initialization. """ super(UploadLogFile, self).__init__() self.version = __RCSID__ self.log = gLogger.getSubLogger('UploadLogFile') self.productionID = None self.jobID = None self.workflow_commons = None self.logFilePath = "" self.logLFNPath = "" self.logdir = '' self.logSE = StorageElement( self.ops.getValue('/LogStorage/LogSE', 'LogSE')) self.root = gConfig.getValue('/LocalSite/Root', os.getcwd()) self.logSizeLimit = self.ops.getValue('/LogFiles/SizeLimit', 20 * 1024 * 1024) self.logExtensions = [] self.failoverSEs = gConfig.getValue( '/Resources/StorageElementGroups/Tier1-Failover', []) self.experiment = 'CLIC' self.enable = True self.failoverTest = False #flag to put log files to failover by default self.jobID = ''
class TestBase( unittest.TestCase ): """ Base test class. Defines all the method to test """ @mock.patch( 'DIRAC.Resources.Storage.StorageFactory.StorageFactory._getConfigStorageName', side_effect = mock_StorageFactory_getConfigStorageName ) @mock.patch( 'DIRAC.Resources.Storage.StorageFactory.StorageFactory._getConfigStorageOptions', side_effect = mock_StorageFactory_getConfigStorageOptions ) @mock.patch( 'DIRAC.Resources.Storage.StorageFactory.StorageFactory._getConfigStorageProtocols', side_effect = mock_StorageFactory_getConfigStorageProtocols ) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value = S_OK( True ) ) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value = None ) # Don't send accounting def setUp( self, mk_getConfigStorageName, mk_getConfigStorageOptions, mk_getConfigStorageProtocols, mk_isLocalSE, mk_addAccountingOperation ): self.se = StorageElementItem( 'FAKE' ) self.se.vo = 'test' self.basePath = tempfile.mkdtemp( dir = '/tmp' ) # Update the basePath of the plugin self.se.storages[0].basePath = self.basePath self.srcPath = tempfile.mkdtemp( dir = '/tmp' ) self.destPath = tempfile.mkdtemp( dir = '/tmp' ) self.existingFile = '/test/file.txt' self.existingFileSize = 0 self.nonExistingFile = '/test/nonExistingFile.txt' self.subDir = '/test/subDir' self.subFile = os.path.join( self.subDir, 'subFile.txt' ) self.subFileSize = 0 self.FILES = [self.existingFile, self.nonExistingFile, self.subFile] self.DIRECTORIES = [self.subDir] self.ALL = self.FILES + self.DIRECTORIES with open( os.path.join( self.srcPath, self.existingFile.replace( '/test/', '' ) ), 'w' ) as f: f.write( "I put something in the file so that it has a size\n" ) self.existingFileSize = os.path.getsize( os.path.join( self.srcPath, self.existingFile.replace( '/test/', '' ) ) ) assert self.existingFileSize os.mkdir( os.path.join( self.srcPath, os.path.basename( self.subDir ) ) ) with open( os.path.join( self.srcPath, self.subFile.replace( '/test/', '' ) ), 'w' ) as f: f.write( "This one should have a size as well\n" ) self.subFileSize = os.path.getsize( os.path.join( self.srcPath, self.subFile.replace( '/test/', '' ) ) ) assert self.subFileSize def tearDown(self): shutil.rmtree( self.basePath ) shutil.rmtree( self.srcPath ) shutil.rmtree( self.destPath ) pass def walkAll( self ): for dirname in [self.basePath, self.destPath]: self.walkPath( dirname ) def walkPath(self, path): for root, dirs, files in os.walk( path ): print root print " dirs" for d in dirs: print " ", os.path.join( root, d ) print " files" for f in files: print " ", os.path.join( root, f ) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value = S_OK( True ) ) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value = None ) # Don't send accounting def test_01_getURL( self, mk_isLocalSE, mk_addAccounting ): """Testing getURL""" # Testing the getURL res = self.se.getURL( self.ALL ) self.assert_( res['OK'], res ) self.assert_( not res['Value']['Failed'], res['Value']['Failed'] ) self.assert_( len( res['Value']['Successful'] ) == len( self.ALL ) ) for lfn, url in res['Value']['Successful'].items(): self.assertEqual( url, self.basePath.rstrip( '/' ) + lfn ) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value = S_OK( True ) ) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value = None ) # Don't send accounting def test_02_FileTest( self, mk_isLocalSE, mk_addAccounting ): """Testing createDirectory""" # Putting the files def localPutFile( fn, size = 0 ): """If fn is '/test/fn.txt', it calls { '/test/fn.txt' : /tmp/generatedPath/fn.txt} """ transfDic = { fn : os.path.join( self.srcPath, fn.replace( '/test/', '' ) )} return self.se.putFile( transfDic, sourceSize = size ) # wrong size res = localPutFile( self.existingFile, size = -1 ) self.assert_( res['OK'], res ) self.assert_( self.existingFile in res['Value']['Failed'] ) self.assert_( 'not match' in res['Value']['Failed'][self.existingFile], res ) self.assert_( not os.path.exists( self.basePath + self.existingFile ) ) # Correct size res = localPutFile( self.existingFile, size = self.existingFileSize ) self.assert_( res['OK'], res ) self.assert_( self.existingFile in res['Value']['Successful'], res ) self.assert_( os.path.exists( self.basePath + self.existingFile ) ) # No size res = localPutFile( self.existingFile ) self.assert_( res['OK'], res ) self.assert_( self.existingFile in res['Value']['Successful'], res ) self.assert_( os.path.exists( self.basePath + self.existingFile ) ) # No existing source file res = localPutFile( self.nonExistingFile ) self.assert_( res['OK'], res ) self.assert_( self.nonExistingFile in res['Value']['Failed'], res ) self.assert_( os.strerror( errno.ENOENT ) in res['Value']['Failed'][self.nonExistingFile], res ) # sub file res = localPutFile( self.subFile ) self.assert_( res['OK'], res ) self.assert_( self.subFile in res['Value']['Successful'], res ) self.assert_( os.path.exists( self.basePath + self.subFile ) ) # Directory res = localPutFile( self.subDir ) self.assert_( res['OK'], res ) self.assert_( self.subDir in res['Value']['Failed'] ) self.assert_( os.strerror( errno.EISDIR ) in res['Value']['Failed'][self.subDir], res ) res = self.se.exists( self.FILES ) self.assert_( res['OK'], res ) self.assert_( not res['Value']['Failed'], res ) self.assert_( res['Value']['Successful'][self.existingFile], res ) self.assert_( not res['Value']['Successful'][self.nonExistingFile], res ) res = self.se.getFileSize( self.ALL ) self.assert_( res['OK'], res ) self.assertEqual( res['Value']['Successful'][self.existingFile], self.existingFileSize ) self.assert_( os.strerror( errno.ENOENT ) in res['Value']['Failed'][self.nonExistingFile], res ) self.assert_( os.strerror( errno.EISDIR ) in res['Value']['Failed'][self.subDir], res ) res = self.se.getFileMetadata( self.ALL ) self.assert_( res['OK'], res ) self.assert_( self.existingFile in res['Value']['Successful'] ) self.assert_( os.strerror( errno.ENOENT ) in res['Value']['Failed'][self.nonExistingFile], res ) self.assert_( os.strerror( errno.EISDIR ) in res['Value']['Failed'][self.subDir], res ) res = self.se.isFile( self.ALL ) self.assert_( res['OK'], res ) self.assert_( res['Value']['Successful'][self.existingFile], res ) self.assert_( not res['Value']['Successful'][self.subDir], res ) self.assert_( os.strerror( errno.ENOENT ) in res['Value']['Failed'][self.nonExistingFile], res ) res = self.se.getFile( self.ALL, localPath = self.destPath ) self.assert_( res['OK'], res ) self.assertEqual( res['Value']['Successful'][self.existingFile], self.existingFileSize ) self.assert_( os.path.exists( os.path.join( self.destPath, os.path.basename( self.existingFile ) ) ) ) self.assertEqual( res['Value']['Successful'][self.subFile], self.subFileSize ) self.assert_( os.path.exists( os.path.join( self.destPath, os.path.basename( self.subFile ) ) ) ) self.assert_( os.strerror( errno.ENOENT ) in res['Value']['Failed'][self.nonExistingFile], res ) self.assert_( os.strerror( errno.EISDIR ) in res['Value']['Failed'][self.subDir], res ) res = self.se.removeFile( self.ALL ) self.assert_( res['OK'], res ) self.assert_( res['Value']['Successful'][self.existingFile] ) self.assert_( not os.path.exists( self.basePath + self.existingFile ) ) self.assert_( res['Value']['Successful'][self.subFile] ) self.assert_( not os.path.exists( self.basePath + self.subFile ) ) self.assert_( res['Value']['Successful'][self.nonExistingFile] ) self.assert_( os.strerror( errno.EISDIR ) in res['Value']['Failed'][self.subDir] ) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value = S_OK( True ) ) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value = None ) # Don't send accounting def test_03_createDirectory( self, mk_isLocalSE, mk_addAccounting ): """Testing creating directories""" res = self.se.createDirectory( self.subDir ) self.assert_( res['OK'], res ) self.assert_( self.subDir in res['Value']['Successful'] ) self.assert_( os.path.exists( self.basePath + self.subDir ) ) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value = S_OK( True ) ) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value = None ) # Don't send accounting def test_04_putDirectory( self, mk_isLocalSE, mk_addAccounting ): """Testing putDirectory""" nonExistingDir = '/test/forsuredoesnotexist' localdirs = ['/test', nonExistingDir] # Correct size res = self.se.putDirectory( { '/test' : self.srcPath} ) self.assert_( res['OK'], res ) self.assert_( '/test' in res['Value']['Successful'], res ) self.assertEqual( res['Value']['Successful']['/test'], {'Files': 2, 'Size': self.existingFileSize + self.subFileSize} ) self.assert_( os.path.exists( self.basePath + '/test' ) ) self.assert_( os.path.exists( self.basePath + self.existingFile ) ) self.assert_( os.path.exists( self.basePath + self.subFile ) ) # No existing source directory res = self.se.putDirectory( { '/test' : nonExistingDir} ) self.assert_( res['OK'], res ) self.assert_( '/test' in res['Value']['Failed'], res ) self.assertEqual( res['Value']['Failed']['/test'], {'Files': 0, 'Size': 0} ) # sub file res = self.se.putDirectory( { '/test' : self.existingFile} ) self.assert_( res['OK'], res ) self.assert_( '/test' in res['Value']['Failed'], res ) self.assertEqual( res['Value']['Failed']['/test'], {'Files': 0, 'Size': 0} ) res = self.se.exists( self.DIRECTORIES + localdirs ) self.assert_( res['OK'], res ) self.assert_( not res['Value']['Failed'], res ) self.assert_( res['Value']['Successful'][self.subDir], res ) self.assert_( not res['Value']['Successful'][nonExistingDir], res ) res = self.se.getDirectorySize( self.ALL + localdirs ) self.assert_( res['OK'], res ) self.assertEqual( res['Value']['Successful'][self.subDir], { 'Files' : 1, 'Size' : self.subFileSize, 'SubDirs' : 0 } ) self.assertEqual( res['Value']['Successful']['/test'], { 'Files' : 1, 'Size' : self.existingFileSize, 'SubDirs' : 1 } ) self.assert_( os.strerror( errno.ENOENT ) in res['Value']['Failed'][self.nonExistingFile], res ) self.assert_( os.strerror( errno.ENOTDIR ) in res['Value']['Failed'][self.existingFile], res ) self.assert_( os.strerror( errno.ENOENT ) in res['Value']['Failed'][nonExistingDir], res ) res = self.se.getDirectoryMetadata( self.ALL + localdirs ) self.assert_( res['OK'], res ) self.assert_( self.subDir in res['Value']['Successful'] ) self.assert_( os.strerror( errno.ENOENT ) in res['Value']['Failed'][self.nonExistingFile], res ) self.assert_( os.strerror( errno.ENOENT ) in res['Value']['Failed'][nonExistingDir], res ) self.assert_( os.strerror( errno.ENOTDIR ) in res['Value']['Failed'][self.existingFile], res ) res = self.se.isDirectory( self.ALL + localdirs ) self.assert_( res['OK'], res ) self.assert_( not res['Value']['Successful'][self.existingFile] ) self.assert_( res['Value']['Successful'][self.subDir], res ) self.assert_( os.strerror( errno.ENOENT ) in res['Value']['Failed'][self.nonExistingFile], res ) self.assert_( os.strerror( errno.ENOENT ) in res['Value']['Failed'][nonExistingDir], res ) res = self.se.listDirectory( self.ALL + localdirs ) self.assert_( res['OK'], res ) self.assertEqual( res['Value']['Successful'][self.subDir], {'Files': [self.subFile], 'SubDirs': []} ) self.assertEqual( res['Value']['Successful']['/test'], {'Files': [self.existingFile], 'SubDirs': [self.subDir]} ) self.assert_( os.strerror( errno.ENOENT ) in res['Value']['Failed'][self.nonExistingFile], res ) self.assert_( os.strerror( errno.ENOTDIR ) in res['Value']['Failed'][self.existingFile], res ) self.assert_( os.strerror( errno.ENOENT ) in res['Value']['Failed'][nonExistingDir], res ) res = self.se.getDirectory( self.ALL + localdirs, localPath = self.destPath ) self.assert_( res['OK'], res ) self.assertEqual( res['Value']['Successful']['/test'], {'Files' : 2, 'Size' : self.existingFileSize + self.subFileSize} ) self.assert_( os.path.exists( self.destPath + self.existingFile ) ) self.assert_( os.path.exists( self.destPath + self.subFile ) ) self.assertEqual( res['Value']['Successful'][self.subDir], {'Files' : 1, 'Size' : self.subFileSize} ) self.assert_( os.path.exists( self.destPath + self.subFile.replace( '/test', '' ) ) ) self.assertEqual( res['Value']['Failed'][self.nonExistingFile], {'Files': 0, 'Size': 0} ) self.assertEqual( res['Value']['Failed'][self.existingFile], {'Files': 0, 'Size': 0} ) self.assertEqual( res['Value']['Failed'][nonExistingDir], {'Files': 0, 'Size': 0} ) res = self.se.removeDirectory( nonExistingDir, recursive = False ) self.assert_( res['OK'], res ) self.assertEqual( res['Value']['Successful'][nonExistingDir], True ) res = self.se.removeDirectory( nonExistingDir, recursive = True ) self.assert_( res['OK'], res ) self.assertEqual( res['Value']['Failed'][nonExistingDir], {'FilesRemoved':0, 'SizeRemoved':0} ) res = self.se.removeDirectory( self.nonExistingFile, recursive = False ) self.assert_( res['OK'], res ) self.assertEqual( res['Value']['Successful'][self.nonExistingFile], True ) res = self.se.removeDirectory( self.nonExistingFile, recursive = True ) self.assert_( res['OK'], res ) self.assertEqual( res['Value']['Failed'][self.nonExistingFile], {'FilesRemoved':0, 'SizeRemoved':0} ) res = self.se.removeDirectory( self.existingFile, recursive = False ) self.assert_( res['OK'], res ) self.assert_( os.strerror( errno.ENOTDIR ) in res['Value']['Failed'][self.existingFile], res ) res = self.se.removeDirectory( self.existingFile, recursive = True ) self.assert_( res['OK'], res ) self.assertEqual( res['Value']['Failed'][self.existingFile], {'FilesRemoved':0, 'SizeRemoved':0} ) res = self.se.removeDirectory( '/test', recursive = False ) self.assert_( res['OK'], res ) self.assertEqual( res['Value']['Successful']['/test'], True ) self.assert_( not os.path.exists( self.basePath + self.existingFile ) ) self.assert_( os.path.exists( self.basePath + self.subFile ) ) res = self.se.removeDirectory( '/test', recursive = True ) self.assert_( res['OK'], res ) self.assertEqual( res['Value']['Successful']['/test'], {'FilesRemoved':1, 'SizeRemoved':self.subFileSize} ) self.assert_( not os.path.exists( self.basePath + '/test' ) )
class TestBase(unittest.TestCase): """ Base test class. Defines all the method to test """ @mock.patch( 'DIRAC.Resources.Storage.StorageFactory.StorageFactory._getConfigStorageName', side_effect=mock_StorageFactory_getConfigStorageName) @mock.patch( 'DIRAC.Resources.Storage.StorageFactory.StorageFactory._getConfigStorageOptions', side_effect=mock_StorageFactory_getConfigStorageOptions) @mock.patch( 'DIRAC.Resources.Storage.StorageFactory.StorageFactory._getConfigStorageProtocols', side_effect=mock_StorageFactory_getConfigStorageProtocols) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value=S_OK(True)) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value=None) # Don't send accounting def setUp(self, mk_getConfigStorageName, mk_getConfigStorageOptions, mk_getConfigStorageProtocols, mk_isLocalSE, mk_addAccountingOperation): self.se = StorageElementItem('FAKE') self.se.vo = 'test' self.basePath = tempfile.mkdtemp(dir='/tmp') # Update the basePath of the plugin self.se.storages[0].basePath = self.basePath self.srcPath = tempfile.mkdtemp(dir='/tmp') self.destPath = tempfile.mkdtemp(dir='/tmp') self.existingFile = '/test/file.txt' self.existingFileSize = 0 self.nonExistingFile = '/test/nonExistingFile.txt' self.subDir = '/test/subDir' self.subFile = os.path.join(self.subDir, 'subFile.txt') self.subFileSize = 0 self.FILES = [self.existingFile, self.nonExistingFile, self.subFile] self.DIRECTORIES = [self.subDir] self.ALL = self.FILES + self.DIRECTORIES with open( os.path.join(self.srcPath, self.existingFile.replace('/test/', '')), 'w') as f: f.write("I put something in the file so that it has a size\n") self.existingFileSize = os.path.getsize( os.path.join(self.srcPath, self.existingFile.replace('/test/', ''))) assert self.existingFileSize os.mkdir(os.path.join(self.srcPath, os.path.basename(self.subDir))) with open( os.path.join(self.srcPath, self.subFile.replace('/test/', '')), 'w') as f: f.write("This one should have a size as well\n") self.subFileSize = os.path.getsize( os.path.join(self.srcPath, self.subFile.replace('/test/', ''))) assert self.subFileSize def tearDown(self): shutil.rmtree(self.basePath) shutil.rmtree(self.srcPath) shutil.rmtree(self.destPath) pass def walkAll(self): for dirname in [self.basePath, self.destPath]: self.walkPath(dirname) def walkPath(self, path): for root, dirs, files in os.walk(path): print(root) print(" dirs") for d in dirs: print(" ", os.path.join(root, d)) print(" files") for f in files: print(" ", os.path.join(root, f)) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value=S_OK(True)) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value=None) # Don't send accounting def test_01_getURL(self, mk_isLocalSE, mk_addAccounting): """Testing getURL""" # Testing the getURL res = self.se.getURL(self.ALL) self.assertTrue(res['OK'], res) self.assertTrue(not res['Value']['Failed'], res['Value']['Failed']) self.assertTrue(len(res['Value']['Successful']) == len(self.ALL)) for lfn, url in res['Value']['Successful'].items(): self.assertEqual(url, self.basePath.rstrip('/') + lfn) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value=S_OK(True)) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value=None) # Don't send accounting def test_02_FileTest(self, mk_isLocalSE, mk_addAccounting): """Testing createDirectory""" # Putting the files def localPutFile(fn, size=0): """If fn is '/test/fn.txt', it calls { '/test/fn.txt' : /tmp/generatedPath/fn.txt} """ transfDic = { fn: os.path.join(self.srcPath, fn.replace('/test/', '')) } return self.se.putFile(transfDic, sourceSize=size) # wrong size res = localPutFile(self.existingFile, size=-1) self.assertTrue(res['OK'], res) self.assertTrue(self.existingFile in res['Value']['Failed'], res) self.assertTrue( 'not match' in res['Value']['Failed'][self.existingFile], res) self.assertTrue(not os.path.exists(self.basePath + self.existingFile)) # Correct size res = localPutFile(self.existingFile, size=self.existingFileSize) self.assertTrue(res['OK'], res) self.assertTrue(self.existingFile in res['Value']['Successful'], res) self.assertTrue(os.path.exists(self.basePath + self.existingFile)) # No size res = localPutFile(self.existingFile) self.assertTrue(res['OK'], res) self.assertTrue(self.existingFile in res['Value']['Successful'], res) self.assertTrue(os.path.exists(self.basePath + self.existingFile)) # No existing source file res = localPutFile(self.nonExistingFile) self.assertTrue(res['OK'], res) self.assertTrue(self.nonExistingFile in res['Value']['Failed'], res) self.assertTrue( os.strerror(errno.ENOENT) in res['Value']['Failed'][self.nonExistingFile], res) # sub file res = localPutFile(self.subFile) self.assertTrue(res['OK'], res) self.assertTrue(self.subFile in res['Value']['Successful'], res) self.assertTrue(os.path.exists(self.basePath + self.subFile)) # Directory res = localPutFile(self.subDir) self.assertTrue(res['OK'], res) self.assertTrue(self.subDir in res['Value']['Failed']) self.assertTrue( os.strerror(errno.EISDIR) in res['Value']['Failed'][self.subDir], res) res = self.se.exists(self.FILES) self.assertTrue(res['OK'], res) self.assertTrue(not res['Value']['Failed'], res) self.assertTrue(res['Value']['Successful'][self.existingFile], res) self.assertTrue(not res['Value']['Successful'][self.nonExistingFile], res) res = self.se.getFileSize(self.ALL) self.assertTrue(res['OK'], res) self.assertEqual(res['Value']['Successful'][self.existingFile], self.existingFileSize) self.assertTrue( os.strerror(errno.ENOENT) in res['Value']['Failed'][self.nonExistingFile], res) self.assertTrue( os.strerror(errno.EISDIR) in res['Value']['Failed'][self.subDir], res) res = self.se.getFileMetadata(self.ALL) self.assertTrue(res['OK'], res) self.assertTrue(self.existingFile in res['Value']['Successful']) self.assertTrue( os.strerror(errno.ENOENT) in res['Value']['Failed'][self.nonExistingFile], res) self.assertTrue( os.strerror(errno.EISDIR) in res['Value']['Failed'][self.subDir], res) res = self.se.isFile(self.ALL) self.assertTrue(res['OK'], res) self.assertTrue(res['Value']['Successful'][self.existingFile], res) self.assertTrue(not res['Value']['Successful'][self.subDir], res) self.assertTrue( os.strerror(errno.ENOENT) in res['Value']['Failed'][self.nonExistingFile], res) res = self.se.getFile(self.ALL, localPath=self.destPath) self.assertTrue(res['OK'], res) self.assertEqual(res['Value']['Successful'][self.existingFile], self.existingFileSize) self.assertTrue( os.path.exists( os.path.join(self.destPath, os.path.basename(self.existingFile)))) self.assertEqual(res['Value']['Successful'][self.subFile], self.subFileSize) self.assertTrue( os.path.exists( os.path.join(self.destPath, os.path.basename(self.subFile)))) self.assertTrue( os.strerror(errno.ENOENT) in res['Value']['Failed'][self.nonExistingFile], res) self.assertTrue( os.strerror(errno.EISDIR) in res['Value']['Failed'][self.subDir], res) res = self.se.removeFile(self.ALL) self.assertTrue(res['OK'], res) self.assertTrue(res['Value']['Successful'][self.existingFile]) self.assertTrue(not os.path.exists(self.basePath + self.existingFile)) self.assertTrue(res['Value']['Successful'][self.subFile]) self.assertTrue(not os.path.exists(self.basePath + self.subFile)) self.assertTrue(res['Value']['Successful'][self.nonExistingFile]) self.assertTrue( os.strerror(errno.EISDIR) in res['Value']['Failed'][self.subDir]) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value=S_OK(True)) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value=None) # Don't send accounting def test_03_createDirectory(self, mk_isLocalSE, mk_addAccounting): """Testing creating directories""" res = self.se.createDirectory(self.subDir) self.assertTrue(res['OK'], res) self.assertTrue(self.subDir in res['Value']['Successful']) self.assertTrue(os.path.exists(self.basePath + self.subDir)) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value=S_OK(True)) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value=None) # Don't send accounting def test_04_putDirectory(self, mk_isLocalSE, mk_addAccounting): """Testing putDirectory""" nonExistingDir = '/test/forsuredoesnotexist' localdirs = ['/test', nonExistingDir] # Correct size res = self.se.putDirectory({'/test': self.srcPath}) self.assertTrue(res['OK'], res) self.assertTrue('/test' in res['Value']['Successful'], res) self.assertEqual(res['Value']['Successful']['/test'], { 'Files': 2, 'Size': self.existingFileSize + self.subFileSize }) self.assertTrue(os.path.exists(self.basePath + '/test')) self.assertTrue(os.path.exists(self.basePath + self.existingFile)) self.assertTrue(os.path.exists(self.basePath + self.subFile)) # No existing source directory res = self.se.putDirectory({'/test': nonExistingDir}) self.assertTrue(res['OK'], res) self.assertTrue('/test' in res['Value']['Failed'], res) self.assertEqual(res['Value']['Failed']['/test'], { 'Files': 0, 'Size': 0 }) # sub file res = self.se.putDirectory({'/test': self.existingFile}) self.assertTrue(res['OK'], res) self.assertTrue('/test' in res['Value']['Failed'], res) self.assertEqual(res['Value']['Failed']['/test'], { 'Files': 0, 'Size': 0 }) res = self.se.exists(self.DIRECTORIES + localdirs) self.assertTrue(res['OK'], res) self.assertTrue(not res['Value']['Failed'], res) self.assertTrue(res['Value']['Successful'][self.subDir], res) self.assertTrue(not res['Value']['Successful'][nonExistingDir], res) res = self.se.getDirectorySize(self.ALL + localdirs) self.assertTrue(res['OK'], res) self.assertEqual(res['Value']['Successful'][self.subDir], { 'Files': 1, 'Size': self.subFileSize, 'SubDirs': 0 }) self.assertEqual(res['Value']['Successful']['/test'], { 'Files': 1, 'Size': self.existingFileSize, 'SubDirs': 1 }) self.assertTrue( os.strerror(errno.ENOENT) in res['Value']['Failed'][self.nonExistingFile], res) self.assertTrue( os.strerror(errno.ENOTDIR) in res['Value']['Failed'][self.existingFile], res) self.assertTrue( os.strerror(errno.ENOENT) in res['Value']['Failed'][nonExistingDir], res) res = self.se.getDirectoryMetadata(self.ALL + localdirs) self.assertTrue(res['OK'], res) self.assertTrue(self.subDir in res['Value']['Successful']) self.assertTrue( os.strerror(errno.ENOENT) in res['Value']['Failed'][self.nonExistingFile], res) self.assertTrue( os.strerror(errno.ENOENT) in res['Value']['Failed'][nonExistingDir], res) self.assertTrue( os.strerror(errno.ENOTDIR) in res['Value']['Failed'][self.existingFile], res) res = self.se.isDirectory(self.ALL + localdirs) self.assertTrue(res['OK'], res) self.assertTrue(not res['Value']['Successful'][self.existingFile]) self.assertTrue(res['Value']['Successful'][self.subDir], res) self.assertTrue( os.strerror(errno.ENOENT) in res['Value']['Failed'][self.nonExistingFile], res) self.assertTrue( os.strerror(errno.ENOENT) in res['Value']['Failed'][nonExistingDir], res) res = self.se.listDirectory(self.ALL + localdirs) self.assertTrue(res['OK'], res) self.assertEqual(res['Value']['Successful'][self.subDir], { 'Files': [self.subFile], 'SubDirs': [] }) self.assertEqual(res['Value']['Successful']['/test'], { 'Files': [self.existingFile], 'SubDirs': [self.subDir] }) self.assertTrue( os.strerror(errno.ENOENT) in res['Value']['Failed'][self.nonExistingFile], res) self.assertTrue( os.strerror(errno.ENOTDIR) in res['Value']['Failed'][self.existingFile], res) self.assertTrue( os.strerror(errno.ENOENT) in res['Value']['Failed'][nonExistingDir], res) res = self.se.getDirectory(self.ALL + localdirs, localPath=self.destPath) self.assertTrue(res['OK'], res) self.assertEqual(res['Value']['Successful']['/test'], { 'Files': 2, 'Size': self.existingFileSize + self.subFileSize }) self.assertTrue(os.path.exists(self.destPath + self.existingFile)) self.assertTrue(os.path.exists(self.destPath + self.subFile)) self.assertEqual(res['Value']['Successful'][self.subDir], { 'Files': 1, 'Size': self.subFileSize }) self.assertTrue( os.path.exists(self.destPath + self.subFile.replace('/test', ''))) self.assertEqual(res['Value']['Failed'][self.nonExistingFile], { 'Files': 0, 'Size': 0 }) self.assertEqual(res['Value']['Failed'][self.existingFile], { 'Files': 0, 'Size': 0 }) self.assertEqual(res['Value']['Failed'][nonExistingDir], { 'Files': 0, 'Size': 0 }) res = self.se.removeDirectory(nonExistingDir, recursive=False) self.assertTrue(res['OK'], res) self.assertEqual(res['Value']['Successful'][nonExistingDir], True) res = self.se.removeDirectory(nonExistingDir, recursive=True) self.assertTrue(res['OK'], res) self.assertEqual(res['Value']['Failed'][nonExistingDir], { 'FilesRemoved': 0, 'SizeRemoved': 0 }) res = self.se.removeDirectory(self.nonExistingFile, recursive=False) self.assertTrue(res['OK'], res) self.assertEqual(res['Value']['Successful'][self.nonExistingFile], True) res = self.se.removeDirectory(self.nonExistingFile, recursive=True) self.assertTrue(res['OK'], res) self.assertEqual(res['Value']['Failed'][self.nonExistingFile], { 'FilesRemoved': 0, 'SizeRemoved': 0 }) res = self.se.removeDirectory(self.existingFile, recursive=False) self.assertTrue(res['OK'], res) self.assertTrue( os.strerror(errno.ENOTDIR) in res['Value']['Failed'][self.existingFile], res) res = self.se.removeDirectory(self.existingFile, recursive=True) self.assertTrue(res['OK'], res) self.assertEqual(res['Value']['Failed'][self.existingFile], { 'FilesRemoved': 0, 'SizeRemoved': 0 }) res = self.se.removeDirectory('/test', recursive=False) self.assertTrue(res['OK'], res) self.assertEqual(res['Value']['Successful']['/test'], True) self.assertTrue(not os.path.exists(self.basePath + self.existingFile)) self.assertTrue(os.path.exists(self.basePath + self.subFile)) res = self.se.removeDirectory('/test', recursive=True) self.assertTrue(res['OK'], res) self.assertEqual(res['Value']['Successful']['/test'], { 'FilesRemoved': 1, 'SizeRemoved': self.subFileSize }) self.assertTrue(not os.path.exists(self.basePath + '/test'))
class TestBase(unittest.TestCase): """Base test class. Defines all the method to test""" @mock.patch( "DIRAC.Resources.Storage.StorageFactory.StorageFactory._getConfigStorageName", side_effect=mock_StorageFactory_getConfigStorageName, ) @mock.patch( "DIRAC.Resources.Storage.StorageFactory.StorageFactory._getConfigStorageOptions", side_effect=mock_StorageFactory_getConfigStorageOptions, ) @mock.patch( "DIRAC.Resources.Storage.StorageFactory.StorageFactory._getConfigStorageProtocols", side_effect=mock_StorageFactory_getConfigStorageProtocols, ) @mock.patch( "DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE", return_value=S_OK(True), ) # Pretend it's local @mock.patch( "DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation", return_value=None) # Don't send accounting def setUp( self, mk_getConfigStorageName, mk_getConfigStorageOptions, mk_getConfigStorageProtocols, mk_isLocalSE, mk_addAccountingOperation, ): self.se = StorageElementItem("FAKE") self.se.vo = "test" self.basePath = tempfile.mkdtemp(dir="/tmp") # Update the basePath of the plugin self.se.storages[0].basePath = self.basePath self.srcPath = tempfile.mkdtemp(dir="/tmp") self.destPath = tempfile.mkdtemp(dir="/tmp") self.existingFile = "/test/file.txt" self.existingFileSize = 0 self.nonExistingFile = "/test/nonExistingFile.txt" self.subDir = "/test/subDir" self.subFile = os.path.join(self.subDir, "subFile.txt") self.subFileSize = 0 self.FILES = [self.existingFile, self.nonExistingFile, self.subFile] self.DIRECTORIES = [self.subDir] self.ALL = self.FILES + self.DIRECTORIES with open( os.path.join(self.srcPath, self.existingFile.replace("/test/", "")), "w") as f: f.write("I put something in the file so that it has a size\n") self.existingFileSize = os.path.getsize( os.path.join(self.srcPath, self.existingFile.replace("/test/", ""))) assert self.existingFileSize os.mkdir(os.path.join(self.srcPath, os.path.basename(self.subDir))) with open( os.path.join(self.srcPath, self.subFile.replace("/test/", "")), "w") as f: f.write("This one should have a size as well\n") self.subFileSize = os.path.getsize( os.path.join(self.srcPath, self.subFile.replace("/test/", ""))) assert self.subFileSize def tearDown(self): shutil.rmtree(self.basePath) shutil.rmtree(self.srcPath) shutil.rmtree(self.destPath) pass def walkAll(self): for dirname in [self.basePath, self.destPath]: self.walkPath(dirname) def walkPath(self, path): for root, dirs, files in os.walk(path): print(root) print(" dirs") for d in dirs: print(" ", os.path.join(root, d)) print(" files") for f in files: print(" ", os.path.join(root, f)) @mock.patch( "DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE", return_value=S_OK(True), ) # Pretend it's local @mock.patch( "DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation", return_value=None) # Don't send accounting def test_01_getURL(self, mk_isLocalSE, mk_addAccounting): """Testing getURL""" # Testing the getURL res = self.se.getURL(self.ALL) self.assertTrue(res["OK"], res) self.assertTrue(not res["Value"]["Failed"], res["Value"]["Failed"]) self.assertTrue(len(res["Value"]["Successful"]) == len(self.ALL)) for lfn, url in res["Value"]["Successful"].items(): self.assertEqual(url, self.basePath.rstrip("/") + lfn) @mock.patch( "DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE", return_value=S_OK(True), ) # Pretend it's local @mock.patch( "DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation", return_value=None) # Don't send accounting def test_02_FileTest(self, mk_isLocalSE, mk_addAccounting): """Testing createDirectory""" # Putting the files def localPutFile(fn, size=0): """If fn is '/test/fn.txt', it calls { '/test/fn.txt' : /tmp/generatedPath/fn.txt} """ transfDic = { fn: os.path.join(self.srcPath, fn.replace("/test/", "")) } return self.se.putFile(transfDic, sourceSize=size) # wrong size res = localPutFile(self.existingFile, size=-1) self.assertTrue(res["OK"], res) self.assertTrue(self.existingFile in res["Value"]["Failed"], res) self.assertTrue( "not match" in res["Value"]["Failed"][self.existingFile], res) self.assertTrue(not os.path.exists(self.basePath + self.existingFile)) # Correct size res = localPutFile(self.existingFile, size=self.existingFileSize) self.assertTrue(res["OK"], res) self.assertTrue(self.existingFile in res["Value"]["Successful"], res) self.assertTrue(os.path.exists(self.basePath + self.existingFile)) # No size res = localPutFile(self.existingFile) self.assertTrue(res["OK"], res) self.assertTrue(self.existingFile in res["Value"]["Successful"], res) self.assertTrue(os.path.exists(self.basePath + self.existingFile)) # No existing source file res = localPutFile(self.nonExistingFile) self.assertTrue(res["OK"], res) self.assertTrue(self.nonExistingFile in res["Value"]["Failed"], res) self.assertTrue( os.strerror(errno.ENOENT) in res["Value"]["Failed"][self.nonExistingFile], res) # sub file res = localPutFile(self.subFile) self.assertTrue(res["OK"], res) self.assertTrue(self.subFile in res["Value"]["Successful"], res) self.assertTrue(os.path.exists(self.basePath + self.subFile)) # Directory res = localPutFile(self.subDir) self.assertTrue(res["OK"], res) self.assertTrue(self.subDir in res["Value"]["Failed"]) self.assertTrue( os.strerror(errno.EISDIR) in res["Value"]["Failed"][self.subDir] or # Python 3.9.7+ improved the Exception that is raised "Directory does not exist" in res["Value"]["Failed"][self.subDir], res, ) res = self.se.exists(self.FILES) self.assertTrue(res["OK"], res) self.assertTrue(not res["Value"]["Failed"], res) self.assertTrue(res["Value"]["Successful"][self.existingFile], res) self.assertTrue(not res["Value"]["Successful"][self.nonExistingFile], res) res = self.se.getFileSize(self.ALL) self.assertTrue(res["OK"], res) self.assertEqual(res["Value"]["Successful"][self.existingFile], self.existingFileSize) self.assertTrue( os.strerror(errno.ENOENT) in res["Value"]["Failed"][self.nonExistingFile], res) self.assertTrue( os.strerror(errno.EISDIR) in res["Value"]["Failed"][self.subDir], res) res = self.se.getFileMetadata(self.ALL) self.assertTrue(res["OK"], res) self.assertTrue(self.existingFile in res["Value"]["Successful"]) self.assertTrue( os.strerror(errno.ENOENT) in res["Value"]["Failed"][self.nonExistingFile], res) self.assertTrue( os.strerror(errno.EISDIR) in res["Value"]["Failed"][self.subDir], res) res = self.se.isFile(self.ALL) self.assertTrue(res["OK"], res) self.assertTrue(res["Value"]["Successful"][self.existingFile], res) self.assertTrue(not res["Value"]["Successful"][self.subDir], res) self.assertTrue( os.strerror(errno.ENOENT) in res["Value"]["Failed"][self.nonExistingFile], res) res = self.se.getFile(self.ALL, localPath=self.destPath) self.assertTrue(res["OK"], res) self.assertEqual(res["Value"]["Successful"][self.existingFile], self.existingFileSize) self.assertTrue( os.path.exists( os.path.join(self.destPath, os.path.basename(self.existingFile)))) self.assertEqual(res["Value"]["Successful"][self.subFile], self.subFileSize) self.assertTrue( os.path.exists( os.path.join(self.destPath, os.path.basename(self.subFile)))) self.assertTrue( os.strerror(errno.ENOENT) in res["Value"]["Failed"][self.nonExistingFile], res) self.assertTrue( os.strerror(errno.EISDIR) in res["Value"]["Failed"][self.subDir] or # Python 3.9.7+ improved the Exception that is raised "Directory does not exist" in res["Value"]["Failed"][self.subDir], res, ) res = self.se.removeFile(self.ALL) self.assertTrue(res["OK"], res) self.assertTrue(res["Value"]["Successful"][self.existingFile]) self.assertTrue(not os.path.exists(self.basePath + self.existingFile)) self.assertTrue(res["Value"]["Successful"][self.subFile]) self.assertTrue(not os.path.exists(self.basePath + self.subFile)) self.assertTrue(res["Value"]["Successful"][self.nonExistingFile]) self.assertTrue( os.strerror(errno.EISDIR) in res["Value"]["Failed"][self.subDir]) @mock.patch( "DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE", return_value=S_OK(True), ) # Pretend it's local @mock.patch( "DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation", return_value=None) # Don't send accounting def test_03_createDirectory(self, mk_isLocalSE, mk_addAccounting): """Testing creating directories""" res = self.se.createDirectory(self.subDir) self.assertTrue(res["OK"], res) self.assertTrue(self.subDir in res["Value"]["Successful"]) self.assertTrue(os.path.exists(self.basePath + self.subDir)) @mock.patch( "DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE", return_value=S_OK(True), ) # Pretend it's local @mock.patch( "DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation", return_value=None) # Don't send accounting def test_04_putDirectory(self, mk_isLocalSE, mk_addAccounting): """Testing putDirectory""" nonExistingDir = "/test/forsuredoesnotexist" localdirs = ["/test", nonExistingDir] # Correct size res = self.se.putDirectory({"/test": self.srcPath}) self.assertTrue(res["OK"], res) self.assertTrue("/test" in res["Value"]["Successful"], res) self.assertEqual(res["Value"]["Successful"]["/test"], { "Files": 2, "Size": self.existingFileSize + self.subFileSize }) self.assertTrue(os.path.exists(self.basePath + "/test")) self.assertTrue(os.path.exists(self.basePath + self.existingFile)) self.assertTrue(os.path.exists(self.basePath + self.subFile)) # No existing source directory res = self.se.putDirectory({"/test": nonExistingDir}) self.assertTrue(res["OK"], res) self.assertTrue("/test" in res["Value"]["Failed"], res) self.assertEqual(res["Value"]["Failed"]["/test"], { "Files": 0, "Size": 0 }) # sub file res = self.se.putDirectory({"/test": self.existingFile}) self.assertTrue(res["OK"], res) self.assertTrue("/test" in res["Value"]["Failed"], res) self.assertEqual(res["Value"]["Failed"]["/test"], { "Files": 0, "Size": 0 }) res = self.se.exists(self.DIRECTORIES + localdirs) self.assertTrue(res["OK"], res) self.assertTrue(not res["Value"]["Failed"], res) self.assertTrue(res["Value"]["Successful"][self.subDir], res) self.assertTrue(not res["Value"]["Successful"][nonExistingDir], res) res = self.se.getDirectorySize(self.ALL + localdirs) self.assertTrue(res["OK"], res) self.assertEqual(res["Value"]["Successful"][self.subDir], { "Files": 1, "Size": self.subFileSize, "SubDirs": 0 }) self.assertEqual(res["Value"]["Successful"]["/test"], { "Files": 1, "Size": self.existingFileSize, "SubDirs": 1 }) self.assertTrue( os.strerror(errno.ENOENT) in res["Value"]["Failed"][self.nonExistingFile], res) self.assertTrue( os.strerror(errno.ENOTDIR) in res["Value"]["Failed"][self.existingFile], res) self.assertTrue( os.strerror(errno.ENOENT) in res["Value"]["Failed"][nonExistingDir], res) res = self.se.getDirectoryMetadata(self.ALL + localdirs) self.assertTrue(res["OK"], res) self.assertTrue(self.subDir in res["Value"]["Successful"]) self.assertTrue( os.strerror(errno.ENOENT) in res["Value"]["Failed"][self.nonExistingFile], res) self.assertTrue( os.strerror(errno.ENOENT) in res["Value"]["Failed"][nonExistingDir], res) self.assertTrue( os.strerror(errno.ENOTDIR) in res["Value"]["Failed"][self.existingFile], res) res = self.se.isDirectory(self.ALL + localdirs) self.assertTrue(res["OK"], res) self.assertTrue(not res["Value"]["Successful"][self.existingFile]) self.assertTrue(res["Value"]["Successful"][self.subDir], res) self.assertTrue( os.strerror(errno.ENOENT) in res["Value"]["Failed"][self.nonExistingFile], res) self.assertTrue( os.strerror(errno.ENOENT) in res["Value"]["Failed"][nonExistingDir], res) res = self.se.listDirectory(self.ALL + localdirs) self.assertTrue(res["OK"], res) self.assertEqual(sorted(list(res["Value"]["Successful"][self.subDir])), sorted(["Files", "SubDirs"])) self.assertEqual( list(res["Value"]["Successful"][self.subDir]["Files"]), [self.subFile]) self.assertEqual( list(res["Value"]["Successful"][self.subDir]["SubDirs"]), []) self.assertEqual(list(res["Value"]["Successful"]["/test"]["Files"]), [self.existingFile]) self.assertEqual(list(res["Value"]["Successful"]["/test"]["SubDirs"]), [self.subDir]) self.assertTrue( os.strerror(errno.ENOENT) in res["Value"]["Failed"][self.nonExistingFile], res) self.assertTrue( os.strerror(errno.ENOTDIR) in res["Value"]["Failed"][self.existingFile], res) self.assertTrue( os.strerror(errno.ENOENT) in res["Value"]["Failed"][nonExistingDir], res) res = self.se.getDirectory(self.ALL + localdirs, localPath=self.destPath) self.assertTrue(res["OK"], res) self.assertEqual(res["Value"]["Successful"]["/test"], { "Files": 2, "Size": self.existingFileSize + self.subFileSize }) self.assertTrue(os.path.exists(self.destPath + self.existingFile)) self.assertTrue(os.path.exists(self.destPath + self.subFile)) self.assertEqual(res["Value"]["Successful"][self.subDir], { "Files": 1, "Size": self.subFileSize }) self.assertTrue( os.path.exists(self.destPath + self.subFile.replace("/test", ""))) self.assertEqual(res["Value"]["Failed"][self.nonExistingFile], { "Files": 0, "Size": 0 }) self.assertEqual(res["Value"]["Failed"][self.existingFile], { "Files": 0, "Size": 0 }) self.assertEqual(res["Value"]["Failed"][nonExistingDir], { "Files": 0, "Size": 0 }) res = self.se.removeDirectory(nonExistingDir, recursive=False) self.assertTrue(res["OK"], res) self.assertEqual(res["Value"]["Successful"][nonExistingDir], True) res = self.se.removeDirectory(nonExistingDir, recursive=True) self.assertTrue(res["OK"], res) self.assertEqual(res["Value"]["Failed"][nonExistingDir], { "FilesRemoved": 0, "SizeRemoved": 0 }) res = self.se.removeDirectory(self.nonExistingFile, recursive=False) self.assertTrue(res["OK"], res) self.assertEqual(res["Value"]["Successful"][self.nonExistingFile], True) res = self.se.removeDirectory(self.nonExistingFile, recursive=True) self.assertTrue(res["OK"], res) self.assertEqual(res["Value"]["Failed"][self.nonExistingFile], { "FilesRemoved": 0, "SizeRemoved": 0 }) res = self.se.removeDirectory(self.existingFile, recursive=False) self.assertTrue(res["OK"], res) self.assertTrue( os.strerror(errno.ENOTDIR) in res["Value"]["Failed"][self.existingFile], res) res = self.se.removeDirectory(self.existingFile, recursive=True) self.assertTrue(res["OK"], res) self.assertEqual(res["Value"]["Failed"][self.existingFile], { "FilesRemoved": 0, "SizeRemoved": 0 }) res = self.se.removeDirectory("/test", recursive=False) self.assertTrue(res["OK"], res) self.assertEqual(res["Value"]["Successful"]["/test"], True) self.assertTrue(not os.path.exists(self.basePath + self.existingFile)) self.assertTrue(os.path.exists(self.basePath + self.subFile)) res = self.se.removeDirectory("/test", recursive=True) self.assertTrue(res["OK"], res) self.assertEqual(res["Value"]["Successful"]["/test"], { "FilesRemoved": 1, "SizeRemoved": self.subFileSize }) self.assertTrue(not os.path.exists(self.basePath + "/test"))
def setUp(self, _mk_generateStorage, _mk_isLocalSE, _mk_addAccountingOperation): # Creating test configuration file self.testCfgFileName = os.path.join(tempfile.gettempdir(), 'test_StorageElement.cfg') cfgContent = ''' DIRAC { Setup=TestSetup } Resources{ StorageElements{ DiskStorageA { BackendType = local ReadAccess = Active WriteAccess = Active GFAL2_SRM2 { Host = srm-diskandtape.cern.ch SpaceToken = Disk Protocol = srm Path = /base/pathDisk } } # Same end point as DiskStorageA, but with a different space token # So they should be considered the same TapeStorageA { BackendType = local ReadAccess = Active WriteAccess = Active GFAL2_SRM2 { Host = srm-diskandtape.cern.ch Protocol = srm SpaceToken = Tape Path = /base/pathDisk } } # Normally does not happen in practice, but this is the same as DiskStorageA with more plugins DiskStorageAWithMoreProtocol { BackendType = local ReadAccess = Active WriteAccess = Active GFAL2_SRM2 { Host = srm-diskandtape.cern.ch SpaceToken = Disk Protocol = srm Path = /base/pathDisk } GFAL2_GSIFTP { Host = gsiftp-diskandtape.cern.ch SpaceToken = Disk Protocol = gsiftp Path = /base/pathDisk } } # A different storage StorageB { BackendType = local ReadAccess = Active WriteAccess = Active GFAL2_GSIFTP { Host = otherstorage.cern.ch SpaceToken = Disk Protocol = gsiftp Path = /base/pathDisk } } # The same endpoint as StorageB but with differetn base path, so not the same StorageBWithOtherBasePath { BackendType = local ReadAccess = Active WriteAccess = Active GFAL2_GSIFTP { Host = otherstorage.cern.ch SpaceToken = Disk Protocol = gsiftp Path = /base/otherPath } } } Operations{ Defaults { DataManagement{ AccessProtocols = fakeProto AccessProtocols += root WriteProtocols = srm } } } ''' with open(self.testCfgFileName, 'w') as f: f.write(cfgContent) # SUPER UGLY: one must recreate the CFG objects of gConfigurationData # not to conflict with other tests that might be using a local dirac.cfg gConfigurationData.localCFG = CFG() gConfigurationData.remoteCFG = CFG() gConfigurationData.mergedCFG = CFG() gConfigurationData.generateNewVersion() gConfig = ConfigurationClient(fileToLoadList=[ self.testCfgFileName ]) # we replace the configuration by our own one. self.diskStorageA = StorageElementItem('DiskStorageA') self.diskStorageA.vo = 'lhcb' self.tapeStorageA = StorageElementItem('TapeStorageA') self.tapeStorageA.vo = 'lhcb' self.diskStorageAWithMoreProtocol = StorageElementItem( 'DiskStorageAWithMoreProtocol') self.diskStorageAWithMoreProtocol.vo = 'lhcb' self.storageB = StorageElementItem('StorageB') self.storageB.vo = 'lhcb' self.storageBWithOtherBasePath = StorageElementItem( 'StorageBWithOtherBasePath') self.storageBWithOtherBasePath.vo = 'lhcb'
def execute(self): """ Run this. """ if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose( 'Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('Workflow status is not OK') result = self.resolveInputVariables() if not result['OK']: self.log.error("Failed to resolve input parameters:", result["Message"]) return result if not self.srmfiles: self.log.error('Files txt was not found correctly: %s' % self.srmfiles) return S_ERROR('Files txt was not found correctly') if not isinstance(self.files[0], dict): self.log.error('Files were not found correctly: %s' % self.files) return S_ERROR('Files were not found correctly') ##Now need to check that there are not that many concurrent jobs getting the overlay at the same time max_concurrent_running = self.ops.getValue( '/GetSRM/MaxConcurrentRunning', 100) error_count = 0 while 1: if error_count > 10: self.log.error( 'JobDB Content does not return expected dictionary') return S_ERROR( 'Failed to get number of concurrent overlay jobs') jobMonitor = RPCClient('WorkloadManagement/JobMonitoring', timeout=60) res = jobMonitor.getCurrentJobCounters( {'ApplicationStatus': 'Downloading SRM files'}) if not res['OK']: error_count += 1 time.sleep(60) continue running = 0 if 'Running' in res['Value']: running = res['Value']['Running'] if running < max_concurrent_running: break else: time.sleep(60) self.setApplicationStatus('Downloading SRM files') for filed in self.files: if 'file' not in filed or 'site' not in filed: self.log.error('Dictionnary does not contain correct keys') return S_ERROR('Dictionnary does not contain correct keys') start = os.getcwd() downloadDir = tempfile.mkdtemp(prefix='InputData_%s' % (self.counter), dir=start) os.chdir(downloadDir) storageElement = StorageElement(filed['site']) result = storageElement.getFile(filed['file']) if result['Value']['Failed']: result = storageElement.getFile(filed['file']) os.chdir(start) if result['Value']['Failed']: self.log.error("Failed to get the file from storage:", result['Value']['Failed']) return result self.counter += 1 return S_OK()
class TestBase(unittest.TestCase): """ Base test class. Defines all the method to test """ @mock.patch( 'DIRAC.Resources.Storage.StorageFactory.StorageFactory._StorageFactory__generateStorageObject', side_effect=mock_StorageFactory_generateStorageObject) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value=S_OK(True)) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value=None) # Don't send accounting def setUp(self, _mk_generateStorage, _mk_isLocalSE, _mk_addAccountingOperation): # Creating test configuration file self.testCfgFileName = os.path.join(tempfile.gettempdir(), 'test_StorageElement.cfg') cfgContent = ''' DIRAC { Setup=TestSetup } Resources{ StorageElements{ StorageA { BackendType = local ReadAccess = Active WriteAccess = Active AccessProtocol.0 { Host = PluginName = File Protocol = file Path = } } StorageB { BackendType = local ReadAccess = Active WriteAccess = Active AccessProtocol.0 { Host = PluginName = SRM2 Protocol = srm Path = } } StorageC { BackendType = local ReadAccess = Active WriteAccess = Active AccessProtocol.0 { Host = PluginName = XROOT Protocol = root Path = } } StorageD { BackendType = local ReadAccess = Active WriteAccess = Active AccessProtocol.0 { Host = PluginName = SRM2 Protocol = srm Path = } AccessProtocol.1 { Host = PluginName = XROOT Protocol = root Path = } } StorageE { BackendType = local ReadAccess = Active WriteAccess = Active WriteProtocols = root WriteProtocols += srm AccessProtocol.0 { Host = PluginName = SRM2 Protocol = srm Path = } AccessProtocol.1 { Host = PluginName = XROOT Protocol = root Path = } } StorageX { BackendType = local ReadAccess = Active WriteAccess = Active WriteProtocols = gsiftp AccessProtocols = root AccessProtocol.0 { Host = PluginName = GSIFTP Protocol = gsiftp Path = } AccessProtocol.1 { Host = PluginName = XROOT Protocol = root Path = } } StorageY { BackendType = local ReadAccess = Active WriteAccess = Active AccessProtocols = gsiftp AccessProtocols += srm AccessProtocol.0 { Host = PluginName = GSIFTP Protocol = gsiftp Path = } AccessProtocol.1 { Host = PluginName = SRM2 Protocol = srm Path = } } StorageZ { BackendType = local ReadAccess = Active WriteAccess = Active AccessProtocols = root AccessProtocols += srm WriteProtocols = root WriteProtocols += srm AccessProtocol.0 { Host = PluginName = ROOT Protocol = root Path = } AccessProtocol.1 { Host = PluginName = SRM2 Protocol = srm Path = } } } } Operations{ Defaults { DataManagement{ AccessProtocols = fakeProto AccessProtocols += root WriteProtocols = srm } } } ''' with open(self.testCfgFileName, 'w') as f: f.write(cfgContent) # SUPER UGLY: one must recreate the CFG objects of gConfigurationData # not to conflict with other tests that might be using a local dirac.cfg gConfigurationData.localCFG = CFG() gConfigurationData.remoteCFG = CFG() gConfigurationData.mergedCFG = CFG() gConfigurationData.generateNewVersion() gConfig = ConfigurationClient(fileToLoadList=[ self.testCfgFileName ]) # we replace the configuration by our own one. self.seA = StorageElementItem('StorageA') self.seA.vo = 'lhcb' self.seB = StorageElementItem('StorageB') self.seB.vo = 'lhcb' self.seC = StorageElementItem('StorageC') self.seC.vo = 'lhcb' self.seD = StorageElementItem('StorageD') self.seD.vo = 'lhcb' self.seE = StorageElementItem('StorageE') self.seE.vo = 'lhcb' self.seX = StorageElementItem('StorageX') self.seX.vo = 'lhcb' self.seY = StorageElementItem('StorageY') self.seY.vo = 'lhcb' self.seZ = StorageElementItem('StorageZ') self.seZ.vo = 'lhcb' def tearDown(self): try: os.remove(self.testCfgFileName) except OSError: pass # SUPER UGLY: one must recreate the CFG objects of gConfigurationData # not to conflict with other tests that might be using a local dirac.cfg gConfigurationData.localCFG = CFG() gConfigurationData.remoteCFG = CFG() gConfigurationData.mergedCFG = CFG() gConfigurationData.generateNewVersion() @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value=S_OK(True)) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value=None) # Don't send accounting def test_01_negociateProtocolWithOtherSE(self, mk_isLocalSE, mk_addAccounting): """Testing negotiation algorithm""" # Find common protocol between SRM2 and File res = self.seA.negociateProtocolWithOtherSE(self.seB) self.assertTrue(res['OK'], res) self.assertEqual(res['Value'], ['file']) # Find common protocol between File and SRM@ res = self.seB.negociateProtocolWithOtherSE(self.seA) self.assertTrue(res['OK'], res) self.assertEqual(res['Value'], ['file']) # Find common protocol between XROOT and File # Nothing goes from xroot to file res = self.seA.negociateProtocolWithOtherSE(self.seC) self.assertTrue(res['OK'], res) self.assertEqual(res['Value'], []) # Find common protocol between File and XROOT res = self.seC.negociateProtocolWithOtherSE(self.seA) self.assertTrue(res['OK'], res) self.assertEqual(res['Value'], ['file']) # Find common protocol between File and File res = self.seA.negociateProtocolWithOtherSE(self.seA) self.assertTrue(res['OK'], res) self.assertEqual(res['Value'], ['file']) # Find common protocol between SRM and SRM res = self.seB.negociateProtocolWithOtherSE(self.seB) self.assertTrue(res['OK'], res) self.assertEqual(sorted(res['Value']), sorted(['file', 'root', 'srm'])) # Find common protocol between SRM and XROOT res = self.seC.negociateProtocolWithOtherSE(self.seB) self.assertTrue(res['OK'], res) self.assertEqual(sorted(res['Value']), sorted(['root', 'file'])) # Find common protocol between XROOT and SRM res = self.seC.negociateProtocolWithOtherSE(self.seB) self.assertTrue(res['OK'], res) self.assertEqual(sorted(res['Value']), sorted(['root', 'file'])) # Testing restrictions res = self.seC.negociateProtocolWithOtherSE(self.seB, protocols=['file']) self.assertTrue(res['OK'], res) self.assertEqual(sorted(res['Value']), ['file']) res = self.seC.negociateProtocolWithOtherSE(self.seB, protocols=['nonexisting']) self.assertTrue(res['OK'], res) self.assertEqual(res['Value'], []) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value=S_OK(True)) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value=None) # Don't send accounting def test_02_followOrder(self, _mk_isLocalSE, _mk_addAccounting): """Testing If the order of preferred protocols is respected""" for permutation in itertools.permutations( ['srm', 'file', 'root', 'nonexisting']): permuList = list(permutation) # Don't get tricked ! remove cannot be put # after the conversion, because it is inplace modification permuList.remove('nonexisting') res = self.seD.negociateProtocolWithOtherSE(self.seD, protocols=permutation) self.assertTrue(res['OK'], res) self.assertEqual(res['Value'], permuList) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value=S_OK(True)) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value=None) # Don't send accounting def test_03_multiProtocolThirdParty(self, _mk_isLocalSE, _mk_addAccounting): """ Test case for storages with several protocols Here comes the fun :-) Suppose we have endpoints that we can read in root, but cannot write If we have root in the accessProtocols and thirdPartyProtocols lists but not in the writeProtocols, we should get a root url to read, and write with SRM We reproduce here the behavior of DataManager.replicate """ thirdPartyProtocols = ['root', 'srm'] lfn = '/lhcb/fake/lfn' res = self.seD.negociateProtocolWithOtherSE( self.seD, protocols=thirdPartyProtocols) self.assertTrue(res['OK'], res) self.assertEqual(res['Value'], thirdPartyProtocols) # Only the XROOT plugin here implements the geTransportURL # that returns what we want, so we know that # if the return is successful, it is because of the XROOT res = self.seD.getURL(lfn, protocol=res['Value']) self.assertTrue(res['OK'], res) self.assertTrue(lfn in res['Value']['Successful'], res) srcUrl = res['Value']['Successful'][lfn] self.assertEqual(srcUrl, "root:getTransportURL") # Only the SRM2 plugin here implements the putFile method # so if we get a success here, it means that we used the SRM plugin res = self.seD.replicateFile({lfn: srcUrl}, sourceSize=123, inputProtocol='root') self.assertTrue(res['OK'], res) self.assertTrue(lfn in res['Value']['Successful'], res) self.assertEqual(res['Value']['Successful'][lfn], "srm:putFile") @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value=S_OK(True)) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value=None) # Don't send accounting def test_04_thirdPartyLocalWrite(self, _mk_isLocalSE, _mk_addAccounting): """ Test case for storages with several protocols Here, we locally define the write protocol to be root and srm So we should be able to do everything with XROOT plugin """ thirdPartyProtocols = ['root', 'srm'] lfn = '/lhcb/fake/lfn' res = self.seE.negociateProtocolWithOtherSE( self.seE, protocols=thirdPartyProtocols) self.assertTrue(res['OK'], res) self.assertEqual(res['Value'], thirdPartyProtocols) res = self.seE.getURL(lfn, protocol=res['Value']) self.assertTrue(res['OK'], res) self.assertTrue(lfn in res['Value']['Successful'], res) srcUrl = res['Value']['Successful'][lfn] self.assertEqual(srcUrl, "root:getTransportURL") res = self.seE.replicateFile({lfn: srcUrl}, sourceSize=123, inputProtocol='root') self.assertTrue(res['OK'], res) self.assertTrue(lfn in res['Value']['Successful'], res) self.assertEqual(res['Value']['Successful'][lfn], "root:putFile") @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value=S_OK(True)) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value=None) # Don't send accounting def test_05_thirdPartyMix(self, _mk_isLocalSE, _mk_addAccounting): """ Test case for storages with several protocols Here, we locally define the write protocol for the destination, so it should all go directly through the XROOT plugin """ thirdPartyProtocols = ['root', 'srm'] lfn = '/lhcb/fake/lfn' res = self.seE.negociateProtocolWithOtherSE( self.seD, protocols=thirdPartyProtocols) self.assertTrue(res['OK'], res) self.assertEqual(res['Value'], thirdPartyProtocols) res = self.seD.getURL(lfn, protocol=res['Value']) self.assertTrue(res['OK'], res) self.assertTrue(lfn in res['Value']['Successful'], res) srcUrl = res['Value']['Successful'][lfn] self.assertEqual(srcUrl, "root:getTransportURL") res = self.seE.replicateFile({lfn: srcUrl}, sourceSize=123, inputProtocol='root') self.assertTrue(res['OK'], res) self.assertTrue(lfn in res['Value']['Successful'], res) self.assertEqual(res['Value']['Successful'][lfn], "root:putFile") @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value=S_OK(True)) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value=None) # Don't send accounting def test_06_thirdPartyMixOpposite(self, _mk_isLocalSE, _mk_addAccounting): """ Test case for storages with several protocols Here, we locally define the write protocol for the source, so it should get the source directly using XROOT, and perform the put using SRM """ thirdPartyProtocols = ['root', 'srm'] lfn = '/lhcb/fake/lfn' res = self.seD.negociateProtocolWithOtherSE( self.seE, protocols=thirdPartyProtocols) self.assertTrue(res['OK'], res) self.assertEqual(res['Value'], thirdPartyProtocols) res = self.seE.getURL(lfn, protocol=res['Value']) self.assertTrue(res['OK'], res) self.assertTrue(lfn in res['Value']['Successful'], res) srcUrl = res['Value']['Successful'][lfn] self.assertEqual(srcUrl, "root:getTransportURL") res = self.seD.replicateFile({lfn: srcUrl}, sourceSize=123, inputProtocol='root') self.assertTrue(res['OK'], res) self.assertTrue(lfn in res['Value']['Successful'], res) self.assertEqual(res['Value']['Successful'][lfn], "srm:putFile") @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value=S_OK(True)) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value=None) # Don't send accounting def test_07_multiProtocolSrmOnly(self, _mk_isLocalSE, _mk_addAccounting): """ Test case for storages with several protocols Here comes the fun :-) Suppose we have endpoints that we can read in root, but cannot write If we have root in the accessProtocols and thirdPartyProtocols lists but not in the writeProtocols, we should get a root url to read, and write with SRM We reproduce here the behavior of DataManager.replicate """ thirdPartyProtocols = ['srm'] lfn = '/lhcb/fake/lfn' res = self.seD.negociateProtocolWithOtherSE( self.seD, protocols=thirdPartyProtocols) self.assertTrue(res['OK'], res) self.assertEqual(res['Value'], thirdPartyProtocols) res = self.seD.getURL(lfn, protocol=res['Value']) self.assertTrue(res['OK'], res) self.assertTrue(lfn in res['Value']['Successful'], res) srcUrl = res['Value']['Successful'][lfn] self.assertEqual(srcUrl, "srm:getTransportURL") res = self.seD.replicateFile({lfn: srcUrl}, sourceSize=123, inputProtocol='srm') self.assertTrue(res['OK'], res) self.assertTrue(lfn in res['Value']['Successful'], res) self.assertEqual(res['Value']['Successful'][lfn], "srm:putFile") @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value=S_OK(True)) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value=None) # Don't send accounting def test_08_multiProtocolFTS(self, _mk_isLocalSE, _mk_addAccounting): """ Test case FTS replication between storages with several protocols Here comes the fun :-) Suppose we have endpoints that we can read in root, but cannot write If we have root in the accessProtocols and thirdPartyProtocols lists but not in the writeProtocols, we should get a root url to read, and write with SRM. And We should get the proper url for source and destination Storage X, Y and Z represents the situation we could now have in LHCb: * X is RAL Echo: you read with root, write with gsiftp * Y is Gridka: you have gsiftp available for read only * Z is CERN EOS: you can do everything with EOS This makes it necessary to add gsiftp as third party option to write to ECHO """ thirdPartyProtocols = ['root', 'gsiftp', 'srm'] rankedProtocols = [ 'root', 'gsiftp', 'gsidcap', 'dcap', 'file', 'srm', 'rfio' ] lfn = '/lhcb/fake/lfn' # RAL -> GRIDKA # We should read using root and write through srm res = self.seY.generateTransferURLsBetweenSEs( lfn, self.seX, protocols=rankedProtocols) self.assertTrue(res['OK'], res) urlPair = res['Value']['Successful'].get(lfn) self.assertTupleEqual(urlPair, ('root:%s' % lfn, 'srm:%s' % lfn)) protoPair = res['Value']['Protocols'] self.assertTupleEqual(protoPair, ('root', 'srm')) # RAL -> CERN # We should read using root and write directly with it res = self.seZ.generateTransferURLsBetweenSEs( lfn, self.seX, protocols=rankedProtocols) self.assertTrue(res['OK'], res) urlPair = res['Value']['Successful'].get(lfn) self.assertTupleEqual(urlPair, ('root:%s' % lfn, 'root:%s' % lfn)) protoPair = res['Value']['Protocols'] self.assertTupleEqual(protoPair, ('root', 'root')) # GRIDKA -> RAL # We should read using gsiftp and write directly with it res = self.seX.generateTransferURLsBetweenSEs( lfn, self.seY, protocols=rankedProtocols) self.assertTrue(res['OK'], res) urlPair = res['Value']['Successful'].get(lfn) self.assertTupleEqual(urlPair, ('gsiftp:%s' % lfn, 'gsiftp:%s' % lfn)) protoPair = res['Value']['Protocols'] self.assertTupleEqual(protoPair, ('gsiftp', 'gsiftp')) # GRIDKA -> CERN # We should read using srm and write with root res = self.seZ.generateTransferURLsBetweenSEs( lfn, self.seY, protocols=rankedProtocols) self.assertTrue(res['OK'], res) urlPair = res['Value']['Successful'].get(lfn) self.assertTupleEqual(urlPair, ('srm:%s' % lfn, 'root:%s' % lfn)) protoPair = res['Value']['Protocols'] self.assertTupleEqual(protoPair, ('srm', 'root')) # CERN -> RAL # We should read using srm and write with gsiftp res = self.seX.generateTransferURLsBetweenSEs( lfn, self.seZ, protocols=rankedProtocols) self.assertTrue(res['OK'], res) urlPair = res['Value']['Successful'].get(lfn) self.assertTupleEqual(urlPair, ('srm:%s' % lfn, 'gsiftp:%s' % lfn)) protoPair = res['Value']['Protocols'] self.assertTupleEqual(protoPair, ('srm', 'gsiftp')) # CERN -> GRIDKA # We should read using root and write directly with srm res = self.seY.generateTransferURLsBetweenSEs( lfn, self.seZ, protocols=rankedProtocols) self.assertTrue(res['OK'], res) urlPair = res['Value']['Successful'].get(lfn) self.assertTupleEqual(urlPair, ('root:%s' % lfn, 'srm:%s' % lfn)) protoPair = res['Value']['Protocols'] self.assertTupleEqual(protoPair, ('root', 'srm'))
def setUp(self, _mk_generateStorage, _mk_isLocalSE, _mk_addAccountingOperation): # Creating test configuration file self.testCfgFileName = os.path.join(tempfile.gettempdir(), 'test_StorageElement.cfg') cfgContent = ''' DIRAC { Setup=TestSetup } Resources{ StorageElements{ StorageA { BackendType = local ReadAccess = Active WriteAccess = Active AccessProtocol.0 { Host = PluginName = File Protocol = file Path = } } StorageB { BackendType = local ReadAccess = Active WriteAccess = Active AccessProtocol.0 { Host = PluginName = SRM2 Protocol = srm Path = } } StorageC { BackendType = local ReadAccess = Active WriteAccess = Active AccessProtocol.0 { Host = PluginName = XROOT Protocol = root Path = } } StorageD { BackendType = local ReadAccess = Active WriteAccess = Active AccessProtocol.0 { Host = PluginName = SRM2 Protocol = srm Path = } AccessProtocol.1 { Host = PluginName = XROOT Protocol = root Path = } } StorageE { BackendType = local ReadAccess = Active WriteAccess = Active WriteProtocols = root WriteProtocols += srm AccessProtocol.0 { Host = PluginName = SRM2 Protocol = srm Path = } AccessProtocol.1 { Host = PluginName = XROOT Protocol = root Path = } } StorageX { BackendType = local ReadAccess = Active WriteAccess = Active WriteProtocols = gsiftp AccessProtocols = root AccessProtocol.0 { Host = PluginName = GSIFTP Protocol = gsiftp Path = } AccessProtocol.1 { Host = PluginName = XROOT Protocol = root Path = } } StorageY { BackendType = local ReadAccess = Active WriteAccess = Active AccessProtocols = gsiftp AccessProtocols += srm AccessProtocol.0 { Host = PluginName = GSIFTP Protocol = gsiftp Path = } AccessProtocol.1 { Host = PluginName = SRM2 Protocol = srm Path = } } StorageZ { BackendType = local ReadAccess = Active WriteAccess = Active AccessProtocols = root AccessProtocols += srm WriteProtocols = root WriteProtocols += srm AccessProtocol.0 { Host = PluginName = ROOT Protocol = root Path = } AccessProtocol.1 { Host = PluginName = SRM2 Protocol = srm Path = } } } } Operations{ Defaults { DataManagement{ AccessProtocols = fakeProto AccessProtocols += root WriteProtocols = srm } } } ''' with open(self.testCfgFileName, 'w') as f: f.write(cfgContent) # SUPER UGLY: one must recreate the CFG objects of gConfigurationData # not to conflict with other tests that might be using a local dirac.cfg gConfigurationData.localCFG = CFG() gConfigurationData.remoteCFG = CFG() gConfigurationData.mergedCFG = CFG() gConfigurationData.generateNewVersion() gConfig = ConfigurationClient(fileToLoadList=[ self.testCfgFileName ]) # we replace the configuration by our own one. self.seA = StorageElementItem('StorageA') self.seA.vo = 'lhcb' self.seB = StorageElementItem('StorageB') self.seB.vo = 'lhcb' self.seC = StorageElementItem('StorageC') self.seC.vo = 'lhcb' self.seD = StorageElementItem('StorageD') self.seD.vo = 'lhcb' self.seE = StorageElementItem('StorageE') self.seE.vo = 'lhcb' self.seX = StorageElementItem('StorageX') self.seX.vo = 'lhcb' self.seY = StorageElementItem('StorageY') self.seY.vo = 'lhcb' self.seZ = StorageElementItem('StorageZ') self.seZ.vo = 'lhcb'
class TestBase( unittest.TestCase ): """ Base test class. Defines all the method to test """ @mock.patch( 'DIRAC.Resources.Storage.StorageFactory.StorageFactory._StorageFactory__generateStorageObject', side_effect = mock_StorageFactory_generateStorageObject ) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value = S_OK( True ) ) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value = None ) # Don't send accounting def setUp( self, _mk_generateStorage, _mk_isLocalSE, _mk_addAccountingOperation ): #Creating test configuration file self.testCfgFileName = os.path.join(tempfile.gettempdir(), 'test_StorageElement.cfg') cfgContent=''' DIRAC { Setup=TestSetup } Resources{ StorageElements{ StorageA { BackendType = local ReadAccess = Active WriteAccess = Active AccessProtocol.0 { Host = PluginName = File Protocol = file Path = } } StorageB { BackendType = local ReadAccess = Active WriteAccess = Active AccessProtocol.0 { Host = PluginName = SRM2 Protocol = srm Path = } } StorageC { BackendType = local ReadAccess = Active WriteAccess = Active AccessProtocol.0 { Host = PluginName = XROOT Protocol = root Path = } } StorageD { BackendType = local ReadAccess = Active WriteAccess = Active AccessProtocol.0 { Host = PluginName = SRM2 Protocol = srm Path = } AccessProtocol.1 { Host = PluginName = XROOT Protocol = root Path = } } StorageE { BackendType = local ReadAccess = Active WriteAccess = Active WriteProtocols = root WriteProtocols += srm AccessProtocol.0 { Host = PluginName = SRM2 Protocol = srm Path = } AccessProtocol.1 { Host = PluginName = XROOT Protocol = root Path = } } StorageX { BackendType = local ReadAccess = Active WriteAccess = Active WriteProtocols = gsiftp AccessProtocols = root AccessProtocol.0 { Host = PluginName = GSIFTP Protocol = gsiftp Path = } AccessProtocol.1 { Host = PluginName = XROOT Protocol = root Path = } } StorageY { BackendType = local ReadAccess = Active WriteAccess = Active AccessProtocols = gsiftp AccessProtocols += srm AccessProtocol.0 { Host = PluginName = GSIFTP Protocol = gsiftp Path = } AccessProtocol.1 { Host = PluginName = SRM2 Protocol = srm Path = } } StorageZ { BackendType = local ReadAccess = Active WriteAccess = Active AccessProtocols = root AccessProtocols += srm WriteProtocols = root WriteProtocols += srm AccessProtocol.0 { Host = PluginName = ROOT Protocol = root Path = } AccessProtocol.1 { Host = PluginName = SRM2 Protocol = srm Path = } } } } Operations{ Defaults { DataManagement{ AccessProtocols = fakeProto AccessProtocols += root WriteProtocols = srm } } } ''' with open(self.testCfgFileName, 'w') as f: f.write(cfgContent) # SUPER UGLY: one must recreate the CFG objects of gConfigurationData # not to conflict with other tests that might be using a local dirac.cfg gConfigurationData.localCFG=CFG() gConfigurationData.remoteCFG=CFG() gConfigurationData.mergedCFG=CFG() gConfigurationData.generateNewVersion() gConfig = ConfigurationClient(fileToLoadList = [self.testCfgFileName]) #we replace the configuration by our own one. self.seA = StorageElementItem( 'StorageA' ) self.seA.vo = 'lhcb' self.seB = StorageElementItem( 'StorageB' ) self.seB.vo = 'lhcb' self.seC = StorageElementItem( 'StorageC' ) self.seC.vo = 'lhcb' self.seD = StorageElementItem( 'StorageD' ) self.seD.vo = 'lhcb' self.seE = StorageElementItem( 'StorageE' ) self.seE.vo = 'lhcb' self.seX = StorageElementItem( 'StorageX' ) self.seX.vo = 'lhcb' self.seY = StorageElementItem( 'StorageY' ) self.seY.vo = 'lhcb' self.seZ = StorageElementItem( 'StorageZ' ) self.seZ.vo = 'lhcb' def tearDown( self ): try: os.remove(self.testCfgFileName) except OSError: pass # SUPER UGLY: one must recreate the CFG objects of gConfigurationData # not to conflict with other tests that might be using a local dirac.cfg gConfigurationData.localCFG=CFG() gConfigurationData.remoteCFG=CFG() gConfigurationData.mergedCFG=CFG() gConfigurationData.generateNewVersion() @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value = S_OK( True ) ) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value = None ) # Don't send accounting def test_01_negociateProtocolWithOtherSE( self, mk_isLocalSE, mk_addAccounting ): """Testing negotiation algorithm""" # Find common protocol between SRM2 and File res = self.seA.negociateProtocolWithOtherSE( self.seB ) self.assertTrue( res['OK'], res ) self.assertEqual( res['Value'], ['file'] ) # Find common protocol between File and SRM@ res = self.seB.negociateProtocolWithOtherSE( self.seA ) self.assertTrue( res['OK'], res ) self.assertEqual( res['Value'], ['file'] ) # Find common protocol between XROOT and File # Nothing goes from xroot to file res = self.seA.negociateProtocolWithOtherSE( self.seC ) self.assertTrue( res['OK'], res ) self.assertEqual( res['Value'], [] ) # Find common protocol between File and XROOT res = self.seC.negociateProtocolWithOtherSE( self.seA ) self.assertTrue( res['OK'], res ) self.assertEqual( res['Value'], ['file'] ) # Find common protocol between File and File res = self.seA.negociateProtocolWithOtherSE( self.seA ) self.assertTrue( res['OK'], res ) self.assertEqual( res['Value'], ['file'] ) # Find common protocol between SRM and SRM res = self.seB.negociateProtocolWithOtherSE( self.seB ) self.assertTrue( res['OK'], res ) self.assertEqual( sorted( res['Value'] ), sorted( ['file', 'root', 'srm'] ) ) # Find common protocol between SRM and XROOT res = self.seC.negociateProtocolWithOtherSE( self.seB ) self.assertTrue( res['OK'], res ) self.assertEqual( sorted( res['Value'] ), sorted( ['root', 'file'] ) ) # Find common protocol between XROOT and SRM res = self.seC.negociateProtocolWithOtherSE( self.seB ) self.assertTrue( res['OK'], res ) self.assertEqual( sorted( res['Value'] ), sorted( ['root', 'file'] ) ) # Testing restrictions res = self.seC.negociateProtocolWithOtherSE( self.seB, protocols = ['file'] ) self.assertTrue( res['OK'], res ) self.assertEqual( sorted( res['Value'] ), ['file'] ) res = self.seC.negociateProtocolWithOtherSE( self.seB, protocols = ['nonexisting'] ) self.assertTrue( res['OK'], res ) self.assertEqual( res['Value'], [] ) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value = S_OK( True ) ) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value = None ) # Don't send accounting def test_02_followOrder( self, _mk_isLocalSE, _mk_addAccounting ): """Testing If the order of preferred protocols is respected""" for permutation in itertools.permutations( ['srm', 'file', 'root', 'nonexisting'] ): permuList = list( permutation ) # Don't get tricked ! remove cannot be put # after the conversion, because it is inplace modification permuList.remove( 'nonexisting' ) res = self.seD.negociateProtocolWithOtherSE( self.seD, protocols = permutation ) self.assertTrue( res['OK'], res ) self.assertEqual( res['Value'], permuList ) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value = S_OK( True ) ) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value = None ) # Don't send accounting def test_03_multiProtocolThirdParty( self, _mk_isLocalSE, _mk_addAccounting ): """ Test case for storages with several protocols Here comes the fun :-) Suppose we have endpoints that we can read in root, but cannot write If we have root in the accessProtocols and thirdPartyProtocols lists but not in the writeProtocols, we should get a root url to read, and write with SRM We reproduce here the behavior of DataManager.replicate """ thirdPartyProtocols = ['root', 'srm'] lfn = '/lhcb/fake/lfn' res = self.seD.negociateProtocolWithOtherSE( self.seD, protocols = thirdPartyProtocols ) self.assertTrue( res['OK'], res ) self.assertEqual( res['Value'] , thirdPartyProtocols ) # Only the XROOT plugin here implements the geTransportURL # that returns what we want, so we know that # if the return is successful, it is because of the XROOT res = self.seD.getURL( lfn, protocol = res['Value'] ) self.assertTrue( res['OK'], res ) self.assertTrue( lfn in res['Value']['Successful'], res ) srcUrl = res['Value']['Successful'][lfn] self.assertEqual( srcUrl, "root:getTransportURL" ) # Only the SRM2 plugin here implements the putFile method # so if we get a success here, it means that we used the SRM plugin res = self.seD.replicateFile( {lfn:srcUrl}, sourceSize = 123, inputProtocol = 'root' ) self.assertTrue( res['OK'], res ) self.assertTrue( lfn in res['Value']['Successful'], res ) self.assertEqual( res['Value']['Successful'][lfn], "srm:putFile" ) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value = S_OK( True ) ) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value = None ) # Don't send accounting def test_04_thirdPartyLocalWrite( self, _mk_isLocalSE, _mk_addAccounting ): """ Test case for storages with several protocols Here, we locally define the write protocol to be root and srm So we should be able to do everything with XROOT plugin """ thirdPartyProtocols = ['root', 'srm'] lfn = '/lhcb/fake/lfn' res = self.seE.negociateProtocolWithOtherSE( self.seE, protocols = thirdPartyProtocols ) self.assertTrue( res['OK'], res ) self.assertEqual( res['Value'] , thirdPartyProtocols ) res = self.seE.getURL( lfn, protocol = res['Value'] ) self.assertTrue( res['OK'], res ) self.assertTrue( lfn in res['Value']['Successful'], res ) srcUrl = res['Value']['Successful'][lfn] self.assertEqual( srcUrl, "root:getTransportURL" ) res = self.seE.replicateFile( {lfn:srcUrl}, sourceSize = 123, inputProtocol = 'root' ) self.assertTrue( res['OK'], res ) self.assertTrue( lfn in res['Value']['Successful'], res ) self.assertEqual( res['Value']['Successful'][lfn], "root:putFile" ) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value = S_OK( True ) ) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value = None ) # Don't send accounting def test_05_thirdPartyMix( self, _mk_isLocalSE, _mk_addAccounting ): """ Test case for storages with several protocols Here, we locally define the write protocol for the destination, so it should all go directly through the XROOT plugin """ thirdPartyProtocols = ['root', 'srm'] lfn = '/lhcb/fake/lfn' res = self.seE.negociateProtocolWithOtherSE( self.seD, protocols = thirdPartyProtocols ) self.assertTrue( res['OK'], res ) self.assertEqual( res['Value'] , thirdPartyProtocols ) res = self.seD.getURL( lfn, protocol = res['Value'] ) self.assertTrue( res['OK'], res ) self.assertTrue( lfn in res['Value']['Successful'], res ) srcUrl = res['Value']['Successful'][lfn] self.assertEqual( srcUrl, "root:getTransportURL" ) res = self.seE.replicateFile( {lfn:srcUrl}, sourceSize = 123, inputProtocol = 'root' ) self.assertTrue( res['OK'], res ) self.assertTrue( lfn in res['Value']['Successful'], res ) self.assertEqual( res['Value']['Successful'][lfn], "root:putFile" ) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value = S_OK( True ) ) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value = None ) # Don't send accounting def test_06_thirdPartyMixOpposite( self, _mk_isLocalSE, _mk_addAccounting ): """ Test case for storages with several protocols Here, we locally define the write protocol for the source, so it should get the source directly using XROOT, and perform the put using SRM """ thirdPartyProtocols = ['root', 'srm'] lfn = '/lhcb/fake/lfn' res = self.seD.negociateProtocolWithOtherSE( self.seE, protocols = thirdPartyProtocols ) self.assertTrue( res['OK'], res ) self.assertEqual( res['Value'] , thirdPartyProtocols ) res = self.seE.getURL( lfn, protocol = res['Value'] ) self.assertTrue( res['OK'], res ) self.assertTrue( lfn in res['Value']['Successful'], res ) srcUrl = res['Value']['Successful'][lfn] self.assertEqual( srcUrl, "root:getTransportURL" ) res = self.seD.replicateFile( {lfn:srcUrl}, sourceSize = 123, inputProtocol = 'root' ) self.assertTrue( res['OK'], res ) self.assertTrue( lfn in res['Value']['Successful'], res ) self.assertEqual( res['Value']['Successful'][lfn], "srm:putFile" ) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value = S_OK( True ) ) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value = None ) # Don't send accounting def test_07_multiProtocolSrmOnly( self, _mk_isLocalSE, _mk_addAccounting ): """ Test case for storages with several protocols Here comes the fun :-) Suppose we have endpoints that we can read in root, but cannot write If we have root in the accessProtocols and thirdPartyProtocols lists but not in the writeProtocols, we should get a root url to read, and write with SRM We reproduce here the behavior of DataManager.replicate """ thirdPartyProtocols = [ 'srm'] print "negociate" lfn = '/lhcb/fake/lfn' res = self.seD.negociateProtocolWithOtherSE( self.seD, protocols = thirdPartyProtocols ) self.assertTrue( res['OK'], res ) self.assertEqual( res['Value'] , thirdPartyProtocols ) print "get source url" res = self.seD.getURL( lfn, protocol = res['Value'] ) self.assertTrue( res['OK'], res ) self.assertTrue( lfn in res['Value']['Successful'], res ) srcUrl = res['Value']['Successful'][lfn] self.assertEqual( srcUrl, "srm:getTransportURL" ) print "replicate" res = self.seD.replicateFile( {lfn:srcUrl}, sourceSize = 123, inputProtocol = 'srm' ) self.assertTrue( res['OK'], res ) self.assertTrue( lfn in res['Value']['Successful'], res ) self.assertEqual( res['Value']['Successful'][lfn], "srm:putFile" ) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value = S_OK( True ) ) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value = None ) # Don't send accounting def test_08_multiProtocolFTS( self, _mk_isLocalSE, _mk_addAccounting ): """ Test case FTS replication between storages with several protocols Here comes the fun :-) Suppose we have endpoints that we can read in root, but cannot write If we have root in the accessProtocols and thirdPartyProtocols lists but not in the writeProtocols, we should get a root url to read, and write with SRM. And We should get the proper url for source and destination Storage X, Y and Z represents the situation we could now have in LHCb: * X is RAL Echo: you read with root, write with gsiftp * Y is Gridka: you have gsiftp available for read only * Z is CERN EOS: you can do everything with EOS This makes it necessary to add gsiftp as third party option to write to ECHO """ thirdPartyProtocols = ['root', 'gsiftp', 'srm'] lfn = '/lhcb/fake/lfn' # RAL -> GRIDKA # We should read using root and write through srm res = self.seY.generateTransferURLsBetweenSEs(lfn, self.seX) self.assertTrue( res['OK'], res ) urlPair = res['Value']['Successful'].get(lfn) self.assertTupleEqual(urlPair, ('root:%s'%lfn, 'srm:%s'%lfn)) # RAL -> CERN # We should read using root and write directly with it res = self.seZ.generateTransferURLsBetweenSEs(lfn, self.seX) self.assertTrue( res['OK'], res ) urlPair = res['Value']['Successful'].get(lfn) self.assertTupleEqual(urlPair, ('root:%s'%lfn, 'root:%s'%lfn)) # GRIDKA -> RAL # We should read using gsiftp and write directly with it res = self.seX.generateTransferURLsBetweenSEs(lfn, self.seY) self.assertTrue( res['OK'], res ) urlPair = res['Value']['Successful'].get(lfn) self.assertTupleEqual(urlPair, ('gsiftp:%s'%lfn, 'gsiftp:%s'%lfn)) # GRIDKA -> CERN # We should read using srm and write with root res = self.seZ.generateTransferURLsBetweenSEs(lfn, self.seY) self.assertTrue( res['OK'], res ) urlPair = res['Value']['Successful'].get(lfn) self.assertTupleEqual(urlPair, ('srm:%s'%lfn, 'root:%s'%lfn)) # CERN -> RAL # We should read using srm and write with gsiftp res = self.seX.generateTransferURLsBetweenSEs(lfn, self.seZ) self.assertTrue( res['OK'], res ) urlPair = res['Value']['Successful'].get(lfn) self.assertTupleEqual(urlPair, ('srm:%s'%lfn, 'gsiftp:%s'%lfn)) # CERN -> GRIDKA # We should read using root and write directly with srm res = self.seY.generateTransferURLsBetweenSEs(lfn, self.seZ) self.assertTrue( res['OK'], res ) urlPair = res['Value']['Successful'].get(lfn) self.assertTupleEqual(urlPair, ('root:%s'%lfn, 'srm:%s'%lfn))
def setUp( self, _mk_generateStorage, _mk_isLocalSE, _mk_addAccountingOperation ): #Creating test configuration file self.testCfgFileName = os.path.join(tempfile.gettempdir(), 'test_StorageElement.cfg') cfgContent=''' DIRAC { Setup=TestSetup } Resources{ StorageElements{ StorageA { BackendType = local ReadAccess = Active WriteAccess = Active AccessProtocol.0 { Host = PluginName = File Protocol = file Path = } } StorageB { BackendType = local ReadAccess = Active WriteAccess = Active AccessProtocol.0 { Host = PluginName = SRM2 Protocol = srm Path = } } StorageC { BackendType = local ReadAccess = Active WriteAccess = Active AccessProtocol.0 { Host = PluginName = XROOT Protocol = root Path = } } StorageD { BackendType = local ReadAccess = Active WriteAccess = Active AccessProtocol.0 { Host = PluginName = SRM2 Protocol = srm Path = } AccessProtocol.1 { Host = PluginName = XROOT Protocol = root Path = } } StorageE { BackendType = local ReadAccess = Active WriteAccess = Active WriteProtocols = root WriteProtocols += srm AccessProtocol.0 { Host = PluginName = SRM2 Protocol = srm Path = } AccessProtocol.1 { Host = PluginName = XROOT Protocol = root Path = } } StorageX { BackendType = local ReadAccess = Active WriteAccess = Active WriteProtocols = gsiftp AccessProtocols = root AccessProtocol.0 { Host = PluginName = GSIFTP Protocol = gsiftp Path = } AccessProtocol.1 { Host = PluginName = XROOT Protocol = root Path = } } StorageY { BackendType = local ReadAccess = Active WriteAccess = Active AccessProtocols = gsiftp AccessProtocols += srm AccessProtocol.0 { Host = PluginName = GSIFTP Protocol = gsiftp Path = } AccessProtocol.1 { Host = PluginName = SRM2 Protocol = srm Path = } } StorageZ { BackendType = local ReadAccess = Active WriteAccess = Active AccessProtocols = root AccessProtocols += srm WriteProtocols = root WriteProtocols += srm AccessProtocol.0 { Host = PluginName = ROOT Protocol = root Path = } AccessProtocol.1 { Host = PluginName = SRM2 Protocol = srm Path = } } } } Operations{ Defaults { DataManagement{ AccessProtocols = fakeProto AccessProtocols += root WriteProtocols = srm } } } ''' with open(self.testCfgFileName, 'w') as f: f.write(cfgContent) # SUPER UGLY: one must recreate the CFG objects of gConfigurationData # not to conflict with other tests that might be using a local dirac.cfg gConfigurationData.localCFG=CFG() gConfigurationData.remoteCFG=CFG() gConfigurationData.mergedCFG=CFG() gConfigurationData.generateNewVersion() gConfig = ConfigurationClient(fileToLoadList = [self.testCfgFileName]) #we replace the configuration by our own one. self.seA = StorageElementItem( 'StorageA' ) self.seA.vo = 'lhcb' self.seB = StorageElementItem( 'StorageB' ) self.seB.vo = 'lhcb' self.seC = StorageElementItem( 'StorageC' ) self.seC.vo = 'lhcb' self.seD = StorageElementItem( 'StorageD' ) self.seD.vo = 'lhcb' self.seE = StorageElementItem( 'StorageE' ) self.seE.vo = 'lhcb' self.seX = StorageElementItem( 'StorageX' ) self.seX.vo = 'lhcb' self.seY = StorageElementItem( 'StorageY' ) self.seY.vo = 'lhcb' self.seZ = StorageElementItem( 'StorageZ' ) self.seZ.vo = 'lhcb'
class UploadLogFile(ModuleBase): """ Handle log file uploads in the production jobs """ ############################################################################# def __init__(self): """Module initialization. """ super(UploadLogFile, self).__init__() self.version = __RCSID__ self.log = gLogger.getSubLogger('UploadLogFile') self.productionID = None self.jobID = None self.workflow_commons = None self.logFilePath = "" self.logLFNPath = "" self.logdir = '' self.logSE = StorageElement( self.ops.getValue('/LogStorage/LogSE', 'LogSE') ) self.root = gConfig.getValue('/LocalSite/Root', os.getcwd()) self.logSizeLimit = self.ops.getValue('/LogFiles/SizeLimit', 20 * 1024 * 1024) self.logExtensions = [] self.failoverSEs = gConfig.getValue('/Resources/StorageElementGroups/Tier1-Failover', []) self.experiment = 'CLIC' self.enable = True self.failoverTest = False #flag to put log files to failover by default self.jobID = '' ###################################################################### def applicationSpecificInputs(self): """resolves the module parameters""" self.enable = self.step_commons.get('Enable', self.enable) if not isinstance(self.enable, bool): self.log.warn('Enable flag set to non-boolean value %s, setting to False' %self.enable) self.enable = False if not isinstance(self.failoverTest, bool): self.log.warn('Test failover flag set to non-boolean value %s, setting to False' % self.failoverTest) self.failoverTest = False self.jobID = os.environ.get('JOBID', self.jobID) if self.jobID != 0: self.log.verbose('Found WMS JobID = %s' % self.jobID) else: self.log.info('No WMS JobID found, disabling module via control flag') self.enable = False self.logFilePath = self.workflow_commons.get('LogFilePath', self.logFilePath) self.logLFNPath = self.workflow_commons.get('LogTargetPath', self.logLFNPath) if not (self.logFilePath and self.logLFNPath): self.log.info('LogFilePath parameter not found, creating on the fly') result = getLogPath(self.workflow_commons) if not result['OK']: self.log.error('Could not create LogFilePath', result['Message']) return result self.logFilePath = result['Value']['LogFilePath'][0] self.logLFNPath = result['Value']['LogTargetPath'][0] if not isinstance(self.logFilePath, basestring): self.logFilePath = self.logFilePath[0] if not isinstance(self.logLFNPath, basestring): self.logLFNPath = self.logLFNPath[0] #FIXME: What if experiments change path, pick this up from the CS instead, or find a better way example_file = self.logFilePath if "/ilc/prod/clic" in example_file: self.experiment = "CLIC" elif "/ilc/prod/ilc/sid" in example_file: self.experiment = 'ILC_SID' elif "/ilc/prod/ilc/mc-dbd" in example_file: self.experiment = 'ILC_ILD' else: self.log.warn("Failed to determine experiment, reverting to default: %s" % self.experiment) return S_OK('Parameters resolved') ###################################################################### def execute(self): """ Main execution method """ self.log.info('Initializing %s' % self.version) # Add global reporting tool res = self.resolveInputVariables() if not res['OK']: self.log.error("Failed to resolve input parameters:", res['Message']) self.logWorkingDirectory() self.log.info('Job root is found to be %s' % (self.root)) self.log.info('PRODUCTION_ID = %s, JOB_ID = %s ' % (self.productionID, self.jobID)) self.logdir = os.path.realpath('./job/log/%s/%s' % (self.productionID, self.jobID)) self.log.info('Selected log files will be temporarily stored in %s' % self.logdir) res = self.finalize() return res ############################################################################# def finalize(self): """ finalize method performs final operations after all the job steps were executed. Only production jobs are treated. """ self.log.verbose('Starting UploadLogFile finalize') self.log.debug("LogFilePath: %s" % self.logFilePath) self.log.debug("LogLFNPath: %s" % self.logLFNPath) ########################################## # First determine the files which should be saved self.log.info('Determining the files to be saved in the logs.') resRelevantFiles = self._determineRelevantFiles() if not resRelevantFiles['OK']: self.log.error('Completely failed to select relevant log files.', resRelevantFiles['Message']) return S_OK()#because if the logs are lost, it's not the end of the world. selectedFiles = resRelevantFiles['Value'] if not selectedFiles: self.log.info("No log files selected") return S_OK() self.log.info('The following %s files were selected to be saved:\n%s' % (len(selectedFiles), "\n".join(selectedFiles))) ######################################### # Create a temporary directory containing these files self.log.info('Populating a temporary directory for selected files.') resLogDir = self._populateLogDirectory(selectedFiles) if not resLogDir['OK']: self.log.error('Completely failed to populate temporary log file directory.', resLogDir['Message']) self.setApplicationStatus('Failed To Populate Log Dir') return S_OK()#because if the logs are lost, it's not the end of the world. self.log.info('%s populated with log files.' % self.logdir) if not self.enable: self.log.info('Module is disabled by control flag') return S_OK('Module is disabled by control flag') ######################################### #Make sure all the files in the log directory have the correct permissions result = self._setLogFilePermissions(self.logdir) if not result['OK']: self.log.error('Could not set permissions of log files to 0755 with message:\n%s' % (result['Message'])) ######################################### #Tar all the files resTarLogs = self._tarTheLogFiles() if not resTarLogs['OK']: return S_OK()#because if the logs are lost, it's not the end of the world. tarFileName = resTarLogs['Value']['fileName'] ######################################### # Attempt to upload log tar ball to the LogSE self.log.info('Transferring log tarball to the %s' % self.logSE.name) resTransfer = S_ERROR("Skipping log upload, because failoverTest") tarFileLFN = '%s/%s' % (self.logFilePath, tarFileName) tarFileLocal = os.path.realpath(os.path.join(self.logdir, tarFileName)) if not self.failoverTest: self.log.info('PutFile %s %s %s' % (tarFileLFN, tarFileLocal, self.logSE.name)) resTransfer = self.logSE.putFile({ tarFileLFN : tarFileLocal }) self.log.debug("putFile result: %s" % resTransfer) if not resTransfer['OK']: self.log.error('Completely failed to upload log files to %s, will attempt upload to failover SE' % self.logSE.name, resTransfer['Message']) elif resTransfer['OK'] and len(resTransfer['Value']['Failed']) == 0: self.log.info('Successfully upload log tarball to %s' % self.logSE.name) self.setJobParameter('Log LFN', tarFileLFN) self.log.info('Logs for this job may be retrieved with dirac-ilc-get-prod-log -F %s' % tarFileLFN) return S_OK() else: self.log.error('Completely failed to upload log files to %s, will attempt upload to failover SE' % self.logSE.name, resTransfer['Value']) ######################################### # Recover the logs to a failover storage element #if res['Value'][0]: #i.e. non-zero status # self.log.error('Failed to create tar file from directory','%s %s' % (self.logdir,res['Value'])) # self.setApplicationStatus('Failed To Create Log Tar Dir') # return S_OK()#because if the logs are lost, it's not the end of the world. ############################################################ #Instantiate the failover transfer client with the global request object resFailover = self._tryFailoverTransfer(tarFileName, self.logdir) if not resFailover['Value']: return resFailover ##log files lost, but who cares... self.workflow_commons['Request'] = resFailover['Value']['Request'] uploadedSE = resFailover['Value']['uploadedSE'] res = self._createLogUploadRequest(self.logSE.name, self.logLFNPath, uploadedSE) if not res['OK']: self.log.error('Failed to create failover request', res['Message']) self.setApplicationStatus('Failed To Upload Logs To Failover') else: self.log.info('Successfully created failover request') self.log.info("Request %s" % self._getRequestContainer()) return S_OK() ############################################################################# def _determineRelevantFiles(self): """ The files which are below a configurable size will be stored in the logs. This will typically pick up everything in the working directory minus the output data files. """ logFileExtensions = ['*.txt', '*.log', '*.out', '*.output', '*.xml', '*.sh', '*.info', '*.err','*.root'] self.logExtensions = self.ops.getValue('/LogFiles/%s/Extensions' % self.experiment, []) if self.logExtensions: self.log.info('Using list of log extensions from CS:\n%s' % (', '.join(self.logExtensions))) logFileExtensions = self.logExtensions else: self.log.info('Using default list of log extensions:\n%s' % (', '.join(logFileExtensions))) candidateFiles = [] for ext in logFileExtensions: self.log.debug('Looking at log file wildcard: %s' % ext) globList = glob.glob(ext) for check in globList: if os.path.isfile(check): self.log.debug('Found locally existing log file: %s' % check) candidateFiles.append(check) selectedFiles = [] try: for candidate in candidateFiles: fileSize = os.stat(candidate)[6] if fileSize < self.logSizeLimit: selectedFiles.append(candidate) else: self.log.error('Log file found to be greater than maximum of %s bytes' % self.logSizeLimit, candidate) return S_OK(selectedFiles) except OSError as x: self.log.exception('Exception while determining files to save.', '', str(x)) return S_ERROR('Could not determine log files') ############################################################################# def _populateLogDirectory(self, selectedFiles): """ A temporary directory is created for all the selected files. These files are then copied into this directory before being uploaded """ # Create the temporary directory try: if not os.path.exists(self.logdir): os.makedirs(self.logdir) except OSError as x: self.log.exception('PopulateLogDir: Exception while trying to create directory.', self.logdir, str(x)) return S_ERROR() # Set proper permissions self.log.info('PopulateLogDir: Changing log directory %s permissions to 0755' % self.logdir ) try: os.chmod(self.logdir, 0755) except OSError as x: self.log.error('PopulateLogDir: Could not set logdir permissions to 0755:', '%s (%s)' % ( self.logdir, str(x) ) ) # Populate the temporary directory try: for myfile in selectedFiles: destinationFile = '%s/%s' % (self.logdir, os.path.basename(myfile)) shutil.copy(myfile, destinationFile) except OSError as x: self.log.exception('PopulateLogDir: Exception while trying to copy file.', myfile, str(x)) self.log.info('PopulateLogDir: File %s will be skipped and can be considered lost.' % myfile) # Now verify the contents of our target log dir successfulFiles = os.listdir(self.logdir) self.log.info("PopulateLogDir: successfulFiles %s" % successfulFiles) if len(successfulFiles) == 0: self.log.info('PopulateLogDir: Failed to copy any files to the target directory.') return S_ERROR() else: self.log.info('PopulateLogDir: Prepared %s files in the temporary directory.' % len(successfulFiles)) return S_OK() ############################################################################# def _createLogUploadRequest(self, targetSE, logFileLFN, uploadedSE): """ Set a request to upload job log files from the output sandbox Changed to be similar to LHCb createLogUploadRequest using LHCb LogUpload Request and Removal Request """ self.log.info('Setting log upload request for %s at %s' %(targetSE, logFileLFN)) request = self._getRequestContainer() logUpload = Operation() logUpload.Type = "LogUpload" logUpload.TargetSE = targetSE upFile = File() upFile.LFN = logFileLFN logUpload.addFile( upFile ) logRemoval = Operation() logRemoval.Type = 'RemoveFile' logRemoval.TargetSE = uploadedSE logRemoval.addFile( upFile ) request.addOperation ( logUpload ) request.addOperation ( logRemoval ) self.workflow_commons['Request'] = request return S_OK() ############################################################################# def _setLogFilePermissions(self, logDir): """ Sets the permissions of all the files in the log directory to ensure they are readable. """ try: for toChange in os.listdir(logDir): if not os.path.islink('%s/%s' % (logDir, toChange)): self.log.debug('Changing permissions of %s/%s to 0755' % (logDir, toChange)) os.chmod('%s/%s' % (logDir, toChange), 0755) except OSError as x: self.log.error('Problem changing shared area permissions', str(x)) return S_ERROR(x) return S_OK() ################################################################################ def _tryFailoverTransfer(self, tarFileName, tarFileDir): """tries to upload the log tarBall to the failoverSE and creates moving request""" failoverTransfer = FailoverTransfer(self._getRequestContainer()) ##determine the experiment self.failoverSEs = self.ops.getValue("Production/%s/FailOverSE" % self.experiment, self.failoverSEs) catalogs = self.ops.getValue('Production/%s/Catalogs' % self.experiment, ['FileCatalog', 'LcgFileCatalog']) random.shuffle(self.failoverSEs) self.log.info("Attempting to store file %s to the following SE(s):\n%s" % (tarFileName, ', '.join(self.failoverSEs ))) result = failoverTransfer.transferAndRegisterFile(tarFileName, '%s/%s' % (tarFileDir, tarFileName), self.logLFNPath, self.failoverSEs, fileMetaDict = { "GUID": None }, fileCatalog = catalogs ) if not result['OK']: self.log.error('Failed to upload logs to all destinations') self.setApplicationStatus('Failed To Upload Logs') return S_OK() #because if the logs are lost, it's not the end of the world. #Now after all operations, return potentially modified request object return S_OK( {'Request': failoverTransfer.request, 'uploadedSE': result['Value']['uploadedSE']}) def _tarTheLogFiles(self): """returns S_OK/S_ERROR and puts all the relevantFiles into a tarball""" self.logLFNPath = '%s.gz' % self.logLFNPath tarFileName = os.path.basename(self.logLFNPath) self.log.debug("Log Directory: %s" % self.logdir ) self.log.debug("Log LFNPath: %s" % self.logLFNPath ) self.log.debug("TarFileName: %s" % tarFileName ) start = os.getcwd() os.chdir(self.logdir) logTarFiles = os.listdir(self.logdir) #comm = 'tar czvf %s %s' % (tarFileName,string.join(logTarFiles,' ')) tfile = tarfile.open(tarFileName, "w:gz") for item in logTarFiles: self.log.info("Adding file %s to tarfile %s" %(item, tarFileName)) tfile.add(item) tfile.close() resExists = S_OK() if os.path.exists(tarFileName) else S_ERROR("File was not created") os.chdir(start) self.log.debug("TarFileName: %s" %(tarFileName,)) if resExists['OK']: return S_OK(dict(fileName=tarFileName)) else: self.log.error('Failed to create tar file from directory','%s %s' % (self.logdir, resExists['Message'])) self.setApplicationStatus('Failed To Create Log Tar Dir') return S_ERROR()
class UploadLogFile(ModuleBase): """ Handle log file uploads in the production jobs """ ############################################################################# def __init__(self): """Module initialization. """ super(UploadLogFile, self).__init__() self.version = __RCSID__ self.log = gLogger.getSubLogger('UploadLogFile') self.productionID = None self.jobID = None self.workflow_commons = None self.logFilePath = "" self.logLFNPath = "" self.logdir = '' self.logSE = StorageElement( self.ops.getValue('/LogStorage/LogSE', 'LogSE')) self.root = gConfig.getValue('/LocalSite/Root', os.getcwd()) self.logSizeLimit = self.ops.getValue('/LogFiles/SizeLimit', 20 * 1024 * 1024) self.logExtensions = [] self.failoverSEs = gConfig.getValue( '/Resources/StorageElementGroups/Tier1-Failover', []) self.experiment = 'CLIC' self.enable = True self.failoverTest = False #flag to put log files to failover by default self.jobID = '' ###################################################################### def applicationSpecificInputs(self): """resolves the module parameters""" self.enable = self.step_commons.get('Enable', self.enable) if not isinstance(self.enable, bool): self.log.warn( 'Enable flag set to non-boolean value %s, setting to False' % self.enable) self.enable = False if not isinstance(self.failoverTest, bool): self.log.warn( 'Test failover flag set to non-boolean value %s, setting to False' % self.failoverTest) self.failoverTest = False self.jobID = os.environ.get('JOBID', self.jobID) if self.jobID != 0: self.log.verbose('Found WMS JobID = %s' % self.jobID) else: self.log.info( 'No WMS JobID found, disabling module via control flag') self.enable = False self.logFilePath = self.workflow_commons.get('LogFilePath', self.logFilePath) self.logLFNPath = self.workflow_commons.get('LogTargetPath', self.logLFNPath) if not (self.logFilePath and self.logLFNPath): self.log.info( 'LogFilePath parameter not found, creating on the fly') result = getLogPath(self.workflow_commons) if not result['OK']: self.log.error('Could not create LogFilePath', result['Message']) return result self.logFilePath = result['Value']['LogFilePath'][0] self.logLFNPath = result['Value']['LogTargetPath'][0] if not isinstance(self.logFilePath, basestring): self.logFilePath = self.logFilePath[0] if not isinstance(self.logLFNPath, basestring): self.logLFNPath = self.logLFNPath[0] #FIXME: What if experiments change path, pick this up from the CS instead, or find a better way example_file = self.logFilePath if "/ilc/prod/clic" in example_file: self.experiment = "CLIC" elif "/ilc/prod/ilc/sid" in example_file: self.experiment = 'ILC_SID' elif "/ilc/prod/ilc/mc-dbd" in example_file: self.experiment = 'ILC_ILD' else: self.log.warn( "Failed to determine experiment, reverting to default: %s" % self.experiment) return S_OK('Parameters resolved') ###################################################################### def execute(self): """ Main execution method """ self.log.info('Initializing %s' % self.version) # Add global reporting tool res = self.resolveInputVariables() if not res['OK']: self.log.error("Failed to resolve input parameters:", res['Message']) self.logWorkingDirectory() self.log.info('Job root is found to be %s' % (self.root)) self.log.info('PRODUCTION_ID = %s, JOB_ID = %s ' % (self.productionID, self.jobID)) self.logdir = os.path.realpath('./job/log/%s/%s' % (self.productionID, self.jobID)) self.log.info('Selected log files will be temporarily stored in %s' % self.logdir) res = self.finalize() return res ############################################################################# def finalize(self): """ finalize method performs final operations after all the job steps were executed. Only production jobs are treated. """ self.log.verbose('Starting UploadLogFile finalize') self.log.debug("LogFilePath: %s" % self.logFilePath) self.log.debug("LogLFNPath: %s" % self.logLFNPath) ########################################## # First determine the files which should be saved self.log.info('Determining the files to be saved in the logs.') resRelevantFiles = self._determineRelevantFiles() if not resRelevantFiles['OK']: self.log.error('Completely failed to select relevant log files.', resRelevantFiles['Message']) return S_OK( ) #because if the logs are lost, it's not the end of the world. selectedFiles = resRelevantFiles['Value'] if not selectedFiles: self.log.info("No log files selected") return S_OK() self.log.info('The following %s files were selected to be saved:\n%s' % (len(selectedFiles), "\n".join(selectedFiles))) ######################################### # Create a temporary directory containing these files self.log.info('Populating a temporary directory for selected files.') resLogDir = self._populateLogDirectory(selectedFiles) if not resLogDir['OK']: self.log.error( 'Completely failed to populate temporary log file directory.', resLogDir['Message']) self.setApplicationStatus('Failed To Populate Log Dir') return S_OK( ) #because if the logs are lost, it's not the end of the world. self.log.info('%s populated with log files.' % self.logdir) if not self.enable: self.log.info('Module is disabled by control flag') return S_OK('Module is disabled by control flag') ######################################### #Make sure all the files in the log directory have the correct permissions result = self._setLogFilePermissions(self.logdir) if not result['OK']: self.log.error( 'Could not set permissions of log files to 0755 with message:\n%s' % (result['Message'])) ######################################### #Tar all the files resTarLogs = self._tarTheLogFiles() if not resTarLogs['OK']: return S_OK( ) #because if the logs are lost, it's not the end of the world. tarFileName = resTarLogs['Value']['fileName'] ######################################### # Attempt to upload log tar ball to the LogSE self.log.info('Transferring log tarball to the %s' % self.logSE.name) resTransfer = S_ERROR("Skipping log upload, because failoverTest") tarFileLFN = '%s/%s' % (self.logFilePath, tarFileName) tarFileLocal = os.path.realpath(os.path.join(self.logdir, tarFileName)) if not self.failoverTest: self.log.info('PutFile %s %s %s' % (tarFileLFN, tarFileLocal, self.logSE.name)) resTransfer = self.logSE.putFile({tarFileLFN: tarFileLocal}) self.log.debug("putFile result: %s" % resTransfer) if not resTransfer['OK']: self.log.error( 'Completely failed to upload log files to %s, will attempt upload to failover SE' % self.logSE.name, resTransfer['Message']) elif resTransfer['OK'] and len( resTransfer['Value']['Failed']) == 0: self.log.info('Successfully upload log tarball to %s' % self.logSE.name) self.setJobParameter('Log LFN', tarFileLFN) self.log.info( 'Logs for this job may be retrieved with dirac-ilc-get-prod-log -F %s' % tarFileLFN) return S_OK() else: self.log.error( 'Completely failed to upload log files to %s, will attempt upload to failover SE' % self.logSE.name, resTransfer['Value']) ######################################### # Recover the logs to a failover storage element #if res['Value'][0]: #i.e. non-zero status # self.log.error('Failed to create tar file from directory','%s %s' % (self.logdir,res['Value'])) # self.setApplicationStatus('Failed To Create Log Tar Dir') # return S_OK()#because if the logs are lost, it's not the end of the world. ############################################################ #Instantiate the failover transfer client with the global request object resFailover = self._tryFailoverTransfer(tarFileName, self.logdir) if not resFailover['Value']: return resFailover ##log files lost, but who cares... self.workflow_commons['Request'] = resFailover['Value']['Request'] uploadedSE = resFailover['Value']['uploadedSE'] res = self._createLogUploadRequest(self.logSE.name, self.logLFNPath, uploadedSE) if not res['OK']: self.log.error('Failed to create failover request', res['Message']) self.setApplicationStatus('Failed To Upload Logs To Failover') else: self.log.info('Successfully created failover request') self.log.info("Request %s" % self._getRequestContainer()) return S_OK() ############################################################################# def _determineRelevantFiles(self): """ The files which are below a configurable size will be stored in the logs. This will typically pick up everything in the working directory minus the output data files. """ logFileExtensions = [ '*.txt', '*.log', '*.out', '*.output', '*.xml', '*.sh', '*.info', '*.err', '*.root' ] self.logExtensions = self.ops.getValue( '/LogFiles/%s/Extensions' % self.experiment, []) if self.logExtensions: self.log.info('Using list of log extensions from CS:\n%s' % (', '.join(self.logExtensions))) logFileExtensions = self.logExtensions else: self.log.info('Using default list of log extensions:\n%s' % (', '.join(logFileExtensions))) candidateFiles = [] for ext in logFileExtensions: self.log.debug('Looking at log file wildcard: %s' % ext) globList = glob.glob(ext) for check in globList: if os.path.isfile(check): self.log.debug('Found locally existing log file: %s' % check) candidateFiles.append(check) selectedFiles = [] try: for candidate in candidateFiles: fileSize = os.stat(candidate)[6] if fileSize < self.logSizeLimit: selectedFiles.append(candidate) else: self.log.error( 'Log file found to be greater than maximum of %s bytes' % self.logSizeLimit, candidate) return S_OK(selectedFiles) except OSError as x: self.log.exception('Exception while determining files to save.', '', str(x)) return S_ERROR('Could not determine log files') ############################################################################# def _populateLogDirectory(self, selectedFiles): """ A temporary directory is created for all the selected files. These files are then copied into this directory before being uploaded """ # Create the temporary directory try: if not os.path.exists(self.logdir): os.makedirs(self.logdir) except OSError as x: self.log.exception( 'PopulateLogDir: Exception while trying to create directory.', self.logdir, str(x)) return S_ERROR() # Set proper permissions self.log.info( 'PopulateLogDir: Changing log directory %s permissions to 0755' % self.logdir) try: os.chmod(self.logdir, 0755) except OSError as x: self.log.error( 'PopulateLogDir: Could not set logdir permissions to 0755:', '%s (%s)' % (self.logdir, str(x))) # Populate the temporary directory try: for myfile in selectedFiles: destinationFile = '%s/%s' % (self.logdir, os.path.basename(myfile)) shutil.copy(myfile, destinationFile) except OSError as x: self.log.exception( 'PopulateLogDir: Exception while trying to copy file.', myfile, str(x)) self.log.info( 'PopulateLogDir: File %s will be skipped and can be considered lost.' % myfile) # Now verify the contents of our target log dir successfulFiles = os.listdir(self.logdir) self.log.info("PopulateLogDir: successfulFiles %s" % successfulFiles) if len(successfulFiles) == 0: self.log.info( 'PopulateLogDir: Failed to copy any files to the target directory.' ) return S_ERROR() else: self.log.info( 'PopulateLogDir: Prepared %s files in the temporary directory.' % len(successfulFiles)) return S_OK() ############################################################################# def _createLogUploadRequest(self, targetSE, logFileLFN, uploadedSE): """ Set a request to upload job log files from the output sandbox Changed to be similar to LHCb createLogUploadRequest using LHCb LogUpload Request and Removal Request """ self.log.info('Setting log upload request for %s at %s' % (targetSE, logFileLFN)) request = self._getRequestContainer() logUpload = Operation() logUpload.Type = "LogUpload" logUpload.TargetSE = targetSE upFile = File() upFile.LFN = logFileLFN logUpload.addFile(upFile) logRemoval = Operation() logRemoval.Type = 'RemoveFile' logRemoval.TargetSE = uploadedSE logRemoval.addFile(upFile) request.addOperation(logUpload) request.addOperation(logRemoval) self.workflow_commons['Request'] = request return S_OK() ############################################################################# def _setLogFilePermissions(self, logDir): """ Sets the permissions of all the files in the log directory to ensure they are readable. """ try: for toChange in os.listdir(logDir): if not os.path.islink('%s/%s' % (logDir, toChange)): self.log.debug('Changing permissions of %s/%s to 0755' % (logDir, toChange)) os.chmod('%s/%s' % (logDir, toChange), 0755) except OSError as x: self.log.error('Problem changing shared area permissions', str(x)) return S_ERROR(x) return S_OK() ################################################################################ def _tryFailoverTransfer(self, tarFileName, tarFileDir): """tries to upload the log tarBall to the failoverSE and creates moving request""" failoverTransfer = FailoverTransfer(self._getRequestContainer()) ##determine the experiment self.failoverSEs = self.ops.getValue( "Production/%s/FailOverSE" % self.experiment, self.failoverSEs) catalogs = self.ops.getValue( 'Production/%s/Catalogs' % self.experiment, ['FileCatalog', 'LcgFileCatalog']) random.shuffle(self.failoverSEs) self.log.info( "Attempting to store file %s to the following SE(s):\n%s" % (tarFileName, ', '.join(self.failoverSEs))) result = failoverTransfer.transferAndRegisterFile( tarFileName, '%s/%s' % (tarFileDir, tarFileName), self.logLFNPath, self.failoverSEs, fileMetaDict={"GUID": None}, fileCatalog=catalogs) if not result['OK']: self.log.error('Failed to upload logs to all destinations') self.setApplicationStatus('Failed To Upload Logs') return S_OK( ) #because if the logs are lost, it's not the end of the world. #Now after all operations, return potentially modified request object return S_OK({ 'Request': failoverTransfer.request, 'uploadedSE': result['Value']['uploadedSE'] }) def _tarTheLogFiles(self): """returns S_OK/S_ERROR and puts all the relevantFiles into a tarball""" self.logLFNPath = '%s.gz' % self.logLFNPath tarFileName = os.path.basename(self.logLFNPath) self.log.debug("Log Directory: %s" % self.logdir) self.log.debug("Log LFNPath: %s" % self.logLFNPath) self.log.debug("TarFileName: %s" % tarFileName) start = os.getcwd() os.chdir(self.logdir) logTarFiles = os.listdir(self.logdir) #comm = 'tar czvf %s %s' % (tarFileName,string.join(logTarFiles,' ')) tfile = tarfile.open(tarFileName, "w:gz") for item in logTarFiles: self.log.info("Adding file %s to tarfile %s" % (item, tarFileName)) tfile.add(item) tfile.close() resExists = S_OK() if os.path.exists(tarFileName) else S_ERROR( "File was not created") os.chdir(start) self.log.debug("TarFileName: %s" % (tarFileName, )) if resExists['OK']: return S_OK(dict(fileName=tarFileName)) else: self.log.error('Failed to create tar file from directory', '%s %s' % (self.logdir, resExists['Message'])) self.setApplicationStatus('Failed To Create Log Tar Dir') return S_ERROR()
class TestBase( unittest.TestCase ): """ Base test class. Defines all the method to test """ @mock.patch( 'DIRAC.Resources.Storage.StorageFactory.StorageFactory._getConfigStorageName', side_effect = mock_StorageFactory_getConfigStorageName ) @mock.patch( 'DIRAC.Resources.Storage.StorageFactory.StorageFactory._getConfigStorageOptions', side_effect = mock_StorageFactory_getConfigStorageOptions ) @mock.patch( 'DIRAC.Resources.Storage.StorageFactory.StorageFactory._getConfigStorageProtocols', side_effect = mock_StorageFactory_getConfigStorageProtocols ) @mock.patch( 'DIRAC.Resources.Storage.StorageFactory.StorageFactory._StorageFactory__generateStorageObject', side_effect = mock_StorageFactory_generateStorageObject ) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value = S_OK( True ) ) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value = None ) # Don't send accounting @mock.patch( 'DIRAC.Resources.Storage.StorageElement.DMSHelpers', side_effect = fake_DMSHelpers ) def setUp( self, _mk_getConfigStorageName, _mk_getConfigStorageOptions, _mk_getConfigStorageProtocols, _mk_generateStorage, _mk_isLocalSE, _mk_addAccountingOperation, _mk_dmsHelpers ): self.seA = StorageElementItem( 'StorageA' ) self.seA.vo = 'lhcb' self.seB = StorageElementItem( 'StorageB' ) self.seB.vo = 'lhcb' self.seC = StorageElementItem( 'StorageC' ) self.seC.vo = 'lhcb' self.seD = StorageElementItem( 'StorageD' ) self.seD.vo = 'lhcb' self.seE = StorageElementItem( 'StorageE' ) self.seE.vo = 'lhcb' def tearDown( self ): pass @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value = S_OK( True ) ) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value = None ) # Don't send accounting def test_01_negociateProtocolWithOtherSE( self, mk_isLocalSE, mk_addAccounting ): """Testing negotiation algorithm""" # Find common protocol between SRM2 and File res = self.seA.negociateProtocolWithOtherSE( self.seB ) self.assertTrue( res['OK'], res ) self.assertEqual( res['Value'], ['file'] ) # Find common protocol between File and SRM@ res = self.seB.negociateProtocolWithOtherSE( self.seA ) self.assertTrue( res['OK'], res ) self.assertEqual( res['Value'], ['file'] ) # Find common protocol between XROOT and File # Nothing goes from xroot to file res = self.seA.negociateProtocolWithOtherSE( self.seC ) self.assertTrue( res['OK'], res ) self.assertEqual( res['Value'], [] ) # Find common protocol between File and XROOT res = self.seC.negociateProtocolWithOtherSE( self.seA ) self.assertTrue( res['OK'], res ) self.assertEqual( res['Value'], ['file'] ) # Find common protocol between File and File res = self.seA.negociateProtocolWithOtherSE( self.seA ) self.assertTrue( res['OK'], res ) self.assertEqual( res['Value'], ['file'] ) # Find common protocol between SRM and SRM res = self.seB.negociateProtocolWithOtherSE( self.seB ) self.assertTrue( res['OK'], res ) self.assertEqual( sorted( res['Value'] ), sorted( ['file', 'root', 'srm'] ) ) # Find common protocol between SRM and XROOT res = self.seC.negociateProtocolWithOtherSE( self.seB ) self.assertTrue( res['OK'], res ) self.assertEqual( sorted( res['Value'] ), sorted( ['root', 'file'] ) ) # Find common protocol between XROOT and SRM res = self.seC.negociateProtocolWithOtherSE( self.seB ) self.assertTrue( res['OK'], res ) self.assertEqual( sorted( res['Value'] ), sorted( ['root', 'file'] ) ) # Testing restrictions res = self.seC.negociateProtocolWithOtherSE( self.seB, protocols = ['file'] ) self.assertTrue( res['OK'], res ) self.assertEqual( sorted( res['Value'] ), ['file'] ) res = self.seC.negociateProtocolWithOtherSE( self.seB, protocols = ['nonexisting'] ) self.assertTrue( res['OK'], res ) self.assertEqual( res['Value'], [] ) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value = S_OK( True ) ) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value = None ) # Don't send accounting def test_02_followOrder( self, _mk_isLocalSE, _mk_addAccounting ): """Testing If the order of preferred protocols is respected""" for permutation in itertools.permutations( ['srm', 'file', 'root', 'nonexisting'] ): permuList = list( permutation ) # Don't get tricked ! remove cannot be put # after the conversion, because it is inplace modification permuList.remove( 'nonexisting' ) res = self.seD.negociateProtocolWithOtherSE( self.seD, protocols = permutation ) self.assertTrue( res['OK'], res ) self.assertEqual( res['Value'], permuList ) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value = S_OK( True ) ) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value = None ) # Don't send accounting def test_03_multiProtocolThirdParty( self, _mk_isLocalSE, _mk_addAccounting ): """ Test case for storages with several protocols Here comes the fun :-) Suppose we have endpoints that we can read in root, but cannot write If we have root in the accessProtocols and thirdPartyProtocols lists but not in the writeProtocols, we should get a root url to read, and write with SRM We reproduce here the behavior of DataManager.replicate """ thirdPartyProtocols = ['root', 'srm'] lfn = '/lhcb/fake/lfn' res = self.seD.negociateProtocolWithOtherSE( self.seD, protocols = thirdPartyProtocols ) self.assertTrue( res['OK'], res ) self.assertEqual( res['Value'] , thirdPartyProtocols ) # Only the XROOT plugin here implements the geTransportURL # that returns what we want, so we know that # if the return is successful, it is because of the XROOT res = self.seD.getURL( lfn, protocol = res['Value'] ) self.assertTrue( res['OK'], res ) self.assertTrue( lfn in res['Value']['Successful'], res ) srcUrl = res['Value']['Successful'][lfn] self.assertEqual( srcUrl, "root:getTransportURL" ) # Only the SRM2 plugin here implements the putFile method # so if we get a success here, it means that we used the SRM plugin res = self.seD.replicateFile( {lfn:srcUrl}, sourceSize = 123, inputProtocol = 'root' ) self.assertTrue( res['OK'], res ) self.assertTrue( lfn in res['Value']['Successful'], res ) self.assertEqual( res['Value']['Successful'][lfn], "srm:putFile" ) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value = S_OK( True ) ) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value = None ) # Don't send accounting def test_04_thirdPartyLocalWrite( self, _mk_isLocalSE, _mk_addAccounting ): """ Test case for storages with several protocols Here, we locally define the write protocol to be root and srm So we should be able to do everything with XROOT plugin """ thirdPartyProtocols = ['root', 'srm'] lfn = '/lhcb/fake/lfn' res = self.seE.negociateProtocolWithOtherSE( self.seE, protocols = thirdPartyProtocols ) self.assertTrue( res['OK'], res ) self.assertEqual( res['Value'] , thirdPartyProtocols ) res = self.seE.getURL( lfn, protocol = res['Value'] ) self.assertTrue( res['OK'], res ) self.assertTrue( lfn in res['Value']['Successful'], res ) srcUrl = res['Value']['Successful'][lfn] self.assertEqual( srcUrl, "root:getTransportURL" ) res = self.seE.replicateFile( {lfn:srcUrl}, sourceSize = 123, inputProtocol = 'root' ) self.assertTrue( res['OK'], res ) self.assertTrue( lfn in res['Value']['Successful'], res ) self.assertEqual( res['Value']['Successful'][lfn], "root:putFile" ) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value = S_OK( True ) ) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value = None ) # Don't send accounting def test_05_thirdPartyMix( self, _mk_isLocalSE, _mk_addAccounting ): """ Test case for storages with several protocols Here, we locally define the write protocol for the destination, so it should all go directly through the XROOT plugin """ thirdPartyProtocols = ['root', 'srm'] lfn = '/lhcb/fake/lfn' res = self.seE.negociateProtocolWithOtherSE( self.seD, protocols = thirdPartyProtocols ) self.assertTrue( res['OK'], res ) self.assertEqual( res['Value'] , thirdPartyProtocols ) res = self.seD.getURL( lfn, protocol = res['Value'] ) self.assertTrue( res['OK'], res ) self.assertTrue( lfn in res['Value']['Successful'], res ) srcUrl = res['Value']['Successful'][lfn] self.assertEqual( srcUrl, "root:getTransportURL" ) res = self.seE.replicateFile( {lfn:srcUrl}, sourceSize = 123, inputProtocol = 'root' ) self.assertTrue( res['OK'], res ) self.assertTrue( lfn in res['Value']['Successful'], res ) self.assertEqual( res['Value']['Successful'][lfn], "root:putFile" ) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value = S_OK( True ) ) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value = None ) # Don't send accounting def test_06_thirdPartyMixOpposite( self, _mk_isLocalSE, _mk_addAccounting ): """ Test case for storages with several protocols Here, we locally define the write protocol for the source, so it should get the source directly using XROOT, and perform the put using SRM """ thirdPartyProtocols = ['root', 'srm'] lfn = '/lhcb/fake/lfn' res = self.seD.negociateProtocolWithOtherSE( self.seE, protocols = thirdPartyProtocols ) self.assertTrue( res['OK'], res ) self.assertEqual( res['Value'] , thirdPartyProtocols ) res = self.seE.getURL( lfn, protocol = res['Value'] ) self.assertTrue( res['OK'], res ) self.assertTrue( lfn in res['Value']['Successful'], res ) srcUrl = res['Value']['Successful'][lfn] self.assertEqual( srcUrl, "root:getTransportURL" ) res = self.seD.replicateFile( {lfn:srcUrl}, sourceSize = 123, inputProtocol = 'root' ) self.assertTrue( res['OK'], res ) self.assertTrue( lfn in res['Value']['Successful'], res ) self.assertEqual( res['Value']['Successful'][lfn], "srm:putFile" ) @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE', return_value = S_OK( True ) ) # Pretend it's local @mock.patch( 'DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation', return_value = None ) # Don't send accounting def test_07_multiProtocolSrmOnly( self, _mk_isLocalSE, _mk_addAccounting ): """ Test case for storages with several protocols Here comes the fun :-) Suppose we have endpoints that we can read in root, but cannot write If we have root in the accessProtocols and thirdPartyProtocols lists but not in the writeProtocols, we should get a root url to read, and write with SRM We reproduce here the behavior of DataManager.replicate """ thirdPartyProtocols = [ 'srm'] print "negociate" lfn = '/lhcb/fake/lfn' res = self.seD.negociateProtocolWithOtherSE( self.seD, protocols = thirdPartyProtocols ) self.assertTrue( res['OK'], res ) self.assertEqual( res['Value'] , thirdPartyProtocols ) print "get source url" res = self.seD.getURL( lfn, protocol = res['Value'] ) self.assertTrue( res['OK'], res ) self.assertTrue( lfn in res['Value']['Successful'], res ) srcUrl = res['Value']['Successful'][lfn] self.assertEqual( srcUrl, "srm:getTransportURL" ) print "replicate" res = self.seD.replicateFile( {lfn:srcUrl}, sourceSize = 123, inputProtocol = 'srm' ) self.assertTrue( res['OK'], res ) self.assertTrue( lfn in res['Value']['Successful'], res ) self.assertEqual( res['Value']['Successful'][lfn], "srm:putFile" )