def test_putRemoveFile( self ): print('\n\n#########################################################' '################\n\n\t\t\tPut and Remove test\n') # First test that we are able to determine whether the file sizes of the transfer don't match srcFile = '/etc/group' fileSize = 10 #This is a made up value testFileName = 'testFile.%s' % time.time() destFile = self.storage.getCurrentURL( testFileName )['Value'] fileTuple = ( srcFile, destFile, fileSize ) failedPutFileRes = self.storage.putFile( fileTuple ) # Now make sure that we can actually upload a file properly fileSize = getSize( srcFile ) fileTuple = ( srcFile, destFile, fileSize ) putFileRes = self.storage.putFile( fileTuple ) # Make sure we are able to remove the file removeFileRes = self.storage.removeFile( destFile ) # Check the failed put file operation self.assertTrue( failedPutFileRes['OK'] ) self.assertTrue( destFile in failedPutFileRes['Value']['Failed'] ) expectedError = 'RFIOStorage.putFile: Source and destination file sizes do not match.' self.assertEqual( failedPutFileRes['Value']['Failed'][destFile], expectedError ) # Check the successful put file operation self.assertTrue( putFileRes['OK'] ) self.assertTrue( destFile in putFileRes['Value']['Successful'] ) # Check the remove file operation self.assertTrue( removeFileRes['OK'] ) self.assertTrue( destFile in removeFileRes['Value']['Successful'] )
def test_putFilegetTransportURL( self ): print '\n\n#########################################################################\n\n\t\t\tGet tURL test\n' # First upload a file to the storage srcFile = '/etc/group' fileSize = getSize( srcFile ) testFileName = 'testFile.%s' % time.time() destFile = self.storage.getCurrentURL( testFileName )['Value'] fileTuple = ( srcFile, destFile, fileSize ) putFileRes = self.storage.putFile( fileTuple ) #Check that we can get a turl getTurlRes = self.storage.getTransportURL( destFile, ['dcap', 'gsidcap'] ) # Clean up the remote mess removeFileRes = self.storage.removeFile( destFile ) # Try and get a turl for a non existant file failedGetTurlRes = self.storage.getTransportURL( destFile, ['dcap', 'gsidcap'] ) # Check the put file operation print putFileRes self.assert_( putFileRes['OK'] ) self.assert_( putFileRes['Value']['Successful'].has_key( destFile ) ) # check the get turl operation print getTurlRes, destFile self.assert_( getTurlRes['OK'] ) self.assert_( getTurlRes['Value']['Successful'].has_key( destFile ) ) # check the remove file operation self.assert_( removeFileRes['OK'] ) self.assert_( removeFileRes['Value']['Successful'].has_key( destFile ) ) #Check the get turl with non existant file operation self.assert_( failedGetTurlRes['OK'] ) self.assert_( failedGetTurlRes['Value']['Failed'].has_key( destFile ) ) expectedError = "RFIOStorage.getTransportURL: File does not exist." self.assertEqual( failedGetTurlRes['Value']['Failed'][destFile], expectedError )
def test_putIsFile( self ): print '\n\n#########################################################################\n\n\t\t\tIs file test\n' # First upload a file to the storage srcFile = '/etc/group' fileSize = getSize( srcFile ) testFileName = 'testFile.%s' % time.time() destFile = self.storage.getCurrentURL( testFileName )['Value'] fileTuple = ( srcFile, destFile, fileSize ) putFileRes = self.storage.putFile( fileTuple ) # Check we are able to determine that it is a file isFileRes = self.storage.isFile( destFile ) # Clean up the remote mess removeFileRes = self.storage.removeFile( destFile ) # Check that everything isn't a file destDir = os.path.dirname( destFile ) failedIsFileRes = self.storage.isFile( destDir ) # Check the put file operation self.assert_( putFileRes['OK'] ) self.assert_( putFileRes['Value']['Successful'].has_key( destFile ) ) # Check the is file operation self.assert_( isFileRes['OK'] ) self.assert_( isFileRes['Value']['Successful'].has_key( destFile ) ) self.assert_( isFileRes['Value']['Successful'][destFile] ) # check the remove file operation self.assert_( removeFileRes['OK'] ) self.assert_( removeFileRes['Value']['Successful'].has_key( destFile ) ) # Check that the directory is not a file self.assert_( failedIsFileRes['OK'] ) self.assert_( failedIsFileRes['Value']['Successful'].has_key( destDir ) ) self.assertFalse( failedIsFileRes['Value']['Successful'][destDir] )
def test_putExistsFile(self): print '\n\n#########################################################################\n\n\t\t\tExists test\n' # First upload a file to the storage srcFile = '/etc/group' srcFileSize = getSize(srcFile) testFileName = 'testFile.%s' % time.time() remoteFile = self.storage.getCurrentURL(testFileName)['Value'] fileDict = {remoteFile: srcFile} putFileRes = self.storage.putFile(fileDict) # Then get the file's existance existsFileRes = self.storage.exists(remoteFile) # Now remove the file removeFileRes = self.storage.removeFile(remoteFile) # Check again that the file exists failedExistRes = self.storage.exists(remoteFile) # Check the put file operation self.assert_(putFileRes['OK']) self.assert_(putFileRes['Value']['Successful'].has_key(remoteFile)) self.assertEqual(putFileRes['Value']['Successful'][remoteFile], srcFileSize) # Check the exists operation self.assert_(existsFileRes['OK']) self.assert_(existsFileRes['Value']['Successful'].has_key(remoteFile)) self.assert_(existsFileRes['Value']['Successful'][remoteFile]) # Check the removal operation self.assert_(removeFileRes['OK']) self.assert_(removeFileRes['Value']['Successful'].has_key(remoteFile)) # Check the failed exists operation self.assert_(failedExistRes['OK']) self.assert_(failedExistRes['Value']['Successful'].has_key(remoteFile)) self.assertFalse(failedExistRes['Value']['Successful'][remoteFile])
def __putFile(self, src_file, dest_url): res = pfnparse(src_file) if not res['OK']: return res localCache = False srcDict = res['Value'] if srcDict['Protocol'] in ['dips', 'dip']: localCache = True srcSEURL = srcDict['Protocol'] + '://' + srcDict[ 'Host'] + ':' + srcDict['Port'] + srcDict['WSUrl'] transferClient = TransferClient(srcSEURL) res = transferClient.receiveFile( srcDict['FileName'], os.path.join(srcDict['Path'], srcDict['FileName'])) if not res['OK']: return res src_file = srcDict['FileName'] if not os.path.exists(src_file): errStr = "DIPStorage.__putFile: The source local file does not exist." gLogger.error(errStr, src_file) return S_ERROR(errStr) sourceSize = getSize(src_file) if sourceSize == -1: errStr = "DIPStorage.__putFile: Failed to get file size." gLogger.error(errStr, src_file) return S_ERROR(errStr) transferClient = TransferClient(self.url) res = transferClient.sendFile(src_file, dest_url, token=self.checkSum) if localCache: os.unlink(src_file) if res['OK']: return S_OK(sourceSize) else: return res
def test_putRemoveFile( self ): print '\n\n#########################################################################\n\n\t\t\tPut and Remove test\n' # First test that we are able to determine whether the file sizes of the transfer don't match srcFile = '/etc/group' fileSize = 10 #This is a made up value testFileName = 'testFile.%s' % time.time() destFile = self.storage.getCurrentURL( testFileName )['Value'] fileTuple = ( srcFile, destFile, fileSize ) failedPutFileRes = self.storage.putFile( fileTuple ) # Now make sure that we can actually upload a file properly fileSize = getSize( srcFile ) fileTuple = ( srcFile, destFile, fileSize ) putFileRes = self.storage.putFile( fileTuple ) # Make sure we are able to remove the file removeFileRes = self.storage.removeFile( destFile ) # Check the failed put file operation self.assert_( failedPutFileRes['OK'] ) self.assert_( failedPutFileRes['Value']['Failed'].has_key( destFile ) ) expectedError = 'RFIOStorage.putFile: Source and destination file sizes do not match.' self.assertEqual( failedPutFileRes['Value']['Failed'][destFile], expectedError ) # Check the successful put file operation self.assert_( putFileRes['OK'] ) self.assert_( putFileRes['Value']['Successful'].has_key( destFile ) ) # Check the remove file operation self.assert_( removeFileRes['OK'] ) self.assert_( removeFileRes['Value']['Successful'].has_key( destFile ) )
def test_putGetFile( self ): print '\n\n#########################################################################\n\n\t\t\tPut and Get test\n' # First upload a file to the storage srcFile = '/etc/group' srcFileSize = getSize( srcFile ) testFileName = 'testFile.%s' % time.time() remoteFile = self.storage.getCurrentURL( testFileName )['Value'] fileDict = {remoteFile:srcFile} putFileRes = self.storage.putFile( fileDict ) # Then make sure we can get a local copy of the file getFileRes = self.storage.getFile( remoteFile ) # Cleanup the remote mess removeFileRes = self.storage.removeFile( remoteFile ) #Cleanup the mess locally os.remove( testFileName ) # Check the put operation self.assert_( putFileRes['OK'] ) self.assert_( putFileRes['Value']['Successful'].has_key( remoteFile ) ) self.assertEqual( putFileRes['Value']['Successful'][remoteFile], srcFileSize ) # Check the get operation self.assert_( getFileRes['OK'] ) self.assert_( getFileRes['Value']['Successful'].has_key( remoteFile ) ) self.assertEqual( getFileRes['Value']['Successful'][remoteFile], srcFileSize ) # Check the remove operation self.assert_( removeFileRes['OK'] ) self.assert_( removeFileRes['Value']['Successful'].has_key( remoteFile ) ) self.assert_( removeFileRes['Value']['Successful'][remoteFile] )
def test_putIsFile(self): print '\n\n#########################################################################\n\n\t\t\tIs file test\n' # First upload a file to the storage srcFile = '/etc/group' srcFileSize = getSize(srcFile) testFileName = 'testFile.%s' % time.time() remoteFile = self.storage.getCurrentURL(testFileName)['Value'] fileDict = {remoteFile: srcFile} putFileRes = self.storage.putFile(fileDict) # Check we are able to determine that it is a file isFileRes = self.storage.isFile(remoteFile) # Clean up the remote mess removeFileRes = self.storage.removeFile(remoteFile) # Check that everything isn't a file remoteDir = os.path.dirname(remoteFile) failedIsFileRes = self.storage.isFile(remoteDir) # Check the put file operation self.assert_(putFileRes['OK']) self.assert_(putFileRes['Value']['Successful'].has_key(remoteFile)) self.assert_(putFileRes['Value']['Successful'][remoteFile]) self.assertEqual(putFileRes['Value']['Successful'][remoteFile], srcFileSize) # Check the is file operation self.assert_(isFileRes['OK']) self.assert_(isFileRes['Value']['Successful'].has_key(remoteFile)) self.assert_(isFileRes['Value']['Successful'][remoteFile]) # check the remove file operation self.assert_(removeFileRes['OK']) self.assert_(removeFileRes['Value']['Successful'].has_key(remoteFile)) # Check that the directory is not a file self.assert_(failedIsFileRes['OK']) self.assert_(failedIsFileRes['Value']['Successful'].has_key(remoteDir)) self.assertFalse(failedIsFileRes['Value']['Successful'][remoteDir])
def test_putExistsFile( self ): print('\n\n#########################################################' '################\n\n\t\t\tExists test\n') # First upload a file to the storage srcFile = '/etc/group' fileSize = getSize( srcFile ) testFileName = 'testFile.%s' % time.time() destFile = self.storage.getCurrentURL( testFileName )['Value'] fileTuple = ( srcFile, destFile, fileSize ) putFileRes = self.storage.putFile( fileTuple ) # Then get the file's existance existsFileRes = self.storage.exists( destFile ) # Now remove the file removeFileRes = self.storage.removeFile( destFile ) # Check again that the file exists failedExistRes = self.storage.exists( destFile ) # Check the put file operation self.assertTrue( putFileRes['OK'] ) self.assertTrue( destFile in putFileRes['Value']['Successful'] ) # Check the exists operation self.assertTrue( existsFileRes['OK'] ) self.assertTrue( destFile in existsFileRes['Value']['Successful'] ) self.assertTrue( existsFileRes['Value']['Successful'][destFile] ) # Check the removal operation self.assertTrue( removeFileRes['OK'] ) self.assertTrue( destFile in removeFileRes['Value']['Successful'] ) # Check the failed exists operation self.assertTrue( failedExistRes['OK'] ) self.assertTrue( destFile in failedExistRes['Value']['Successful'] ) self.assertFalse( failedExistRes['Value']['Successful'][destFile] )
def test_putPinRelease( self ): print '\n\n#########################################################################\n\n\t\t\tPin and Release test\n' # First upload a file to the storage srcFile = '/etc/group' srcFileSize = getSize( srcFile ) testFileName = 'testFile.%s' % time.time() remoteFile = self.storage.getCurrentURL( testFileName )['Value'] fileDict = {remoteFile:srcFile} putFileRes = self.storage.putFile( fileDict ) # Check that we can pin the file pinFileRes = self.storage.pinFile( remoteFile ) srmID = '' if pinFileRes['OK']: if pinFileRes['Value']['Successful'].has_key( remoteFile ): srmID = pinFileRes['Value']['Successful'][remoteFile] # Check that we can release the file releaseFileRes = self.storage.releaseFile( {remoteFile:srmID} ) # Clean up the mess removeFileRes = self.storage.removeFile( remoteFile ) # Check the put file operation self.assert_( putFileRes['OK'] ) self.assert_( putFileRes['Value']['Successful'].has_key( remoteFile ) ) self.assert_( putFileRes['Value']['Successful'][remoteFile] ) self.assertEqual( putFileRes['Value']['Successful'][remoteFile], srcFileSize ) # Check the pin file operation self.assert_( pinFileRes['OK'] ) self.assert_( pinFileRes['Value']['Successful'].has_key( remoteFile ) ) self.assert_( type( pinFileRes['Value']['Successful'][remoteFile] ) in StringTypes ) # Check the release file operation self.assert_( releaseFileRes['OK'] ) self.assert_( releaseFileRes['Value']['Successful'].has_key( remoteFile ) ) # check the remove file operation self.assert_( removeFileRes['OK'] ) self.assert_( removeFileRes['Value']['Successful'].has_key( remoteFile ) )
def test_putFilegetTransportURL(self): print("\n\n#########################################################" "################\n\n\t\t\tGet tURL test\n") # First upload a file to the storage srcFile = "/etc/group" srcFileSize = getSize(srcFile) testFileName = "testFile.%s" % time.time() remoteFile = self.storage.getCurrentURL(testFileName)["Value"] fileDict = {remoteFile: srcFile} putFileRes = self.storage.putFile(fileDict) # Check that we can get a turl getTurlRes = self.storage.getTransportURL(remoteFile) # Clean up the remote mess removeFileRes = self.storage.removeFile(remoteFile) # Try and get a turl for a non existant file failedGetTurlRes = self.storage.getTransportURL(remoteFile) # Check the put file operation self.assertTrue(putFileRes["OK"]) self.assertTrue(remoteFile in putFileRes["Value"]["Successful"]) self.assertTrue(putFileRes["Value"]["Successful"][remoteFile]) self.assertEqual(putFileRes["Value"]["Successful"][remoteFile], srcFileSize) # check the get turl operation self.assertTrue(getTurlRes["OK"]) self.assertTrue(remoteFile in getTurlRes["Value"]["Successful"]) # check the remove file operation self.assertTrue(removeFileRes["OK"]) self.assertTrue(remoteFile in removeFileRes["Value"]["Successful"]) # Check the get turl with non existant file operation self.assertTrue(failedGetTurlRes["OK"]) self.assertTrue(remoteFile in failedGetTurlRes["Value"]["Failed"]) expectedError = "File does not exist" self.assertTrue( expectedError in failedGetTurlRes["Value"]["Failed"][remoteFile])
def test_putFilegetTransportURL( self ): print '\n\n#########################################################################\n\n\t\t\tGet tURL test\n' # First upload a file to the storage srcFile = '/etc/group' srcFileSize = getSize( srcFile ) testFileName = 'testFile.%s' % time.time() remoteFile = self.storage.getCurrentURL( testFileName )['Value'] fileDict = {remoteFile:srcFile} putFileRes = self.storage.putFile( fileDict ) #Check that we can get a turl getTurlRes = self.storage.getTransportURL( remoteFile ) # Clean up the remote mess removeFileRes = self.storage.removeFile( remoteFile ) # Try and get a turl for a non existant file failedGetTurlRes = self.storage.getTransportURL( remoteFile ) # Check the put file operation self.assert_( putFileRes['OK'] ) self.assert_( putFileRes['Value']['Successful'].has_key( remoteFile ) ) self.assert_( putFileRes['Value']['Successful'][remoteFile] ) self.assertEqual( putFileRes['Value']['Successful'][remoteFile], srcFileSize ) # check the get turl operation self.assert_( getTurlRes['OK'] ) self.assert_( getTurlRes['Value']['Successful'].has_key( remoteFile ) ) # check the remove file operation self.assert_( removeFileRes['OK'] ) self.assert_( removeFileRes['Value']['Successful'].has_key( remoteFile ) ) #Check the get turl with non existant file operation self.assert_( failedGetTurlRes['OK'] ) self.assert_( failedGetTurlRes['Value']['Failed'].has_key( remoteFile ) ) expectedError = "File does not exist" self.assert_( expectedError in failedGetTurlRes['Value']['Failed'][remoteFile] )
def test_putGetFile(self): print("\n\n#########################################################" "################\n\n\t\t\tPut and Get test\n") # First upload a file to the storage srcFile = "/etc/group" srcFileSize = getSize(srcFile) testFileName = "testFile.%s" % time.time() remoteFile = self.storage.getCurrentURL(testFileName)["Value"] fileDict = {remoteFile: srcFile} putFileRes = self.storage.putFile(fileDict) # Then make sure we can get a local copy of the file getFileRes = self.storage.getFile(remoteFile) # Cleanup the remote mess removeFileRes = self.storage.removeFile(remoteFile) # Cleanup the mess locally os.remove(testFileName) # Check the put operation self.assertTrue(putFileRes["OK"]) self.assertTrue(remoteFile in putFileRes["Value"]["Successful"]) self.assertEqual(putFileRes["Value"]["Successful"][remoteFile], srcFileSize) # Check the get operation self.assertTrue(getFileRes["OK"]) self.assertTrue(remoteFile in getFileRes["Value"]["Successful"]) self.assertEqual(getFileRes["Value"]["Successful"][remoteFile], srcFileSize) # Check the remove operation self.assertTrue(removeFileRes["OK"]) self.assertTrue(remoteFile in removeFileRes["Value"]["Successful"]) self.assertTrue(removeFileRes["Value"]["Successful"][remoteFile])
def test_putIsFile( self ): print('\n\n#########################################################' '################\n\n\t\t\tIs file test\n') # First upload a file to the storage srcFile = '/etc/group' fileSize = getSize( srcFile ) testFileName = 'testFile.%s' % time.time() destFile = self.storage.getCurrentURL( testFileName )['Value'] fileTuple = ( srcFile, destFile, fileSize ) putFileRes = self.storage.putFile( fileTuple ) # Check we are able to determine that it is a file isFileRes = self.storage.isFile( destFile ) # Clean up the remote mess removeFileRes = self.storage.removeFile( destFile ) # Check that everything isn't a file destDir = os.path.dirname( destFile ) failedIsFileRes = self.storage.isFile( destDir ) # Check the put file operation self.assertTrue( putFileRes['OK'] ) self.assertTrue( destFile in putFileRes['Value']['Successful'] ) # Check the is file operation self.assertTrue( isFileRes['OK'] ) self.assertTrue( destFile in isFileRes['Value']['Successful'] ) self.assertTrue( isFileRes['Value']['Successful'][destFile] ) # check the remove file operation self.assertTrue( removeFileRes['OK'] ) self.assertTrue( destFile in removeFileRes['Value']['Successful'] ) # Check that the directory is not a file self.assertTrue( failedIsFileRes['OK'] ) self.assertTrue( destDir in failedIsFileRes['Value']['Successful'] ) self.assertFalse( failedIsFileRes['Value']['Successful'][destDir] )
def test_putFilegetTransportURL(self): print '\n\n#########################################################################\n\n\t\t\tGet tURL test\n' # First upload a file to the storage srcFile = '/etc/group' srcFileSize = getSize(srcFile) testFileName = 'testFile.%s' % time.time() remoteFile = self.storage.getCurrentURL(testFileName)['Value'] fileDict = {remoteFile: srcFile} putFileRes = self.storage.putFile(fileDict) #Check that we can get a turl getTurlRes = self.storage.getTransportURL(remoteFile) # Clean up the remote mess removeFileRes = self.storage.removeFile(remoteFile) # Try and get a turl for a non existant file failedGetTurlRes = self.storage.getTransportURL(remoteFile) # Check the put file operation self.assert_(putFileRes['OK']) self.assert_(putFileRes['Value']['Successful'].has_key(remoteFile)) self.assert_(putFileRes['Value']['Successful'][remoteFile]) self.assertEqual(putFileRes['Value']['Successful'][remoteFile], srcFileSize) # check the get turl operation self.assert_(getTurlRes['OK']) self.assert_(getTurlRes['Value']['Successful'].has_key(remoteFile)) # check the remove file operation self.assert_(removeFileRes['OK']) self.assert_(removeFileRes['Value']['Successful'].has_key(remoteFile)) #Check the get turl with non existant file operation self.assert_(failedGetTurlRes['OK']) self.assert_(failedGetTurlRes['Value']['Failed'].has_key(remoteFile)) expectedError = "File does not exist" self.assert_( expectedError in failedGetTurlRes['Value']['Failed'][remoteFile])
def test_putPrestageFile( self ): print('\n\n#########################################################' '################\n\n\t\t\tFile prestage test\n') # First upload a file to the storage srcFile = '/etc/group' fileSize = getSize( srcFile ) testFileName = 'testFile.%s' % time.time() destFile = self.storage.getCurrentURL( testFileName )['Value'] fileTuple = ( srcFile, destFile, fileSize ) putFileRes = self.storage.putFile( fileTuple ) # Check that we can issue a stage request prestageRes = self.storage.prestageFile( destFile ) # Clean up the remote mess removeFileRes = self.storage.removeFile( destFile ) # Check the put file operation self.assertTrue( putFileRes['OK'] ) self.assertTrue( destFile in putFileRes['Value']['Successful'] ) # Check the prestage file operation self.assertTrue( prestageRes['OK'] ) self.assertTrue( destFile in prestageRes['Value']['Successful'] ) self.assertTrue( prestageRes['Value']['Successful'][destFile] ) # Check the remove file operation self.assertTrue( removeFileRes['OK'] ) self.assertTrue( destFile in removeFileRes['Value']['Successful'] )
def test_putPrestageFile(self): print("\n\n#########################################################" "################\n\n\t\t\tFile prestage test\n") # First upload a file to the storage srcFile = "/etc/group" fileSize = getSize(srcFile) testFileName = "testFile.%s" % time.time() destFile = self.storage.getCurrentURL(testFileName)["Value"] fileTuple = (srcFile, destFile, fileSize) putFileRes = self.storage.putFile(fileTuple) # Check that we can issue a stage request prestageRes = self.storage.prestageFile(destFile) # Clean up the remote mess removeFileRes = self.storage.removeFile(destFile) # Check the put file operation self.assertTrue(putFileRes["OK"]) self.assertTrue(destFile in putFileRes["Value"]["Successful"]) # Check the prestage file operation self.assertTrue(prestageRes["OK"]) self.assertTrue(destFile in prestageRes["Value"]["Successful"]) self.assertTrue(prestageRes["Value"]["Successful"][destFile]) # Check the remove file operation self.assertTrue(removeFileRes["OK"]) self.assertTrue(destFile in removeFileRes["Value"]["Successful"])
def test_putPrestageFile( self ): print '\n\n#########################################################################\n\n\t\t\tFile prestage test\n' # First upload a file to the storage srcFile = '/etc/group' srcFileSize = getSize( srcFile ) testFileName = 'testFile.%s' % time.time() remoteFile = self.storage.getCurrentURL( testFileName )['Value'] fileDict = {remoteFile:srcFile} putFileRes = self.storage.putFile( fileDict ) # Check that we can issue a stage request prestageRes = self.storage.prestageFile( remoteFile ) # Clean up the remote mess removeFileRes = self.storage.removeFile( remoteFile ) # Check what happens with deleted files deletedPrestageRes = self.storage.prestageFile( remoteFile ) # Check the put file operation self.assert_( putFileRes['OK'] ) self.assert_( putFileRes['Value']['Successful'].has_key( remoteFile ) ) self.assert_( putFileRes['Value']['Successful'][remoteFile] ) self.assertEqual( putFileRes['Value']['Successful'][remoteFile], srcFileSize ) # Check the prestage file operation self.assert_( prestageRes['OK'] ) self.assert_( prestageRes['Value']['Successful'].has_key( remoteFile ) ) self.assert_( prestageRes['Value']['Successful'][remoteFile] ) # Check the remove file operation self.assert_( removeFileRes['OK'] ) self.assert_( removeFileRes['Value']['Successful'].has_key( remoteFile ) ) # Check that pre-staging non-existant file fails self.assert_( deletedPrestageRes['OK'] ) self.assert_( deletedPrestageRes['Value']['Failed'].has_key( remoteFile ) ) expectedError = "No such file or directory" self.assert_( expectedError in deletedPrestageRes['Value']['Failed'][remoteFile] )
def test_putFilegetTransportURL( self ): print('\n\n#########################################################' '################\n\n\t\t\tGet tURL test\n') # First upload a file to the storage srcFile = '/etc/group' fileSize = getSize( srcFile ) testFileName = 'testFile.%s' % time.time() destFile = self.storage.getCurrentURL( testFileName )['Value'] fileTuple = ( srcFile, destFile, fileSize ) putFileRes = self.storage.putFile( fileTuple ) #Check that we can get a turl getTurlRes = self.storage.getTransportURL( destFile, ['dcap', 'gsidcap'] ) # Clean up the remote mess removeFileRes = self.storage.removeFile( destFile ) # Try and get a turl for a non existant file failedGetTurlRes = self.storage.getTransportURL( destFile, ['dcap', 'gsidcap'] ) # Check the put file operation print(putFileRes) self.assertTrue( putFileRes['OK'] ) self.assertTrue( destFile in putFileRes['Value']['Successful'] ) # check the get turl operation print(getTurlRes, destFile) self.assertTrue( getTurlRes['OK'] ) self.assertTrue( destFile in getTurlRes['Value']['Successful'] ) # check the remove file operation self.assertTrue( removeFileRes['OK'] ) self.assertTrue( destFile in removeFileRes['Value']['Successful'] ) #Check the get turl with non existant file operation self.assertTrue( failedGetTurlRes['OK'] ) self.assertTrue( destFile in failedGetTurlRes['Value']['Failed'] ) expectedError = "RFIOStorage.getTransportURL: File does not exist." self.assertEqual( failedGetTurlRes['Value']['Failed'][destFile], expectedError )
def test_putExistsFile( self ): print '\n\n#########################################################################\n\n\t\t\tExists test\n' # First upload a file to the storage srcFile = '/etc/group' fileSize = getSize( srcFile ) testFileName = 'testFile.%s' % time.time() destFile = self.storage.getCurrentURL( testFileName )['Value'] fileTuple = ( srcFile, destFile, fileSize ) putFileRes = self.storage.putFile( fileTuple ) # Then get the file's existance existsFileRes = self.storage.exists( destFile ) # Now remove the file removeFileRes = self.storage.removeFile( destFile ) # Check again that the file exists failedExistRes = self.storage.exists( destFile ) # Check the put file operation self.assert_( putFileRes['OK'] ) self.assert_( putFileRes['Value']['Successful'].has_key( destFile ) ) # Check the exists operation self.assert_( existsFileRes['OK'] ) self.assert_( existsFileRes['Value']['Successful'].has_key( destFile ) ) self.assert_( existsFileRes['Value']['Successful'][destFile] ) # Check the removal operation self.assert_( removeFileRes['OK'] ) self.assert_( removeFileRes['Value']['Successful'].has_key( destFile ) ) # Check the failed exists operation self.assert_( failedExistRes['OK'] ) self.assert_( failedExistRes['Value']['Successful'].has_key( destFile ) ) self.assertFalse( failedExistRes['Value']['Successful'][destFile] )
def test_putPrestageFile(self): print '\n\n#########################################################################\n\n\t\t\tFile prestage test\n' # First upload a file to the storage srcFile = '/etc/group' srcFileSize = getSize(srcFile) testFileName = 'testFile.%s' % time.time() remoteFile = self.storage.getCurrentURL(testFileName)['Value'] fileDict = {remoteFile: srcFile} putFileRes = self.storage.putFile(fileDict) # Check that we can issue a stage request prestageRes = self.storage.prestageFile(remoteFile) # Clean up the remote mess removeFileRes = self.storage.removeFile(remoteFile) # Check what happens with deleted files deletedPrestageRes = self.storage.prestageFile(remoteFile) # Check the put file operation self.assert_(putFileRes['OK']) self.assert_(putFileRes['Value']['Successful'].has_key(remoteFile)) self.assert_(putFileRes['Value']['Successful'][remoteFile]) self.assertEqual(putFileRes['Value']['Successful'][remoteFile], srcFileSize) # Check the prestage file operation self.assert_(prestageRes['OK']) self.assert_(prestageRes['Value']['Successful'].has_key(remoteFile)) self.assert_(prestageRes['Value']['Successful'][remoteFile]) # Check the remove file operation self.assert_(removeFileRes['OK']) self.assert_(removeFileRes['Value']['Successful'].has_key(remoteFile)) # Check that pre-staging non-existant file fails self.assert_(deletedPrestageRes['OK']) self.assert_(deletedPrestageRes['Value']['Failed'].has_key(remoteFile)) expectedError = "No such file or directory" self.assert_( expectedError in deletedPrestageRes['Value']['Failed'][remoteFile])
def test_putGetFile(self): print '\n\n#########################################################################\n\n\t\t\tPut and Get test\n' # First upload a file to the storage srcFile = '/etc/group' srcFileSize = getSize(srcFile) testFileName = 'testFile.%s' % time.time() remoteFile = self.storage.getCurrentURL(testFileName)['Value'] fileDict = {remoteFile: srcFile} putFileRes = self.storage.putFile(fileDict) # Then make sure we can get a local copy of the file getFileRes = self.storage.getFile(remoteFile) # Cleanup the remote mess removeFileRes = self.storage.removeFile(remoteFile) #Cleanup the mess locally os.remove(testFileName) # Check the put operation self.assert_(putFileRes['OK']) self.assert_(putFileRes['Value']['Successful'].has_key(remoteFile)) self.assertEqual(putFileRes['Value']['Successful'][remoteFile], srcFileSize) # Check the get operation self.assert_(getFileRes['OK']) self.assert_(getFileRes['Value']['Successful'].has_key(remoteFile)) self.assertEqual(getFileRes['Value']['Successful'][remoteFile], srcFileSize) # Check the remove operation self.assert_(removeFileRes['OK']) self.assert_(removeFileRes['Value']['Successful'].has_key(remoteFile)) self.assert_(removeFileRes['Value']['Successful'][remoteFile])
def test_getDirectoryMetadata(self): print '\n\n#########################################################################\n\n\t\t\tDirectory metadata test\n' directory = "%s/%s" % (self.destDirectory, 'getDirectoryMetadataTest') destDirectory = self.storageElement.getPfnForLfn(directory)['Value'] # Create a local directory to upload localDir = '/tmp/unit-test' srcFile = '/etc/group' sizeOfLocalFile = getSize(srcFile) if not os.path.exists(localDir): os.mkdir(localDir) for i in range(self.numberOfFiles): shutil.copy(srcFile, '%s/testFile.%s' % (localDir, time.time())) time.sleep(1) # Check that we can successfully upload the directory to the storage element dirDict = {destDirectory: localDir} putDirRes = self.storageElement.putDirectory(dirDict, singleDirectory=True) # Get the directory metadata metadataDirRes = self.storageElement.getDirectoryMetadata( destDirectory, singleDirectory=True) # Now remove the remove directory removeDirRes = self.storageElement.removeDirectory( destDirectory, recursive=True, singleDirectory=True) #Clean up the locally created directory shutil.rmtree(localDir) # Perform the checks for the put dir operation self.assert_(putDirRes['OK']) self.assert_(putDirRes['Value']) if putDirRes['Value']['Files']: self.assertEqual(putDirRes['Value']['Files'], self.numberOfFiles) self.assertEqual(putDirRes['Value']['Size'], self.numberOfFiles * sizeOfLocalFile) self.assert_( type(putDirRes['Value']['Files']) in [types.IntType, types.LongType]) self.assert_( type(putDirRes['Value']['Size']) in [types.IntType, types.LongType]) # Perform the checks for the list dir operation self.assert_(metadataDirRes['OK']) self.assert_(metadataDirRes['Value']) self.assert_(metadataDirRes['Value']['Mode']) self.assert_(type(metadataDirRes['Value']['Mode']) == types.IntType) self.assert_(metadataDirRes['Value']['Directory']) self.assertFalse(metadataDirRes['Value']['File']) # Perform the checks for the remove directory operation self.assert_(removeDirRes['OK']) self.assert_(removeDirRes['Value']) if removeDirRes['Value']['FilesRemoved']: self.assertEqual(removeDirRes['Value']['FilesRemoved'], self.numberOfFiles) self.assertEqual(removeDirRes['Value']['SizeRemoved'], self.numberOfFiles * sizeOfLocalFile) self.assert_( type(removeDirRes['Value']['FilesRemoved']) in [types.IntType, types.LongType]) self.assert_( type(removeDirRes['Value']['SizeRemoved']) in [types.IntType, types.LongType])
def test_putExistsFile(self): print("\n\n#########################################################" "################\n\n\t\t\tExists test\n") # First upload a file to the storage srcFile = "/etc/group" srcFileSize = getSize(srcFile) testFileName = "testFile.%s" % time.time() remoteFile = self.storage.getCurrentURL(testFileName)["Value"] fileDict = {remoteFile: srcFile} putFileRes = self.storage.putFile(fileDict) # Then get the file's existance existsFileRes = self.storage.exists(remoteFile) # Now remove the file removeFileRes = self.storage.removeFile(remoteFile) # Check again that the file exists failedExistRes = self.storage.exists(remoteFile) # Check the put file operation self.assertTrue(putFileRes["OK"]) self.assertTrue(remoteFile in putFileRes["Value"]["Successful"]) self.assertEqual(putFileRes["Value"]["Successful"][remoteFile], srcFileSize) # Check the exists operation self.assertTrue(existsFileRes["OK"]) self.assertTrue(remoteFile in existsFileRes["Value"]["Successful"]) self.assertTrue(existsFileRes["Value"]["Successful"][remoteFile]) # Check the removal operation self.assertTrue(removeFileRes["OK"]) self.assertTrue(remoteFile in removeFileRes["Value"]["Successful"]) # Check the failed exists operation self.assertTrue(failedExistRes["OK"]) self.assertTrue(remoteFile in failedExistRes["Value"]["Successful"]) self.assertFalse(failedExistRes["Value"]["Successful"][remoteFile])
def test_putIsFile(self): print("\n\n#########################################################" "################\n\n\t\t\tIs file test\n") # First upload a file to the storage srcFile = "/etc/group" srcFileSize = getSize(srcFile) testFileName = "testFile.%s" % time.time() remoteFile = self.storage.getCurrentURL(testFileName)["Value"] fileDict = {remoteFile: srcFile} putFileRes = self.storage.putFile(fileDict) # Check we are able to determine that it is a file isFileRes = self.storage.isFile(remoteFile) # Clean up the remote mess removeFileRes = self.storage.removeFile(remoteFile) # Check that everything isn't a file remoteDir = os.path.dirname(remoteFile) failedIsFileRes = self.storage.isFile(remoteDir) # Check the put file operation self.assertTrue(putFileRes["OK"]) self.assertTrue(remoteFile in putFileRes["Value"]["Successful"]) self.assertTrue(putFileRes["Value"]["Successful"][remoteFile]) self.assertEqual(putFileRes["Value"]["Successful"][remoteFile], srcFileSize) # Check the is file operation self.assertTrue(isFileRes["OK"]) self.assertTrue(remoteFile in isFileRes["Value"]["Successful"]) self.assertTrue(isFileRes["Value"]["Successful"][remoteFile]) # check the remove file operation self.assertTrue(removeFileRes["OK"]) self.assertTrue(remoteFile in removeFileRes["Value"]["Successful"]) # Check that the directory is not a file self.assertTrue(failedIsFileRes["OK"]) self.assertTrue(remoteDir in failedIsFileRes["Value"]["Successful"]) self.assertFalse(failedIsFileRes["Value"]["Successful"][remoteDir])
def __putFile( self, src_file, dest_url ): res = pfnparse( src_file ) if not res['OK']: return res localCache = False srcDict = res['Value'] if srcDict['Protocol'] in ['dips', 'dip']: localCache = True srcSEURL = srcDict['Protocol'] + '://' + srcDict['Host'] + ':' + srcDict['Port'] + srcDict['WSUrl'] transferClient = TransferClient( srcSEURL ) res = transferClient.receiveFile( srcDict['FileName'], os.path.join( srcDict['Path'], srcDict['FileName'] ) ) if not res['OK']: return res src_file = srcDict['FileName'] if not os.path.exists( src_file ): errStr = "DIPStorage.__putFile: The source local file does not exist." gLogger.error( errStr, src_file ) return S_ERROR( errStr ) sourceSize = getSize( src_file ) if sourceSize == -1: errStr = "DIPStorage.__putFile: Failed to get file size." gLogger.error( errStr, src_file ) return S_ERROR( errStr ) transferClient = TransferClient( self.url ) res = transferClient.sendFile( src_file, dest_url, token = self.checkSum ) if localCache: os.unlink( src_file ) if res['OK']: return S_OK( sourceSize ) else: return res
def test_putGetDirectory( self ): print('\n\n#########################################################' '################\n\n\t\t\tGet Directory test\n') # First clean the remote directory incase something was left there remoteDir = self.storage.getCurrentURL( 'putDirTest' )['Value'] ignore = self.storage.removeDirectory( remoteDir ) # Create a local directory to upload localDir = '/tmp/unit-test' srcFile = '/etc/group' numberOfFiles = 5 sizeOfLocalFile = getSize( srcFile ) print('Creating local directory: %s' % localDir) if not os.path.exists( localDir ): os.mkdir( localDir ) for i in range( numberOfFiles ): shutil.copy( srcFile, '%s/testFile.%s' % ( localDir, time.time() ) ) time.sleep( 1 ) # Check that we can successfully upload the directory to the storage element dirTuple = ( localDir, remoteDir ) putDirRes = self.storage.putDirectory( dirTuple ) #Clean up the locally created directory print('Removing local directory: %s' % localDir) localFiles = os.listdir( localDir ) for fileName in localFiles: fullPath = '%s/%s' % ( localDir, fileName ) os.remove( fullPath ) os.removedirs( localDir ) # Check that we can get directories from the storage element directoryTuple = ( remoteDir, localDir ) getDirRes = self.storage.getDirectory( directoryTuple ) # Now remove the remove directory removeDirRes = self.storage.removeDirectory( remoteDir ) #Clean up the locally created directory print('Removing local directory: %s' % localDir) localFiles = os.listdir( localDir ) for fileName in localFiles: fullPath = '%s/%s' % ( localDir, fileName ) os.remove( fullPath ) os.removedirs( localDir ) # Perform the checks for the put dir operation self.assertTrue( putDirRes['OK'] ) self.assertTrue( remoteDir in putDirRes['Value']['Successful'] ) resDict = putDirRes['Value']['Successful'][remoteDir] self.assertEqual( resDict['Files'], numberOfFiles ) self.assertEqual( resDict['Size'], numberOfFiles * sizeOfLocalFile ) # Perform the checks for the get dir operation self.assertTrue( getDirRes['OK'] ) self.assertTrue( remoteDir in getDirRes['Value']['Successful'] ) resDict = getDirRes['Value']['Successful'][remoteDir] self.assertEqual( resDict['Files'], numberOfFiles ) self.assertEqual( resDict['Size'], numberOfFiles * sizeOfLocalFile ) # Perform the checks for the remove directory operation self.assertTrue( removeDirRes['OK'] ) self.assertTrue( remoteDir in removeDirRes['Value']['Successful'] ) resDict = removeDirRes['Value']['Successful'][remoteDir] self.assertEqual( resDict['Files'], numberOfFiles ) self.assertEqual( resDict['Size'], numberOfFiles * sizeOfLocalFile )
def test_putRemoveDirectory(self): print("\n\n#########################################################" "################\n\n\t\t\tPut Directory test\n") # First clean the remote directory incase something was left there remoteDir = self.storage.getCurrentURL("")["Value"] _ = self.storage.removeDirectory(remoteDir, True) # Create a local directory to upload localDir = "/tmp/unit-test" srcFile = "/etc/group" sizeOfLocalFile = getSize(srcFile) if not os.path.exists(localDir): os.mkdir(localDir) for i in range(self.numberOfFiles): shutil.copy(srcFile, "%s/testFile.%s" % (localDir, time.time())) time.sleep(1) # Check that we can successfully upload the directory to the storage element dirDict = {remoteDir: localDir} putDirRes = self.storage.putDirectory(dirDict) # Now remove the remove directory removeDirRes = self.storage.removeDirectory(remoteDir, True) # Clean up the locally created directory shutil.rmtree(localDir) # Perform the checks for the put dir operation self.assertTrue(putDirRes["OK"]) self.assertTrue(remoteDir in putDirRes["Value"]["Successful"]) if putDirRes["Value"]["Successful"][remoteDir]["Files"]: self.assertEqual( putDirRes["Value"]["Successful"][remoteDir]["Files"], self.numberOfFiles) self.assertEqual( putDirRes["Value"]["Successful"][remoteDir]["Size"], self.numberOfFiles * sizeOfLocalFile) self.assertTrue( isinstance(putDirRes["Value"]["Successful"][remoteDir]["Files"], int)) self.assertTrue( type(putDirRes["Value"]["Successful"][remoteDir]["Size"]) in six.integer_types) # Perform the checks for the remove dir operation self.assertTrue(removeDirRes["OK"]) self.assertTrue(remoteDir in removeDirRes["Value"]["Successful"]) if removeDirRes["Value"]["Successful"][remoteDir]["FilesRemoved"]: self.assertEqual( removeDirRes["Value"]["Successful"][remoteDir]["FilesRemoved"], self.numberOfFiles) self.assertEqual( removeDirRes["Value"]["Successful"][remoteDir]["SizeRemoved"], self.numberOfFiles * sizeOfLocalFile) self.assertTrue( isinstance( removeDirRes["Value"]["Successful"][remoteDir]["FilesRemoved"], int)) self.assertTrue( type(removeDirRes["Value"]["Successful"][remoteDir]["SizeRemoved"]) in six.integer_types)
def test_putGetDirectory( self ): print '\n\n#########################################################################\n\n\t\t\tGet Directory test\n' # First clean the remote directory incase something was left there remoteDir = self.storage.getCurrentURL( 'putDirTest' )['Value'] ignore = self.storage.removeDirectory( remoteDir ) # Create a local directory to upload localDir = '/tmp/unit-test' srcFile = '/etc/group' numberOfFiles = 5 sizeOfLocalFile = getSize( srcFile ) print 'Creating local directory: %s' % localDir if not os.path.exists( localDir ): os.mkdir( localDir ) for i in range( numberOfFiles ): shutil.copy( srcFile, '%s/testFile.%s' % ( localDir, time.time() ) ) time.sleep( 1 ) # Check that we can successfully upload the directory to the storage element dirTuple = ( localDir, remoteDir ) putDirRes = self.storage.putDirectory( dirTuple ) #Clean up the locally created directory print 'Removing local directory: %s' % localDir localFiles = os.listdir( localDir ) for fileName in localFiles: fullPath = '%s/%s' % ( localDir, fileName ) os.remove( fullPath ) os.removedirs( localDir ) # Check that we can get directories from the storage element directoryTuple = ( remoteDir, localDir ) getDirRes = self.storage.getDirectory( directoryTuple ) # Now remove the remove directory removeDirRes = self.storage.removeDirectory( remoteDir ) #Clean up the locally created directory print 'Removing local directory: %s' % localDir localFiles = os.listdir( localDir ) for fileName in localFiles: fullPath = '%s/%s' % ( localDir, fileName ) os.remove( fullPath ) os.removedirs( localDir ) # Perform the checks for the put dir operation self.assert_( putDirRes['OK'] ) self.assert_( putDirRes['Value']['Successful'].has_key( remoteDir ) ) resDict = putDirRes['Value']['Successful'][remoteDir] self.assertEqual( resDict['Files'], numberOfFiles ) self.assertEqual( resDict['Size'], numberOfFiles * sizeOfLocalFile ) # Perform the checks for the get dir operation self.assert_( getDirRes['OK'] ) self.assert_( getDirRes['Value']['Successful'].has_key( remoteDir ) ) resDict = getDirRes['Value']['Successful'][remoteDir] self.assertEqual( resDict['Files'], numberOfFiles ) self.assertEqual( resDict['Size'], numberOfFiles * sizeOfLocalFile ) # Perform the checks for the remove directory operation self.assert_( removeDirRes['OK'] ) self.assert_( removeDirRes['Value']['Successful'].has_key( remoteDir ) ) resDict = removeDirRes['Value']['Successful'][remoteDir] self.assertEqual( resDict['Files'], numberOfFiles ) self.assertEqual( resDict['Size'], numberOfFiles * sizeOfLocalFile )
def setUp( self ): self.numberOfFiles = 1 self.storageElement = StorageElement( storageElementToTest ) self.localSourceFile = "/etc/group" self.localFileSize = getSize( self.localSourceFile ) self.destDirectory = "/lhcb/test/unit-test/TestStorageElement" destinationDir = self.storageElement.getPfnForLfn( self.destDirectory )['Value'] res = self.storageElement.createDirectory( destinationDir, singleDirectory = True ) self.assert_( res['OK'] )
def test_putRemoveDirectory(self): print('\n\n#########################################################' '################\n\n\t\t\tPut Directory test\n') # First clean the remote directory incase something was left there remoteDir = self.storage.getCurrentURL('')['Value'] ignore = self.storage.removeDirectory(remoteDir, True) # Create a local directory to upload localDir = '/tmp/unit-test' srcFile = '/etc/group' sizeOfLocalFile = getSize(srcFile) if not os.path.exists(localDir): os.mkdir(localDir) for i in range(self.numberOfFiles): shutil.copy(srcFile, '%s/testFile.%s' % (localDir, time.time())) time.sleep(1) # Check that we can successfully upload the directory to the storage element dirDict = {remoteDir: localDir} putDirRes = self.storage.putDirectory(dirDict) # Now remove the remove directory removeDirRes = self.storage.removeDirectory(remoteDir, True) #Clean up the locally created directory shutil.rmtree(localDir) # Perform the checks for the put dir operation self.assertTrue(putDirRes['OK']) self.assertTrue(putDirRes['Value']['Successful'].has_key(remoteDir)) if putDirRes['Value']['Successful'][remoteDir]['Files']: self.assertEqual( putDirRes['Value']['Successful'][remoteDir]['Files'], self.numberOfFiles) self.assertEqual( putDirRes['Value']['Successful'][remoteDir]['Size'], self.numberOfFiles * sizeOfLocalFile) self.assertTrue( type(putDirRes['Value']['Successful'][remoteDir]['Files']) == IntType) self.assertTrue( type(putDirRes['Value']['Successful'][remoteDir]['Size']) in [LongType, IntType]) # Perform the checks for the remove dir operation self.assertTrue(removeDirRes['OK']) self.assertTrue(removeDirRes['Value']['Successful'].has_key(remoteDir)) if removeDirRes['Value']['Successful'][remoteDir]['FilesRemoved']: self.assertEqual( removeDirRes['Value']['Successful'][remoteDir]['FilesRemoved'], self.numberOfFiles) self.assertEqual( removeDirRes['Value']['Successful'][remoteDir]['SizeRemoved'], self.numberOfFiles * sizeOfLocalFile) self.assertTrue( type(removeDirRes['Value']['Successful'][remoteDir] ['FilesRemoved']) == IntType) self.assertTrue( type(removeDirRes['Value']['Successful'][remoteDir]['SizeRemoved']) in [LongType, IntType])
def test_removeDirectory(self): print('\n\n#########################################################' '################\n\n\t\t\tRemove directory test\n') destDirectory = "%s/%s" % (self.destDirectory, 'removeDirectoryTest') # destDirectory = returnSingleResult( self.storageElement.getURL( directory ) )['Value'] # Create a local directory to upload localDir = '/tmp/unit-test' srcFile = '/etc/group' sizeOfLocalFile = getSize(srcFile) if not os.path.exists(localDir): os.mkdir(localDir) for i in range(self.numberOfFiles): shutil.copy(srcFile, '%s/testFile.%s' % (localDir, time.time())) time.sleep(1) # Check that we can successfully upload the directory to the storage element dirDict = {destDirectory: localDir} putDirRes = self.storageElement.putDirectory(dirDict) # Get the directory metadata # Now remove the remove directory removeDirRes = self.storageElement.removeDirectory(destDirectory, recursive=True) # Clean up the locally created directory shutil.rmtree(localDir) # Perform the checks for the put dir operation self.assertTrue(putDirRes['OK']) self.assertTrue(putDirRes['Value']) if putDirRes['Value']['Successful'][destDirectory]['Files']: self.assertEqual( putDirRes['Value']['Successful'][destDirectory]['Files'], self.numberOfFiles) self.assertEqual( putDirRes['Value']['Successful'][destDirectory]['Size'], self.numberOfFiles * sizeOfLocalFile) self.assertTrue( type(putDirRes['Value']['Successful'][destDirectory]['Files']) in six.integer_types) self.assertTrue( type(putDirRes['Value']['Successful'][destDirectory]['Size']) in six.integer_types) # Perform the checks for the remove directory operation self.assertTrue(removeDirRes['OK']) self.assertTrue(removeDirRes['Value']) if removeDirRes['Value']['Successful'][destDirectory]['FilesRemoved']: self.assertEqual( removeDirRes['Value']['Successful'][destDirectory] ['FilesRemoved'], self.numberOfFiles) self.assertEqual( removeDirRes['Value']['Successful'][destDirectory] ['SizeRemoved'], self.numberOfFiles * sizeOfLocalFile) self.assertTrue( type(removeDirRes['Value']['Successful'][destDirectory] ['FilesRemoved']) in six.integer_types) self.assertTrue( type(removeDirRes['Value']['Successful'][destDirectory] ['SizeRemoved']) in six.integer_types)
def setUp( self ): self.numberOfFiles = 1 self.storageElement = StorageElement( storageElementToTest ) self.localSourceFile = fileToTest self.localFileSize = getSize( self.localSourceFile ) self.destDirectory = lfnDirToTest # destinationDir = returnSingleResult( self.storageElement.getURL( self.destDirectory ) )['Value'] destinationDir = self.destDirectory res = self.storageElement.createDirectory( destinationDir ) self.assert_( res['OK'] )
def setUp(self): self.numberOfFiles = 1 self.storageElement = StorageElement(storageElementToTest) self.localSourceFile = fileToTest self.localFileSize = getSize(self.localSourceFile) self.destDirectory = lfnDirToTest # destinationDir = returnSingleResult( self.storageElement.getURL( self.destDirectory ) )['Value'] destinationDir = self.destDirectory res = self.storageElement.createDirectory(destinationDir) self.assertTrue(res['OK'])
def setUp(self): self.numberOfFiles = 1 self.storageElement = StorageElement(storageElementToTest) self.localSourceFile = "/etc/group" self.localFileSize = getSize(self.localSourceFile) self.destDirectory = "/lhcb/test/unit-test/TestStorageElement" destinationDir = returnSingleResult( self.storageElement.getPfnForLfn(self.destDirectory)) res = self.storageElement.createDirectory(destinationDir, singleDirectory=True) self.assert_(res['OK'])
def test_putGetDirectory( self ): print '\n\n#########################################################################\n\n\t\t\tGet Directory test\n' # First clean the remote directory incase something was left there remoteDir = self.storage.getCurrentURL( '' )['Value'] ignore = self.storage.removeDirectory( remoteDir, True ) # Create a local directory to upload localDir = '/tmp/unit-test' srcFile = '/etc/group' sizeOfLocalFile = getSize( srcFile ) if not os.path.exists( localDir ): os.mkdir( localDir ) for i in range( self.numberOfFiles ): shutil.copy( srcFile, '%s/testFile.%s' % ( localDir, time.time() ) ) time.sleep( 1 ) # Check that we can successfully upload the directory to the storage element dirDict = {remoteDir:localDir} putDirRes = self.storage.putDirectory( dirDict ) #Clean up the locally created directory shutil.rmtree( localDir ) # Check that we can get directories from the storage element getDirRes = self.storage.getDirectory( remoteDir, localPath = localDir ) # Now remove the remove directory removeDirRes = self.storage.removeDirectory( remoteDir, True ) #Clean up the locally created directory shutil.rmtree( localDir ) # Perform the checks for the put dir operation self.assert_( putDirRes['OK'] ) self.assert_( putDirRes['Value']['Successful'].has_key( remoteDir ) ) if putDirRes['Value']['Successful'][remoteDir]['Files']: self.assertEqual( putDirRes['Value']['Successful'][remoteDir]['Files'], self.numberOfFiles ) self.assertEqual( putDirRes['Value']['Successful'][remoteDir]['Size'], self.numberOfFiles * sizeOfLocalFile ) self.assert_( type( putDirRes['Value']['Successful'][remoteDir]['Files'] ) == IntType ) self.assert_( type( putDirRes['Value']['Successful'][remoteDir]['Size'] ) in [LongType, IntType] ) # Perform the checks for the get dir operation self.assert_( getDirRes['OK'] ) self.assert_( getDirRes['Value']['Successful'].has_key( remoteDir ) ) resDict = getDirRes['Value']['Successful'][remoteDir] if resDict['Files']: self.assertEqual( resDict['Files'], self.numberOfFiles ) self.assertEqual( resDict['Size'], self.numberOfFiles * sizeOfLocalFile ) self.assert_( type( resDict['Files'] ) == IntType ) self.assert_( type( resDict['Size'] ) in [LongType, IntType] ) # Perform the checks for the remove directory operation self.assert_( removeDirRes['OK'] ) self.assert_( removeDirRes['Value']['Successful'].has_key( remoteDir ) ) if removeDirRes['Value']['Successful'][remoteDir]['FilesRemoved']: self.assertEqual( removeDirRes['Value']['Successful'][remoteDir]['FilesRemoved'], self.numberOfFiles ) self.assertEqual( removeDirRes['Value']['Successful'][remoteDir]['SizeRemoved'], self.numberOfFiles * sizeOfLocalFile ) self.assert_( type( removeDirRes['Value']['Successful'][remoteDir]['FilesRemoved'] ) == IntType ) self.assert_( type( removeDirRes['Value']['Successful'][remoteDir]['SizeRemoved'] ) in [LongType, IntType] )
def test_getDirectory( self ): print '\n\n#########################################################################\n\n\t\t\tGet directory test\n' destDirectory = "%s/%s" % ( self.destDirectory, 'getDirectoryTest' ) # destDirectory = returnSingleResult( self.storageElement.getURL( directory ) )['Value'] # Create a local directory to upload localDir = '/tmp/unit-test' srcFile = '/etc/group' sizeOfLocalFile = getSize( srcFile ) if not os.path.exists( localDir ): os.mkdir( localDir ) for i in range( self.numberOfFiles ): shutil.copy( srcFile, '%s/testFile.%s' % ( localDir, time.time() ) ) time.sleep( 1 ) # Check that we can successfully upload the directory to the storage element dirDict = {destDirectory:localDir} putDirRes = self.storageElement.putDirectory( dirDict ) # Get the directory metadata #Clean up the locally created directory shutil.rmtree( localDir ) getDirRes = self.storageElement.getDirectory( destDirectory, localPath = localDir ) # Now remove the remove directory removeDirRes = self.storageElement.removeDirectory( destDirectory, recursive = True ) #Clean up the locally created directory if os.path.exists( localDir ): shutil.rmtree( localDir ) # Perform the checks for the put dir operation self.assert_( putDirRes['OK'] ) self.assert_( putDirRes['Value'] ) for _dir in dirDict: if putDirRes['Value']['Successful'][_dir]['Files']: self.assertEqual( putDirRes['Value']['Successful'][_dir]['Files'], self.numberOfFiles ) self.assertEqual( putDirRes['Value']['Successful'][_dir]['Size'], self.numberOfFiles * sizeOfLocalFile ) self.assert_( type( putDirRes['Value']['Successful'][_dir]['Files'] ) in [types.IntType, types.LongType] ) self.assert_( type( putDirRes['Value']['Successful'][_dir]['Size'] ) in [types.IntType, types.LongType] ) # Perform the checks for the get directory operation self.assert_( getDirRes['OK'] ) self.assert_( getDirRes['Value'] ) for _dir in dirDict: if getDirRes['Value']['Successful'][_dir]['Files']: self.assertEqual( getDirRes['Value']['Successful'][_dir]['Files'], self.numberOfFiles ) self.assertEqual( getDirRes['Value']['Successful'][_dir]['Size'], self.numberOfFiles * sizeOfLocalFile ) self.assert_( type( getDirRes['Value']['Successful'][_dir]['Files'] ) in [types.IntType, types.LongType] ) self.assert_( type( getDirRes['Value']['Successful'][_dir]['Size'] ) in [types.IntType, types.LongType] ) # Perform the checks for the remove directory operation self.assert_( removeDirRes['OK'] ) self.assert_( removeDirRes['Value'] ) if removeDirRes['Value']['Successful'][destDirectory]['FilesRemoved']: self.assertEqual( removeDirRes['Value']['Successful'][destDirectory]['FilesRemoved'], self.numberOfFiles ) self.assertEqual( removeDirRes['Value']['Successful'][destDirectory]['SizeRemoved'], self.numberOfFiles * sizeOfLocalFile ) self.assert_( type( removeDirRes['Value']['Successful'][destDirectory]['FilesRemoved'] ) in [types.IntType, types.LongType] ) self.assert_( type( removeDirRes['Value']['Successful'][destDirectory]['SizeRemoved'] ) in [types.IntType, types.LongType] )
def test_putListDirectory(self): print("\n\n#########################################################" "################\n\n\t\t\tList Directory test\n") # First clean the remote directory incase something was left there remoteDir = self.storage.getCurrentURL("putDirTest")["Value"] ignore = self.storage.removeDirectory(remoteDir) # Create a local directory to upload localDir = "/tmp/unit-test" srcFile = "/etc/group" numberOfFiles = 5 sizeOfLocalFile = getSize(srcFile) print("Creating local directory: %s" % localDir) if not os.path.exists(localDir): os.mkdir(localDir) for i in range(numberOfFiles): shutil.copy(srcFile, "%s/testFile.%s" % (localDir, time.time())) time.sleep(1) # Check that we can successfully upload the directory to the storage element dirTuple = (localDir, remoteDir) putDirRes = self.storage.putDirectory(dirTuple) # List the remote directory listDirRes = self.storage.listDirectory(remoteDir) # Now remove the remove directory removeDirRes = self.storage.removeDirectory(remoteDir) # Clean up the locally created directory print("Removing local directory: %s" % localDir) localFiles = os.listdir(localDir) for fileName in localFiles: fullPath = "%s/%s" % (localDir, fileName) os.remove(fullPath) os.removedirs(localDir) # Perform the checks for the put dir operation self.assertTrue(putDirRes["OK"]) self.assertTrue(remoteDir in putDirRes["Value"]["Successful"]) resDict = putDirRes["Value"]["Successful"][remoteDir] self.assertEqual(resDict["Files"], numberOfFiles) self.assertEqual(resDict["Size"], numberOfFiles * sizeOfLocalFile) # Perform the checks for the list dir operation self.assertTrue(listDirRes["OK"]) self.assertTrue(remoteDir in listDirRes["Value"]["Successful"]) resDict = listDirRes["Value"]["Successful"][remoteDir] self.assertTrue("SubDirs" in resDict) self.assertTrue("Files" in resDict) self.assertEqual(len(resDict["Files"]), numberOfFiles) # Perform the checks for the remove directory operation self.assertTrue(removeDirRes["OK"]) self.assertTrue(remoteDir in removeDirRes["Value"]["Successful"]) resDict = removeDirRes["Value"]["Successful"][remoteDir] self.assertEqual(resDict["Files"], numberOfFiles) self.assertEqual(resDict["Size"], numberOfFiles * sizeOfLocalFile)
def test_putGetFileMetaData(self): print('\n\n#########################################################' '################\n\n\t\t\tGet file metadata test\n') # First upload a file to the storage srcFile = '/etc/group' srcFileSize = getSize(srcFile) testFileName = 'testFile.%s' % time.time() remoteFile = self.storage.getCurrentURL(testFileName)['Value'] fileDict = {remoteFile: srcFile} putFileRes = self.storage.putFile(fileDict) # Check that we can get the file metadata getMetadataRes = self.storage.getFileMetadata(remoteFile) # Clean up the remote mess removeFileRes = self.storage.removeFile(remoteFile) # See what happens with non existant files failedMetadataRes = self.storage.getFileMetadata(remoteFile) # Check directories are handled properly remoteDir = os.path.dirname(remoteFile) directoryMetadataRes = self.storage.getFileMetadata(remoteDir) # Check the put file operation self.assertTrue(putFileRes['OK']) self.assertTrue(putFileRes['Value']['Successful'].has_key(remoteFile)) self.assertTrue(putFileRes['Value']['Successful'][remoteFile]) self.assertEqual(putFileRes['Value']['Successful'][remoteFile], srcFileSize) # Check the get metadata operation self.assertTrue(getMetadataRes['OK']) self.assertTrue( getMetadataRes['Value']['Successful'].has_key(remoteFile)) fileMetaData = getMetadataRes['Value']['Successful'][remoteFile] self.assertTrue(fileMetaData['Cached']) self.assertFalse(fileMetaData['Migrated']) self.assertEqual(fileMetaData['Size'], srcFileSize) # check the remove file operation self.assertTrue(removeFileRes['OK']) self.assertTrue( removeFileRes['Value']['Successful'].has_key(remoteFile)) # Check the get metadata for non existant file self.assertTrue(failedMetadataRes['OK']) self.assertTrue( failedMetadataRes['Value']['Failed'].has_key(remoteFile)) expectedError = "File does not exist" self.assertTrue( expectedError in failedMetadataRes['Value']['Failed'][remoteFile]) # Check that metadata operation with a directory self.assertTrue(directoryMetadataRes['OK']) self.assertTrue( directoryMetadataRes['Value']['Failed'].has_key(remoteDir)) expectedError = "Supplied path is not a file" self.assertTrue(expectedError in directoryMetadataRes['Value'] ['Failed'][remoteDir])
def test_putListDirectory(self): print "\n\n#########################################################################\n\n\t\t\tList Directory test\n" # First clean the remote directory incase something was left there remoteDir = self.storage.getCurrentURL("putDirTest")["Value"] ignore = self.storage.removeDirectory(remoteDir) # Create a local directory to upload localDir = "/tmp/unit-test" srcFile = "/etc/group" numberOfFiles = 5 sizeOfLocalFile = getSize(srcFile) print "Creating local directory: %s" % localDir if not os.path.exists(localDir): os.mkdir(localDir) for i in range(numberOfFiles): shutil.copy(srcFile, "%s/testFile.%s" % (localDir, time.time())) time.sleep(1) # Check that we can successfully upload the directory to the storage element dirTuple = (localDir, remoteDir) putDirRes = self.storage.putDirectory(dirTuple) # List the remote directory listDirRes = self.storage.listDirectory(remoteDir) # Now remove the remove directory removeDirRes = self.storage.removeDirectory(remoteDir) # Clean up the locally created directory print "Removing local directory: %s" % localDir localFiles = os.listdir(localDir) for fileName in localFiles: fullPath = "%s/%s" % (localDir, fileName) os.remove(fullPath) os.removedirs(localDir) # Perform the checks for the put dir operation self.assert_(putDirRes["OK"]) self.assert_(putDirRes["Value"]["Successful"].has_key(remoteDir)) resDict = putDirRes["Value"]["Successful"][remoteDir] self.assertEqual(resDict["Files"], numberOfFiles) self.assertEqual(resDict["Size"], numberOfFiles * sizeOfLocalFile) # Perform the checks for the list dir operation self.assert_(listDirRes["OK"]) self.assert_(listDirRes["Value"]["Successful"].has_key(remoteDir)) resDict = listDirRes["Value"]["Successful"][remoteDir] self.assert_(resDict.has_key("SubDirs")) self.assert_(resDict.has_key("Files")) self.assertEqual(len(resDict["Files"].keys()), numberOfFiles) # Perform the checks for the remove directory operation self.assert_(removeDirRes["OK"]) self.assert_(removeDirRes["Value"]["Successful"].has_key(remoteDir)) resDict = removeDirRes["Value"]["Successful"][remoteDir] self.assertEqual(resDict["Files"], numberOfFiles) self.assertEqual(resDict["Size"], numberOfFiles * sizeOfLocalFile)
def test_getDirectoryMetadata( self ): print '\n\n#########################################################################\n\n\t\t\tDirectory metadata test\n' directory = "%s/%s" % ( self.destDirectory, 'getDirectoryMetadataTest' ) destDirectory = returnSingleResult( self.storageElement.getPfnForLfn( directory ) )['Value'] # Create a local directory to upload localDir = '/tmp/unit-test' srcFile = '/etc/group' sizeOfLocalFile = getSize( srcFile ) if not os.path.exists( localDir ): os.mkdir( localDir ) for i in range( self.numberOfFiles ): shutil.copy( srcFile, '%s/testFile.%s' % ( localDir, time.time() ) ) time.sleep( 1 ) # Check that we can successfully upload the directory to the storage element dirDict = {destDirectory:localDir} putDirRes = self.storageElement.putDirectory( dirDict, singleDirectory = True ) # Get the directory metadata metadataDirRes = self.storageElement.getDirectoryMetadata( destDirectory, singleDirectory = True ) # Now remove the remove directory removeDirRes = self.storageElement.removeDirectory( destDirectory, recursive = True, singleDirectory = True ) #Clean up the locally created directory shutil.rmtree( localDir ) # Perform the checks for the put dir operation self.assert_( putDirRes['OK'] ) self.assert_( putDirRes['Value'] ) if putDirRes['Value']['Files']: self.assertEqual( putDirRes['Value']['Files'], self.numberOfFiles ) self.assertEqual( putDirRes['Value']['Size'], self.numberOfFiles * sizeOfLocalFile ) self.assert_( type( putDirRes['Value']['Files'] ) in [types.IntType, types.LongType] ) self.assert_( type( putDirRes['Value']['Size'] ) in [types.IntType, types.LongType] ) # Perform the checks for the list dir operation self.assert_( metadataDirRes['OK'] ) self.assert_( metadataDirRes['Value'] ) # Works only for the SRM2 plugin # self.assert_( metadataDirRes['Value']['Mode'] ) # self.assert_( type( metadataDirRes['Value']['Mode'] ) == types.IntType ) self.assert_( metadataDirRes['Value']['Directory'] ) self.assertFalse( metadataDirRes['Value']['File'] ) # Perform the checks for the remove directory operation self.assert_( removeDirRes['OK'] ) self.assert_( removeDirRes['Value'] ) if removeDirRes['Value']['FilesRemoved']: self.assertEqual( removeDirRes['Value']['FilesRemoved'], self.numberOfFiles ) self.assertEqual( removeDirRes['Value']['SizeRemoved'], self.numberOfFiles * sizeOfLocalFile ) self.assert_( type( removeDirRes['Value']['FilesRemoved'] ) in [types.IntType, types.LongType] ) self.assert_( type( removeDirRes['Value']['SizeRemoved'] ) in [types.IntType, types.LongType] )
def test_putGetDirectoryMetadata(self): print('\n\n#########################################################' '################\n\n\t\t\tGet Directory Metadata test\n') # First clean the remote directory incase something was left there remoteDir = self.storage.getCurrentURL('')['Value'] _ = self.storage.removeDirectory(remoteDir, True) # Create a local directory to upload localDir = '/tmp/unit-test' srcFile = '/etc/group' sizeOfLocalFile = getSize(srcFile) if not os.path.exists(localDir): os.mkdir(localDir) for i in range(self.numberOfFiles): shutil.copy(srcFile, '%s/testFile.%s' % (localDir, time.time())) time.sleep(1) # Check that we can successfully upload the directory to the storage element dirDict = {remoteDir: localDir} putDirRes = self.storage.putDirectory(dirDict) # Get the directory metadata getMetadataRes = self.storage.getDirectoryMetadata(remoteDir) # Now remove the remove directory removeDirRes = self.storage.removeDirectory(remoteDir, True) # Clean up the locally created directory shutil.rmtree(localDir) # Perform the checks for the put dir operation self.assertTrue(putDirRes['OK']) self.assertTrue(remoteDir in putDirRes['Value']['Successful']) if putDirRes['Value']['Successful'][remoteDir]['Files']: self.assertEqual(putDirRes['Value']['Successful'][remoteDir]['Files'], self.numberOfFiles) self.assertEqual(putDirRes['Value']['Successful'][remoteDir]['Size'], self.numberOfFiles * sizeOfLocalFile) self.assertTrue(isinstance(putDirRes['Value']['Successful'][remoteDir]['Files'], int)) self.assertTrue(type(putDirRes['Value']['Successful'][remoteDir]['Size']) in six.integer_types) # Perform the checks for the get metadata operation self.assertTrue(getMetadataRes['OK']) self.assertTrue(remoteDir in getMetadataRes['Value']['Successful']) resDict = getMetadataRes['Value']['Successful'][remoteDir] self.assertTrue('Mode' in resDict) self.assertTrue(isinstance(resDict['Mode'], int)) # Perform the checks for the remove directory operation if removeDirRes['Value']['Successful'][remoteDir]['FilesRemoved']: self.assertEqual(removeDirRes['Value']['Successful'][remoteDir]['FilesRemoved'], self.numberOfFiles) self.assertEqual(removeDirRes['Value']['Successful'][remoteDir] ['SizeRemoved'], self.numberOfFiles * sizeOfLocalFile) self.assertTrue(isinstance(removeDirRes['Value']['Successful'][remoteDir]['FilesRemoved'], int)) self.assertTrue(type(removeDirRes['Value']['Successful'][remoteDir]['SizeRemoved']) in six.integer_types)
def __getFile( self, src_url, dest_file ): transferClient = TransferClient( self.url ) res = transferClient.receiveFile( dest_file, src_url, token = self.checkSum ) if not res['OK']: return res if not os.path.exists( dest_file ): errStr = "DIPStorage.__getFile: The destination local file does not exist." gLogger.error( errStr, dest_file ) return S_ERROR( errStr ) destSize = getSize( dest_file ) if destSize == -1: errStr = "DIPStorage.__getFile: Failed to get the local file size." gLogger.error( errStr, dest_file ) return S_ERROR( errStr ) return S_OK( destSize )
def test_putGetFileMetaData(self): print("\n\n#########################################################" "################\n\n\t\t\tGet file metadata test\n") # First upload a file to the storage srcFile = "/etc/group" srcFileSize = getSize(srcFile) testFileName = "testFile.%s" % time.time() remoteFile = self.storage.getCurrentURL(testFileName)["Value"] fileDict = {remoteFile: srcFile} putFileRes = self.storage.putFile(fileDict) # Check that we can get the file metadata getMetadataRes = self.storage.getFileMetadata(remoteFile) # Clean up the remote mess removeFileRes = self.storage.removeFile(remoteFile) # See what happens with non existant files failedMetadataRes = self.storage.getFileMetadata(remoteFile) # Check directories are handled properly remoteDir = os.path.dirname(remoteFile) directoryMetadataRes = self.storage.getFileMetadata(remoteDir) # Check the put file operation self.assertTrue(putFileRes["OK"]) self.assertTrue(remoteFile in putFileRes["Value"]["Successful"]) self.assertTrue(putFileRes["Value"]["Successful"][remoteFile]) self.assertEqual(putFileRes["Value"]["Successful"][remoteFile], srcFileSize) # Check the get metadata operation self.assertTrue(getMetadataRes["OK"]) self.assertTrue(remoteFile in getMetadataRes["Value"]["Successful"]) fileMetaData = getMetadataRes["Value"]["Successful"][remoteFile] self.assertTrue(fileMetaData["Cached"]) self.assertFalse(fileMetaData["Migrated"]) self.assertEqual(fileMetaData["Size"], srcFileSize) # check the remove file operation self.assertTrue(removeFileRes["OK"]) self.assertTrue(remoteFile in removeFileRes["Value"]["Successful"]) # Check the get metadata for non existant file self.assertTrue(failedMetadataRes["OK"]) self.assertTrue(remoteFile in failedMetadataRes["Value"]["Failed"]) expectedError = "File does not exist" self.assertTrue( expectedError in failedMetadataRes["Value"]["Failed"][remoteFile]) # Check that metadata operation with a directory self.assertTrue(directoryMetadataRes["OK"]) self.assertTrue(remoteDir in directoryMetadataRes["Value"]["Failed"]) expectedError = "Supplied path is not a file" self.assertTrue(expectedError in directoryMetadataRes["Value"] ["Failed"][remoteDir])
def test_putPrestageStatus(self): print('\n\n#########################################################' '################\n\n\t\t\tPrestage status test\n') # First upload a file to the storage srcFile = '/etc/group' srcFileSize = getSize(srcFile) testFileName = 'testFile.%s' % time.time() remoteFile = self.storage.getCurrentURL(testFileName)['Value'] fileDict = {remoteFile: srcFile} putFileRes = self.storage.putFile(fileDict) # Check that we can issue a stage request prestageRes = self.storage.prestageFile(remoteFile) srmID = '' if prestageRes['OK']: if prestageRes['Value']['Successful'].has_key(remoteFile): srmID = prestageRes['Value']['Successful'][remoteFile] # Take a quick break to allow the SRM to realise the file is available sleepTime = 10 print('Sleeping for %s seconds' % sleepTime) time.sleep(sleepTime) # Check that we can monitor the stage request prestageStatusRes = self.storage.prestageFileStatus( {remoteFile: srmID}) # Clean up the remote mess removeFileRes = self.storage.removeFile(remoteFile) # Check the put file operation self.assertTrue(putFileRes['OK']) self.assertTrue(putFileRes['Value']['Successful'].has_key(remoteFile)) self.assertTrue(putFileRes['Value']['Successful'][remoteFile]) self.assertEqual(putFileRes['Value']['Successful'][remoteFile], srcFileSize) # Check the prestage file operation self.assertTrue(prestageRes['OK']) self.assertTrue(prestageRes['Value']['Successful'].has_key(remoteFile)) self.assertTrue(prestageRes['Value']['Successful'][remoteFile]) self.assertTrue( type(prestageRes['Value']['Successful'][remoteFile]) in StringTypes) # Check the prestage status operation self.assertTrue(prestageStatusRes['OK']) self.assertTrue( prestageStatusRes['Value']['Successful'].has_key(remoteFile)) self.assertTrue(prestageStatusRes['Value']['Successful'][remoteFile]) # Check the remove file operation self.assertTrue(removeFileRes['OK']) self.assertTrue( removeFileRes['Value']['Successful'].has_key(remoteFile))
def test_listDirectory( self ): print '\n\n#########################################################################\n\n\t\t\tList directory test\n' directory = "%s/%s" % ( self.destDirectory, 'listDirectoryTest' ) destDirectory = self.storageElement.getPfnForLfn( directory )['Value'] # Create a local directory to upload localDir = '/tmp/unit-test' srcFile = '/etc/group' sizeOfLocalFile = getSize( srcFile ) if not os.path.exists( localDir ): os.mkdir( localDir ) for i in range( self.numberOfFiles ): shutil.copy( srcFile, '%s/testFile.%s' % ( localDir, time.time() ) ) time.sleep( 1 ) # Check that we can successfully upload the directory to the storage element dirDict = {destDirectory:localDir} putDirRes = self.storageElement.putDirectory( dirDict, singleDirectory = True ) print putDirRes # List the remote directory listDirRes = self.storageElement.listDirectory( destDirectory, singleDirectory = True ) # Now remove the remove directory removeDirRes = self.storageElement.removeDirectory( destDirectory, recursive = True, singleDirectory = True ) print removeDirRes #Clean up the locally created directory shutil.rmtree( localDir ) # Perform the checks for the put dir operation self.assert_( putDirRes['OK'] ) self.assert_( putDirRes['Value'] ) if putDirRes['Value']['Files']: self.assertEqual( putDirRes['Value']['Files'], self.numberOfFiles ) self.assertEqual( putDirRes['Value']['Size'], self.numberOfFiles * sizeOfLocalFile ) self.assert_( type( putDirRes['Value']['Files'] ) in [types.IntType, types.LongType] ) self.assert_( type( putDirRes['Value']['Size'] ) in [types.IntType, types.LongType] ) # Perform the checks for the list dir operation self.assert_( listDirRes['OK'] ) self.assert_( listDirRes['Value'] ) self.assert_( listDirRes.has_key( 'SubDirs' ) ) self.assert_( listDirRes.has_key( 'Files' ) ) self.assertEqual( len( listDirRes['Files'].keys() ), self.numberOfFiles ) # Perform the checks for the remove directory operation self.assert_( removeDirRes['OK'] ) self.assert_( removeDirRes['Value'] ) if removeDirRes['Value']['FilesRemoved']: self.assertEqual( removeDirRes['Value']['FilesRemoved'], self.numberOfFiles ) self.assertEqual( removeDirRes['Value']['SizeRemoved'], self.numberOfFiles * sizeOfLocalFile ) self.assert_( type( removeDirRes['Value']['FilesRemoved'] ) in [types.IntType, types.LongType] ) self.assert_( type( removeDirRes['Value']['SizeRemoved'] ) in [types.IntType, types.LongType] )
def test_putGetFileMetaData( self ): print '\n\n#########################################################################\n\n\t\t\tGet file metadata test\n' # First upload a file to the storage srcFile = '/etc/group' srcFileSize = getSize( srcFile ) testFileName = 'testFile.%s' % time.time() remoteFile = self.storage.getCurrentURL( testFileName )['Value'] fileDict = {remoteFile:srcFile} putFileRes = self.storage.putFile( fileDict ) # Check that we can get the file metadata getMetadataRes = self.storage.getFileMetadata( remoteFile ) # Clean up the remote mess removeFileRes = self.storage.removeFile( remoteFile ) # See what happens with non existant files failedMetadataRes = self.storage.getFileMetadata( remoteFile ) # Check directories are handled properly remoteDir = os.path.dirname( remoteFile ) directoryMetadataRes = self.storage.getFileMetadata( remoteDir ) # Check the put file operation self.assert_( putFileRes['OK'] ) self.assert_( putFileRes['Value']['Successful'].has_key( remoteFile ) ) self.assert_( putFileRes['Value']['Successful'][remoteFile] ) self.assertEqual( putFileRes['Value']['Successful'][remoteFile], srcFileSize ) # Check the get metadata operation self.assert_( getMetadataRes['OK'] ) self.assert_( getMetadataRes['Value']['Successful'].has_key( remoteFile ) ) fileMetaData = getMetadataRes['Value']['Successful'][remoteFile] self.assert_( fileMetaData['Cached'] ) self.assertFalse( fileMetaData['Migrated'] ) self.assertEqual( fileMetaData['Size'], srcFileSize ) # check the remove file operation self.assert_( removeFileRes['OK'] ) self.assert_( removeFileRes['Value']['Successful'].has_key( remoteFile ) ) # Check the get metadata for non existant file self.assert_( failedMetadataRes['OK'] ) self.assert_( failedMetadataRes['Value']['Failed'].has_key( remoteFile ) ) expectedError = "File does not exist" self.assert_( expectedError in failedMetadataRes['Value']['Failed'][remoteFile] ) # Check that metadata operation with a directory self.assert_( directoryMetadataRes['OK'] ) self.assert_( directoryMetadataRes['Value']['Failed'].has_key( remoteDir ) ) expectedError = "Supplied path is not a file" self.assert_( expectedError in directoryMetadataRes['Value']['Failed'][remoteDir] )
def test_putGetFileMetaData(self): print "\n\n#########################################################################\n\n\t\t\tGet file metadata test\n" # First upload a file to the storage srcFile = "/etc/group" fileSize = getSize(srcFile) testFileName = "testFile.%s" % time.time() destFile = self.storage.getCurrentURL(testFileName)["Value"] fileTuple = (srcFile, destFile, fileSize) putFileRes = self.storage.putFile(fileTuple) # Check that we can get the file metadata getMetadataRes = self.storage.getFileMetadata(destFile) # Clean up the remote mess removeFileRes = self.storage.removeFile(destFile) # See what happens with non existant files failedMetadataRes = self.storage.getFileMetadata(destFile) # Check directories are handled properly destDir = os.path.dirname(destFile) directoryMetadataRes = self.storage.getFileMetadata(destDir) # Check the put file operation self.assert_(putFileRes["OK"]) self.assert_(putFileRes["Value"]["Successful"].has_key(destFile)) # Check the get metadata operation self.assert_(getMetadataRes["OK"]) self.assert_(getMetadataRes["Value"]["Successful"].has_key(destFile)) fileMetaData = getMetadataRes["Value"]["Successful"][destFile] # self.assert_(fileMetaData['Cached']) # self.assertFalse(fileMetaData['Migrated']) self.assertEqual(fileMetaData["Size"], fileSize) # check the remove file operation self.assert_(removeFileRes["OK"]) self.assert_(removeFileRes["Value"]["Successful"].has_key(destFile)) # Check the get metadata for non existant file self.assert_(failedMetadataRes["OK"]) self.assert_(failedMetadataRes["Value"]["Failed"].has_key(destFile)) expectedError = "No such file or directory" self.assertEqual(failedMetadataRes["Value"]["Failed"][destFile], expectedError) # Check that metadata operation with a directory self.assert_(directoryMetadataRes["OK"]) self.assert_(directoryMetadataRes["Value"]["Failed"].has_key(destDir)) expectedError = "RFIOStorage.getFileMetadata: Supplied path is not a file." self.assertEqual(directoryMetadataRes["Value"]["Failed"][destDir], expectedError)
def test_putGetFileSize( self ): print '\n\n#########################################################################\n\n\t\t\tGet file size test\n' # First upload a file to the storage srcFile = '/etc/group' srcFileSize = getSize( srcFile ) testFileName = 'testFile.%s' % time.time() remoteFile = self.storage.getCurrentURL( testFileName )['Value'] fileDict = {remoteFile:srcFile} putFileRes = self.storage.putFile( fileDict ) # Check that we can get the file size getSizeRes = self.storage.getFileSize( remoteFile ) # Clean up the remote mess removeFileRes = self.storage.removeFile( remoteFile ) # Check non existant files failedSizeRes = self.storage.getFileMetadata( remoteFile ) # Check directories are handled properly remoteDir = os.path.dirname( remoteFile ) directorySizeRes = self.storage.getFileSize( remoteDir ) # Check the put file operation self.assert_( putFileRes['OK'] ) self.assert_( putFileRes['Value']['Successful'].has_key( remoteFile ) ) self.assert_( putFileRes['Value']['Successful'][remoteFile] ) self.assertEqual( putFileRes['Value']['Successful'][remoteFile], srcFileSize ) # Check that we got the file size correctly self.assert_( getSizeRes['OK'] ) self.assert_( getSizeRes['Value']['Successful'].has_key( remoteFile ) ) self.assertEqual( getSizeRes['Value']['Successful'][remoteFile], srcFileSize ) # check the remove file operation self.assert_( removeFileRes['OK'] ) self.assert_( removeFileRes['Value']['Successful'].has_key( remoteFile ) ) # Check the get size with non existant file works properly self.assert_( failedSizeRes['OK'] ) self.assert_( failedSizeRes['Value']['Failed'].has_key( remoteFile ) ) expectedError = "File does not exist" self.assert_( expectedError in failedSizeRes['Value']['Failed'][remoteFile] ) # Check that the passing a directory is handled correctly self.assert_( directorySizeRes['OK'] ) self.assert_( directorySizeRes['Value']['Failed'].has_key( remoteDir ) ) expectedError = "Supplied path is not a file" self.assert_( expectedError in directorySizeRes['Value']['Failed'][remoteDir] )
def test_putPrestageStatus( self ): print '\n\n#########################################################################\n\n\t\t\tPrestage status test\n' # First upload a file to the storage srcFile = '/etc/group' srcFileSize = getSize( srcFile ) testFileName = 'testFile.%s' % time.time() remoteFile = self.storage.getCurrentURL( testFileName )['Value'] fileDict = {remoteFile:srcFile} putFileRes = self.storage.putFile( fileDict ) # Check that we can issue a stage request prestageRes = self.storage.prestageFile( remoteFile ) srmID = '' if prestageRes['OK']: if prestageRes['Value']['Successful'].has_key( remoteFile ): srmID = prestageRes['Value']['Successful'][remoteFile] # Take a quick break to allow the SRM to realise the file is available sleepTime = 10 print 'Sleeping for %s seconds' % sleepTime time.sleep( sleepTime ) # Check that we can monitor the stage request prestageStatusRes = self.storage.prestageFileStatus( {remoteFile:srmID} ) # Clean up the remote mess removeFileRes = self.storage.removeFile( remoteFile ) # Check the put file operation self.assert_( putFileRes['OK'] ) self.assert_( putFileRes['Value']['Successful'].has_key( remoteFile ) ) self.assert_( putFileRes['Value']['Successful'][remoteFile] ) self.assertEqual( putFileRes['Value']['Successful'][remoteFile], srcFileSize ) # Check the prestage file operation self.assert_( prestageRes['OK'] ) self.assert_( prestageRes['Value']['Successful'].has_key( remoteFile ) ) self.assert_( prestageRes['Value']['Successful'][remoteFile] ) self.assert_( type( prestageRes['Value']['Successful'][remoteFile] ) in StringTypes ) # Check the prestage status operation self.assert_( prestageStatusRes['OK'] ) self.assert_( prestageStatusRes['Value']['Successful'].has_key( remoteFile ) ) self.assert_( prestageStatusRes['Value']['Successful'][remoteFile] ) # Check the remove file operation self.assert_( removeFileRes['OK'] ) self.assert_( removeFileRes['Value']['Successful'].has_key( remoteFile ) )
def __putFile(self, src_file, dest_url): res = pfnparse(src_file) if not res['OK']: return res localCache = False srcDict = res['Value'] if srcDict['Protocol'] in ['dips', 'dip']: # Make the service URL from the file URL by stripping off the file part serviceDict = dict(srcDict) serviceDict['Path'] = '/'.join(srcDict['Path'].split('/')[:3]) serviceDict['FileName'] = '' res = pfnunparse(serviceDict) if not res['OK']: return res srcSEURL = res['Value'] localCache = True transferClient = TransferClient(srcSEURL) res = transferClient.receiveFile( srcDict['FileName'], os.path.join( srcDict['Path'], srcDict['FileName'])) if not res['OK']: return res src_file = srcDict['FileName'] if not os.path.exists(src_file): errStr = "DIPStorage.__putFile: The source local file does not exist." gLogger.error(errStr, src_file) return S_ERROR(errStr) sourceSize = getSize(src_file) if sourceSize == -1: errStr = "DIPStorage.__putFile: Failed to get file size." gLogger.error(errStr, src_file) return S_ERROR(errStr) transferClient = TransferClient(self.url) res = transferClient.sendFile(src_file, dest_url, token=self.checkSum) if localCache: os.unlink(src_file) if res['OK']: return S_OK(sourceSize) else: return res
def test_putRemoveFile( self ): print '\n\n#########################################################################\n\n\t\t\tPut and Remove test\n' # Make sure that we can actually upload a file properly srcFile = '/etc/group' srcFileSize = getSize( srcFile ) testFileName = 'testFile.%s' % time.time() destFile = self.storage.getCurrentURL( testFileName )['Value'] fileDict = {destFile:srcFile} putFileRes = self.storage.putFile( fileDict ) # Make sure we are able to remove the file removeFileRes = self.storage.removeFile( destFile ) # Check the successful put file operation self.assert_( putFileRes['OK'] ) self.assert_( putFileRes['Value']['Successful'].has_key( destFile ) ) self.assertEqual( putFileRes['Value']['Successful'][destFile], srcFileSize ) # Check the remove file operation self.assert_( removeFileRes['OK'] ) self.assert_( removeFileRes['Value']['Successful'].has_key( destFile ) ) self.assert_( removeFileRes['Value']['Successful'][destFile] )
def getFile( self, path, localPath = False ): res = checkArgumentFormat( path ) if not res['OK']: return res urls = res['Value'] failed = {} successful = {} client = RPCClient( self.url ) # Make sure transferClient uses the same ProxyStorage instance. # Only the this one holds the file we want to transfer. transferClient = TransferClient( client.serviceURL ) for src_url in urls.keys(): res = client.prepareFile( self.name, src_url ) if not res['OK']: gLogger.error( "ProxyStorage.getFile: Failed to prepare file on remote server.", res['Message'] ) failed[src_url] = res['Message'] else: fileName = os.path.basename( src_url ) if localPath: dest_file = "%s/%s" % ( localPath, fileName ) else: dest_file = "%s/%s" % ( os.getcwd(), fileName ) res = transferClient.receiveFile( dest_file, 'getFile/%s' % fileName ) if not res['OK']: gLogger.error( "ProxyStorage.getFile: Failed to recieve file from proxy server.", res['Message'] ) failed[src_url] = res['Message'] elif not os.path.exists( dest_file ): errStr = "ProxyStorage.getFile: The destination local file does not exist." gLogger.error( errStr, dest_file ) failed[src_url] = errStr else: destSize = getSize( dest_file ) if destSize == -1: errStr = "ProxyStorage.getFile: Failed to get the local file size." gLogger.error( errStr, dest_file ) failed[src_url] = errStr else: successful[src_url] = destSize resDict = {'Failed':failed, 'Successful':successful} return S_OK( resDict )
def test_putGetFileSize(self): print "\n\n#########################################################################\n\n\t\t\tGet file size test\n" # First upload a file to the storage srcFile = "/etc/group" fileSize = getSize(srcFile) testFileName = "testFile.%s" % time.time() destFile = self.storage.getCurrentURL(testFileName)["Value"] fileTuple = (srcFile, destFile, fileSize) putFileRes = self.storage.putFile(fileTuple) # Check that we can get the file size getSizeRes = self.storage.getFileSize(destFile) # Clean up the remote mess removeFileRes = self.storage.removeFile(destFile) # Check non existant files failedSizeRes = self.storage.getFileMetadata(destFile) # Check directories are handled properly destDir = os.path.dirname(destFile) directorySizeRes = self.storage.getFileSize(destDir) # Check the put file operation self.assert_(putFileRes["OK"]) self.assert_(putFileRes["Value"]["Successful"].has_key(destFile)) # Check that we got the file size correctly self.assert_(getSizeRes["OK"]) self.assert_(getSizeRes["Value"]["Successful"].has_key(destFile)) self.assertEqual(getSizeRes["Value"]["Successful"][destFile], fileSize) # check the remove file operation self.assert_(removeFileRes["OK"]) self.assert_(removeFileRes["Value"]["Successful"].has_key(destFile)) # Check the get size with non existant file works properly self.assert_(failedSizeRes["OK"]) self.assert_(failedSizeRes["Value"]["Failed"].has_key(destFile)) expectedError = "No such file or directory" self.assertEqual(failedSizeRes["Value"]["Failed"][destFile], expectedError) # Check that the passing a directory is handled correctly self.assert_(directorySizeRes["OK"]) self.assert_(directorySizeRes["Value"]["Failed"].has_key(destDir)) expectedError = "RFIOStorage.getFileSize: Supplied path is not a file." self.assertEqual(directorySizeRes["Value"]["Failed"][destDir], expectedError)
def __getFile( self, src_url, dest_file ): """Get a local copy in the current directory of a physical file specified by its path """ if not os.path.exists( os.path.dirname( dest_file ) ): os.makedirs( os.path.dirname( dest_file ) ) if os.path.exists( dest_file ): gLogger.debug( "RFIOStorage.getFile: Local file already exists %s. Removing..." % dest_file ) os.remove( dest_file ) res = self.__executeOperation( src_url, 'getFileSize' ) if not res['OK']: return S_ERROR( res['Message'] ) remoteSize = res['Value'] MIN_BANDWIDTH = 1024 * 100 # 100 KB/s timeout = remoteSize / MIN_BANDWIDTH + 300 gLogger.debug( "RFIOStorage.getFile: Executing transfer of %s to %s" % ( src_url, dest_file ) ) comm = "rfcp %s %s" % ( src_url, dest_file ) res = shellCall( timeout, comm ) if res['OK']: returncode, _stdout, stderr = res['Value'] if returncode == 0: gLogger.debug( 'RFIOStorage.__getFile: Got file from storage, performing post transfer check.' ) localSize = getSize( dest_file ) if localSize == remoteSize: gLogger.debug( "RFIOStorage.getFile: Post transfer check successful." ) return S_OK( localSize ) errorMessage = "RFIOStorage.__getFile: Source and destination file sizes do not match." gLogger.error( errorMessage, src_url ) else: errStr = "RFIOStorage.__getFile: Failed to get local copy of file." gLogger.error( errStr, stderr ) errorMessage = "%s %s" % ( errStr, stderr ) else: errStr = "RFIOStorage.__getFile: Failed to get local copy of file." gLogger.error( errStr, res['Message'] ) errorMessage = "%s %s" % ( errStr, res['Message'] ) if os.path.exists( dest_file ): gLogger.debug( "RFIOStorage.getFile: Removing local file %s." % dest_file ) os.remove( dest_file ) return S_ERROR( errorMessage )
def test_putPrestageFile( self ): print '\n\n#########################################################################\n\n\t\t\tFile prestage test\n' # First upload a file to the storage srcFile = '/etc/group' fileSize = getSize( srcFile ) testFileName = 'testFile.%s' % time.time() destFile = self.storage.getCurrentURL( testFileName )['Value'] fileTuple = ( srcFile, destFile, fileSize ) putFileRes = self.storage.putFile( fileTuple ) # Check that we can issue a stage request prestageRes = self.storage.prestageFile( destFile ) # Clean up the remote mess removeFileRes = self.storage.removeFile( destFile ) # Check the put file operation self.assert_( putFileRes['OK'] ) self.assert_( putFileRes['Value']['Successful'].has_key( destFile ) ) # Check the prestage file operation self.assert_( prestageRes['OK'] ) self.assert_( prestageRes['Value']['Successful'].has_key( destFile ) ) self.assert_( prestageRes['Value']['Successful'][destFile] ) # Check the remove file operation self.assert_( removeFileRes['OK'] ) self.assert_( removeFileRes['Value']['Successful'].has_key( destFile ) )
def testGetSize( self ): """ getSize tests """ # non existing file self.assertEqual( getSize("/spam/eggs/eggs"), -1, "inexisting file" ) # file unreadable self.assertEqual( getSize('/root/.login'), -1 , "unreadable file")
def __putFile( self, src_file, dest_url, sourceSize ): """Put a copy of the local file to the current directory on the physical storage """ # Pre-transfer check res = self.__executeOperation( dest_url, 'exists' ) if not res['OK']: gLogger.debug( "RFIOStorage.__putFile: Failed to find pre-existance of destination file." ) return res if res['Value']: res = self.__executeOperation( dest_url, 'removeFile' ) if not res['OK']: gLogger.debug( "RFIOStorage.__putFile: Failed to remove remote file %s." % dest_url ) else: gLogger.debug( "RFIOStorage.__putFile: Removed remote file %s." % dest_url ) if not os.path.exists( src_file ): errStr = "RFIOStorage.__putFile: The source local file does not exist." gLogger.error( errStr, src_file ) return S_ERROR( errStr ) sourceSize = getSize( src_file ) if sourceSize == -1: errStr = "RFIOStorage.__putFile: Failed to get file size." gLogger.error( errStr, src_file ) return S_ERROR( errStr ) res = self.__getTransportURL( dest_url ) if not res['OK']: gLogger.debug( "RFIOStorage.__putFile: Failed to get transport URL for file." ) return res turl = res['Value'] MIN_BANDWIDTH = 1024 * 100 # 100 KB/s timeout = sourceSize / MIN_BANDWIDTH + 300 gLogger.debug( "RFIOStorage.putFile: Executing transfer of %s to %s" % ( src_file, turl ) ) comm = "rfcp %s '%s'" % ( src_file, turl ) res = shellCall( timeout, comm ) if res['OK']: returncode, _stdout, stderr = res['Value'] if returncode == 0: gLogger.debug( 'RFIOStorage.putFile: Put file to storage, performing post transfer check.' ) res = self.__executeOperation( dest_url, 'getFileSize' ) if res['OK']: destinationSize = res['Value'] if sourceSize == destinationSize : gLogger.debug( "RFIOStorage.__putFile: Post transfer check successful." ) return S_OK( destinationSize ) errorMessage = "RFIOStorage.__putFile: Source and destination file sizes do not match." gLogger.error( errorMessage, dest_url ) else: errStr = "RFIOStorage.__putFile: Failed to put file to remote storage." gLogger.error( errStr, stderr ) errorMessage = "%s %s" % ( errStr, stderr ) else: errStr = "RFIOStorage.__putFile: Failed to put file to remote storage." gLogger.error( errStr, res['Message'] ) errorMessage = "%s %s" % ( errStr, res['Message'] ) res = self.__executeOperation( dest_url, 'removeFile' ) if res['OK']: gLogger.debug( "RFIOStorage.__putFile: Removed remote file remnant %s." % dest_url ) else: gLogger.debug( "RFIOStorage.__putFile: Unable to remove remote file remnant %s." % dest_url ) return S_ERROR( errorMessage )