def test_putRemoveFile( self ):
print('\n\n#########################################################'
'################\n\n\t\t\tPut and Remove test\n')
# First test that we are able to determine whether the file sizes of the transfer don't match
srcFile = '/etc/group'
fileSize = 10 #This is a made up value
testFileName = 'testFile.%s' % time.time()
destFile = self.storage.getCurrentURL( testFileName )['Value']
fileTuple = ( srcFile, destFile, fileSize )
failedPutFileRes = self.storage.putFile( fileTuple )
# Now make sure that we can actually upload a file properly
fileSize = getSize( srcFile )
fileTuple = ( srcFile, destFile, fileSize )
putFileRes = self.storage.putFile( fileTuple )
# Make sure we are able to remove the file
removeFileRes = self.storage.removeFile( destFile )
# Check the failed put file operation
self.assertTrue( failedPutFileRes['OK'] )
self.assertTrue( destFile in failedPutFileRes['Value']['Failed'] )
expectedError = 'RFIOStorage.putFile: Source and destination file sizes do not match.'
self.assertEqual( failedPutFileRes['Value']['Failed'][destFile], expectedError )
# Check the successful put file operation
self.assertTrue( putFileRes['OK'] )
self.assertTrue( destFile in putFileRes['Value']['Successful'] )
# Check the remove file operation
self.assertTrue( removeFileRes['OK'] )
self.assertTrue( destFile in removeFileRes['Value']['Successful'] )
def test_putFilegetTransportURL( self ):
print '\n\n#########################################################################\n\n\t\t\tGet tURL test\n'
# First upload a file to the storage
srcFile = '/etc/group'
fileSize = getSize( srcFile )
testFileName = 'testFile.%s' % time.time()
destFile = self.storage.getCurrentURL( testFileName )['Value']
fileTuple = ( srcFile, destFile, fileSize )
putFileRes = self.storage.putFile( fileTuple )
#Check that we can get a turl
getTurlRes = self.storage.getTransportURL( destFile, ['dcap', 'gsidcap'] )
# Clean up the remote mess
removeFileRes = self.storage.removeFile( destFile )
# Try and get a turl for a non existant file
failedGetTurlRes = self.storage.getTransportURL( destFile, ['dcap', 'gsidcap'] )
# Check the put file operation
print putFileRes
self.assert_( putFileRes['OK'] )
self.assert_( putFileRes['Value']['Successful'].has_key( destFile ) )
# check the get turl operation
print getTurlRes, destFile
self.assert_( getTurlRes['OK'] )
self.assert_( getTurlRes['Value']['Successful'].has_key( destFile ) )
# check the remove file operation
self.assert_( removeFileRes['OK'] )
self.assert_( removeFileRes['Value']['Successful'].has_key( destFile ) )
#Check the get turl with non existant file operation
self.assert_( failedGetTurlRes['OK'] )
self.assert_( failedGetTurlRes['Value']['Failed'].has_key( destFile ) )
expectedError = "RFIOStorage.getTransportURL: File does not exist."
self.assertEqual( failedGetTurlRes['Value']['Failed'][destFile], expectedError )
def test_putIsFile( self ):
print '\n\n#########################################################################\n\n\t\t\tIs file test\n'
# First upload a file to the storage
srcFile = '/etc/group'
fileSize = getSize( srcFile )
testFileName = 'testFile.%s' % time.time()
destFile = self.storage.getCurrentURL( testFileName )['Value']
fileTuple = ( srcFile, destFile, fileSize )
putFileRes = self.storage.putFile( fileTuple )
# Check we are able to determine that it is a file
isFileRes = self.storage.isFile( destFile )
# Clean up the remote mess
removeFileRes = self.storage.removeFile( destFile )
# Check that everything isn't a file
destDir = os.path.dirname( destFile )
failedIsFileRes = self.storage.isFile( destDir )
# Check the put file operation
self.assert_( putFileRes['OK'] )
self.assert_( putFileRes['Value']['Successful'].has_key( destFile ) )
# Check the is file operation
self.assert_( isFileRes['OK'] )
self.assert_( isFileRes['Value']['Successful'].has_key( destFile ) )
self.assert_( isFileRes['Value']['Successful'][destFile] )
# check the remove file operation
self.assert_( removeFileRes['OK'] )
self.assert_( removeFileRes['Value']['Successful'].has_key( destFile ) )
# Check that the directory is not a file
self.assert_( failedIsFileRes['OK'] )
self.assert_( failedIsFileRes['Value']['Successful'].has_key( destDir ) )
self.assertFalse( failedIsFileRes['Value']['Successful'][destDir] )
def test_putExistsFile(self):
print '\n\n#########################################################################\n\n\t\t\tExists test\n'
# First upload a file to the storage
srcFile = '/etc/group'
srcFileSize = getSize(srcFile)
testFileName = 'testFile.%s' % time.time()
remoteFile = self.storage.getCurrentURL(testFileName)['Value']
fileDict = {remoteFile: srcFile}
putFileRes = self.storage.putFile(fileDict)
# Then get the file's existance
existsFileRes = self.storage.exists(remoteFile)
# Now remove the file
removeFileRes = self.storage.removeFile(remoteFile)
# Check again that the file exists
failedExistRes = self.storage.exists(remoteFile)
# Check the put file operation
self.assert_(putFileRes['OK'])
self.assert_(putFileRes['Value']['Successful'].has_key(remoteFile))
self.assertEqual(putFileRes['Value']['Successful'][remoteFile],
srcFileSize)
# Check the exists operation
self.assert_(existsFileRes['OK'])
self.assert_(existsFileRes['Value']['Successful'].has_key(remoteFile))
self.assert_(existsFileRes['Value']['Successful'][remoteFile])
# Check the removal operation
self.assert_(removeFileRes['OK'])
self.assert_(removeFileRes['Value']['Successful'].has_key(remoteFile))
# Check the failed exists operation
self.assert_(failedExistRes['OK'])
self.assert_(failedExistRes['Value']['Successful'].has_key(remoteFile))
self.assertFalse(failedExistRes['Value']['Successful'][remoteFile])
def __putFile(self, src_file, dest_url):
res = pfnparse(src_file)
if not res['OK']:
return res
localCache = False
srcDict = res['Value']
if srcDict['Protocol'] in ['dips', 'dip']:
localCache = True
srcSEURL = srcDict['Protocol'] + '://' + srcDict[
'Host'] + ':' + srcDict['Port'] + srcDict['WSUrl']
transferClient = TransferClient(srcSEURL)
res = transferClient.receiveFile(
srcDict['FileName'],
os.path.join(srcDict['Path'], srcDict['FileName']))
if not res['OK']:
return res
src_file = srcDict['FileName']
if not os.path.exists(src_file):
errStr = "DIPStorage.__putFile: The source local file does not exist."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
sourceSize = getSize(src_file)
if sourceSize == -1:
errStr = "DIPStorage.__putFile: Failed to get file size."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
transferClient = TransferClient(self.url)
res = transferClient.sendFile(src_file, dest_url, token=self.checkSum)
if localCache:
os.unlink(src_file)
if res['OK']:
return S_OK(sourceSize)
else:
return res
def test_putRemoveFile( self ):
print '\n\n#########################################################################\n\n\t\t\tPut and Remove test\n'
# First test that we are able to determine whether the file sizes of the transfer don't match
srcFile = '/etc/group'
fileSize = 10 #This is a made up value
testFileName = 'testFile.%s' % time.time()
destFile = self.storage.getCurrentURL( testFileName )['Value']
fileTuple = ( srcFile, destFile, fileSize )
failedPutFileRes = self.storage.putFile( fileTuple )
# Now make sure that we can actually upload a file properly
fileSize = getSize( srcFile )
fileTuple = ( srcFile, destFile, fileSize )
putFileRes = self.storage.putFile( fileTuple )
# Make sure we are able to remove the file
removeFileRes = self.storage.removeFile( destFile )
# Check the failed put file operation
self.assert_( failedPutFileRes['OK'] )
self.assert_( failedPutFileRes['Value']['Failed'].has_key( destFile ) )
expectedError = 'RFIOStorage.putFile: Source and destination file sizes do not match.'
self.assertEqual( failedPutFileRes['Value']['Failed'][destFile], expectedError )
# Check the successful put file operation
self.assert_( putFileRes['OK'] )
self.assert_( putFileRes['Value']['Successful'].has_key( destFile ) )
# Check the remove file operation
self.assert_( removeFileRes['OK'] )
self.assert_( removeFileRes['Value']['Successful'].has_key( destFile ) )
def test_putGetFile( self ):
print '\n\n#########################################################################\n\n\t\t\tPut and Get test\n'
# First upload a file to the storage
srcFile = '/etc/group'
srcFileSize = getSize( srcFile )
testFileName = 'testFile.%s' % time.time()
remoteFile = self.storage.getCurrentURL( testFileName )['Value']
fileDict = {remoteFile:srcFile}
putFileRes = self.storage.putFile( fileDict )
# Then make sure we can get a local copy of the file
getFileRes = self.storage.getFile( remoteFile )
# Cleanup the remote mess
removeFileRes = self.storage.removeFile( remoteFile )
#Cleanup the mess locally
os.remove( testFileName )
# Check the put operation
self.assert_( putFileRes['OK'] )
self.assert_( putFileRes['Value']['Successful'].has_key( remoteFile ) )
self.assertEqual( putFileRes['Value']['Successful'][remoteFile], srcFileSize )
# Check the get operation
self.assert_( getFileRes['OK'] )
self.assert_( getFileRes['Value']['Successful'].has_key( remoteFile ) )
self.assertEqual( getFileRes['Value']['Successful'][remoteFile], srcFileSize )
# Check the remove operation
self.assert_( removeFileRes['OK'] )
self.assert_( removeFileRes['Value']['Successful'].has_key( remoteFile ) )
self.assert_( removeFileRes['Value']['Successful'][remoteFile] )
def test_putIsFile(self):
print '\n\n#########################################################################\n\n\t\t\tIs file test\n'
# First upload a file to the storage
srcFile = '/etc/group'
srcFileSize = getSize(srcFile)
testFileName = 'testFile.%s' % time.time()
remoteFile = self.storage.getCurrentURL(testFileName)['Value']
fileDict = {remoteFile: srcFile}
putFileRes = self.storage.putFile(fileDict)
# Check we are able to determine that it is a file
isFileRes = self.storage.isFile(remoteFile)
# Clean up the remote mess
removeFileRes = self.storage.removeFile(remoteFile)
# Check that everything isn't a file
remoteDir = os.path.dirname(remoteFile)
failedIsFileRes = self.storage.isFile(remoteDir)
# Check the put file operation
self.assert_(putFileRes['OK'])
self.assert_(putFileRes['Value']['Successful'].has_key(remoteFile))
self.assert_(putFileRes['Value']['Successful'][remoteFile])
self.assertEqual(putFileRes['Value']['Successful'][remoteFile],
srcFileSize)
# Check the is file operation
self.assert_(isFileRes['OK'])
self.assert_(isFileRes['Value']['Successful'].has_key(remoteFile))
self.assert_(isFileRes['Value']['Successful'][remoteFile])
# check the remove file operation
self.assert_(removeFileRes['OK'])
self.assert_(removeFileRes['Value']['Successful'].has_key(remoteFile))
# Check that the directory is not a file
self.assert_(failedIsFileRes['OK'])
self.assert_(failedIsFileRes['Value']['Successful'].has_key(remoteDir))
self.assertFalse(failedIsFileRes['Value']['Successful'][remoteDir])
def test_putExistsFile( self ):
print('\n\n#########################################################'
'################\n\n\t\t\tExists test\n')
# First upload a file to the storage
srcFile = '/etc/group'
fileSize = getSize( srcFile )
testFileName = 'testFile.%s' % time.time()
destFile = self.storage.getCurrentURL( testFileName )['Value']
fileTuple = ( srcFile, destFile, fileSize )
putFileRes = self.storage.putFile( fileTuple )
# Then get the file's existance
existsFileRes = self.storage.exists( destFile )
# Now remove the file
removeFileRes = self.storage.removeFile( destFile )
# Check again that the file exists
failedExistRes = self.storage.exists( destFile )
# Check the put file operation
self.assertTrue( putFileRes['OK'] )
self.assertTrue( destFile in putFileRes['Value']['Successful'] )
# Check the exists operation
self.assertTrue( existsFileRes['OK'] )
self.assertTrue( destFile in existsFileRes['Value']['Successful'] )
self.assertTrue( existsFileRes['Value']['Successful'][destFile] )
# Check the removal operation
self.assertTrue( removeFileRes['OK'] )
self.assertTrue( destFile in removeFileRes['Value']['Successful'] )
# Check the failed exists operation
self.assertTrue( failedExistRes['OK'] )
self.assertTrue( destFile in failedExistRes['Value']['Successful'] )
self.assertFalse( failedExistRes['Value']['Successful'][destFile] )
def test_putPinRelease( self ):
print '\n\n#########################################################################\n\n\t\t\tPin and Release test\n'
# First upload a file to the storage
srcFile = '/etc/group'
srcFileSize = getSize( srcFile )
testFileName = 'testFile.%s' % time.time()
remoteFile = self.storage.getCurrentURL( testFileName )['Value']
fileDict = {remoteFile:srcFile}
putFileRes = self.storage.putFile( fileDict )
# Check that we can pin the file
pinFileRes = self.storage.pinFile( remoteFile )
srmID = ''
if pinFileRes['OK']:
if pinFileRes['Value']['Successful'].has_key( remoteFile ):
srmID = pinFileRes['Value']['Successful'][remoteFile]
# Check that we can release the file
releaseFileRes = self.storage.releaseFile( {remoteFile:srmID} )
# Clean up the mess
removeFileRes = self.storage.removeFile( remoteFile )
# Check the put file operation
self.assert_( putFileRes['OK'] )
self.assert_( putFileRes['Value']['Successful'].has_key( remoteFile ) )
self.assert_( putFileRes['Value']['Successful'][remoteFile] )
self.assertEqual( putFileRes['Value']['Successful'][remoteFile], srcFileSize )
# Check the pin file operation
self.assert_( pinFileRes['OK'] )
self.assert_( pinFileRes['Value']['Successful'].has_key( remoteFile ) )
self.assert_( type( pinFileRes['Value']['Successful'][remoteFile] ) in StringTypes )
# Check the release file operation
self.assert_( releaseFileRes['OK'] )
self.assert_( releaseFileRes['Value']['Successful'].has_key( remoteFile ) )
# check the remove file operation
self.assert_( removeFileRes['OK'] )
self.assert_( removeFileRes['Value']['Successful'].has_key( remoteFile ) )
def test_putFilegetTransportURL(self):
print("\n\n#########################################################"
"################\n\n\t\t\tGet tURL test\n")
# First upload a file to the storage
srcFile = "/etc/group"
srcFileSize = getSize(srcFile)
testFileName = "testFile.%s" % time.time()
remoteFile = self.storage.getCurrentURL(testFileName)["Value"]
fileDict = {remoteFile: srcFile}
putFileRes = self.storage.putFile(fileDict)
# Check that we can get a turl
getTurlRes = self.storage.getTransportURL(remoteFile)
# Clean up the remote mess
removeFileRes = self.storage.removeFile(remoteFile)
# Try and get a turl for a non existant file
failedGetTurlRes = self.storage.getTransportURL(remoteFile)
# Check the put file operation
self.assertTrue(putFileRes["OK"])
self.assertTrue(remoteFile in putFileRes["Value"]["Successful"])
self.assertTrue(putFileRes["Value"]["Successful"][remoteFile])
self.assertEqual(putFileRes["Value"]["Successful"][remoteFile],
srcFileSize)
# check the get turl operation
self.assertTrue(getTurlRes["OK"])
self.assertTrue(remoteFile in getTurlRes["Value"]["Successful"])
# check the remove file operation
self.assertTrue(removeFileRes["OK"])
self.assertTrue(remoteFile in removeFileRes["Value"]["Successful"])
# Check the get turl with non existant file operation
self.assertTrue(failedGetTurlRes["OK"])
self.assertTrue(remoteFile in failedGetTurlRes["Value"]["Failed"])
expectedError = "File does not exist"
self.assertTrue(
expectedError in failedGetTurlRes["Value"]["Failed"][remoteFile])
def test_putFilegetTransportURL( self ):
print '\n\n#########################################################################\n\n\t\t\tGet tURL test\n'
# First upload a file to the storage
srcFile = '/etc/group'
srcFileSize = getSize( srcFile )
testFileName = 'testFile.%s' % time.time()
remoteFile = self.storage.getCurrentURL( testFileName )['Value']
fileDict = {remoteFile:srcFile}
putFileRes = self.storage.putFile( fileDict )
#Check that we can get a turl
getTurlRes = self.storage.getTransportURL( remoteFile )
# Clean up the remote mess
removeFileRes = self.storage.removeFile( remoteFile )
# Try and get a turl for a non existant file
failedGetTurlRes = self.storage.getTransportURL( remoteFile )
# Check the put file operation
self.assert_( putFileRes['OK'] )
self.assert_( putFileRes['Value']['Successful'].has_key( remoteFile ) )
self.assert_( putFileRes['Value']['Successful'][remoteFile] )
self.assertEqual( putFileRes['Value']['Successful'][remoteFile], srcFileSize )
# check the get turl operation
self.assert_( getTurlRes['OK'] )
self.assert_( getTurlRes['Value']['Successful'].has_key( remoteFile ) )
# check the remove file operation
self.assert_( removeFileRes['OK'] )
self.assert_( removeFileRes['Value']['Successful'].has_key( remoteFile ) )
#Check the get turl with non existant file operation
self.assert_( failedGetTurlRes['OK'] )
self.assert_( failedGetTurlRes['Value']['Failed'].has_key( remoteFile ) )
expectedError = "File does not exist"
self.assert_( expectedError in failedGetTurlRes['Value']['Failed'][remoteFile] )
def test_putGetFile(self):
print("\n\n#########################################################"
"################\n\n\t\t\tPut and Get test\n")
# First upload a file to the storage
srcFile = "/etc/group"
srcFileSize = getSize(srcFile)
testFileName = "testFile.%s" % time.time()
remoteFile = self.storage.getCurrentURL(testFileName)["Value"]
fileDict = {remoteFile: srcFile}
putFileRes = self.storage.putFile(fileDict)
# Then make sure we can get a local copy of the file
getFileRes = self.storage.getFile(remoteFile)
# Cleanup the remote mess
removeFileRes = self.storage.removeFile(remoteFile)
# Cleanup the mess locally
os.remove(testFileName)
# Check the put operation
self.assertTrue(putFileRes["OK"])
self.assertTrue(remoteFile in putFileRes["Value"]["Successful"])
self.assertEqual(putFileRes["Value"]["Successful"][remoteFile],
srcFileSize)
# Check the get operation
self.assertTrue(getFileRes["OK"])
self.assertTrue(remoteFile in getFileRes["Value"]["Successful"])
self.assertEqual(getFileRes["Value"]["Successful"][remoteFile],
srcFileSize)
# Check the remove operation
self.assertTrue(removeFileRes["OK"])
self.assertTrue(remoteFile in removeFileRes["Value"]["Successful"])
self.assertTrue(removeFileRes["Value"]["Successful"][remoteFile])
def test_putIsFile( self ):
print('\n\n#########################################################'
'################\n\n\t\t\tIs file test\n')
# First upload a file to the storage
srcFile = '/etc/group'
fileSize = getSize( srcFile )
testFileName = 'testFile.%s' % time.time()
destFile = self.storage.getCurrentURL( testFileName )['Value']
fileTuple = ( srcFile, destFile, fileSize )
putFileRes = self.storage.putFile( fileTuple )
# Check we are able to determine that it is a file
isFileRes = self.storage.isFile( destFile )
# Clean up the remote mess
removeFileRes = self.storage.removeFile( destFile )
# Check that everything isn't a file
destDir = os.path.dirname( destFile )
failedIsFileRes = self.storage.isFile( destDir )
# Check the put file operation
self.assertTrue( putFileRes['OK'] )
self.assertTrue( destFile in putFileRes['Value']['Successful'] )
# Check the is file operation
self.assertTrue( isFileRes['OK'] )
self.assertTrue( destFile in isFileRes['Value']['Successful'] )
self.assertTrue( isFileRes['Value']['Successful'][destFile] )
# check the remove file operation
self.assertTrue( removeFileRes['OK'] )
self.assertTrue( destFile in removeFileRes['Value']['Successful'] )
# Check that the directory is not a file
self.assertTrue( failedIsFileRes['OK'] )
self.assertTrue( destDir in failedIsFileRes['Value']['Successful'] )
self.assertFalse( failedIsFileRes['Value']['Successful'][destDir] )
def test_putFilegetTransportURL(self):
print '\n\n#########################################################################\n\n\t\t\tGet tURL test\n'
# First upload a file to the storage
srcFile = '/etc/group'
srcFileSize = getSize(srcFile)
testFileName = 'testFile.%s' % time.time()
remoteFile = self.storage.getCurrentURL(testFileName)['Value']
fileDict = {remoteFile: srcFile}
putFileRes = self.storage.putFile(fileDict)
#Check that we can get a turl
getTurlRes = self.storage.getTransportURL(remoteFile)
# Clean up the remote mess
removeFileRes = self.storage.removeFile(remoteFile)
# Try and get a turl for a non existant file
failedGetTurlRes = self.storage.getTransportURL(remoteFile)
# Check the put file operation
self.assert_(putFileRes['OK'])
self.assert_(putFileRes['Value']['Successful'].has_key(remoteFile))
self.assert_(putFileRes['Value']['Successful'][remoteFile])
self.assertEqual(putFileRes['Value']['Successful'][remoteFile],
srcFileSize)
# check the get turl operation
self.assert_(getTurlRes['OK'])
self.assert_(getTurlRes['Value']['Successful'].has_key(remoteFile))
# check the remove file operation
self.assert_(removeFileRes['OK'])
self.assert_(removeFileRes['Value']['Successful'].has_key(remoteFile))
#Check the get turl with non existant file operation
self.assert_(failedGetTurlRes['OK'])
self.assert_(failedGetTurlRes['Value']['Failed'].has_key(remoteFile))
expectedError = "File does not exist"
self.assert_(
expectedError in failedGetTurlRes['Value']['Failed'][remoteFile])
def test_putPrestageFile( self ):
print('\n\n#########################################################'
'################\n\n\t\t\tFile prestage test\n')
# First upload a file to the storage
srcFile = '/etc/group'
fileSize = getSize( srcFile )
testFileName = 'testFile.%s' % time.time()
destFile = self.storage.getCurrentURL( testFileName )['Value']
fileTuple = ( srcFile, destFile, fileSize )
putFileRes = self.storage.putFile( fileTuple )
# Check that we can issue a stage request
prestageRes = self.storage.prestageFile( destFile )
# Clean up the remote mess
removeFileRes = self.storage.removeFile( destFile )
# Check the put file operation
self.assertTrue( putFileRes['OK'] )
self.assertTrue( destFile in putFileRes['Value']['Successful'] )
# Check the prestage file operation
self.assertTrue( prestageRes['OK'] )
self.assertTrue( destFile in prestageRes['Value']['Successful'] )
self.assertTrue( prestageRes['Value']['Successful'][destFile] )
# Check the remove file operation
self.assertTrue( removeFileRes['OK'] )
self.assertTrue( destFile in removeFileRes['Value']['Successful'] )
def test_putPrestageFile(self):
print("\n\n#########################################################"
"################\n\n\t\t\tFile prestage test\n")
# First upload a file to the storage
srcFile = "/etc/group"
fileSize = getSize(srcFile)
testFileName = "testFile.%s" % time.time()
destFile = self.storage.getCurrentURL(testFileName)["Value"]
fileTuple = (srcFile, destFile, fileSize)
putFileRes = self.storage.putFile(fileTuple)
# Check that we can issue a stage request
prestageRes = self.storage.prestageFile(destFile)
# Clean up the remote mess
removeFileRes = self.storage.removeFile(destFile)
# Check the put file operation
self.assertTrue(putFileRes["OK"])
self.assertTrue(destFile in putFileRes["Value"]["Successful"])
# Check the prestage file operation
self.assertTrue(prestageRes["OK"])
self.assertTrue(destFile in prestageRes["Value"]["Successful"])
self.assertTrue(prestageRes["Value"]["Successful"][destFile])
# Check the remove file operation
self.assertTrue(removeFileRes["OK"])
self.assertTrue(destFile in removeFileRes["Value"]["Successful"])
def test_putPrestageFile( self ):
print '\n\n#########################################################################\n\n\t\t\tFile prestage test\n'
# First upload a file to the storage
srcFile = '/etc/group'
srcFileSize = getSize( srcFile )
testFileName = 'testFile.%s' % time.time()
remoteFile = self.storage.getCurrentURL( testFileName )['Value']
fileDict = {remoteFile:srcFile}
putFileRes = self.storage.putFile( fileDict )
# Check that we can issue a stage request
prestageRes = self.storage.prestageFile( remoteFile )
# Clean up the remote mess
removeFileRes = self.storage.removeFile( remoteFile )
# Check what happens with deleted files
deletedPrestageRes = self.storage.prestageFile( remoteFile )
# Check the put file operation
self.assert_( putFileRes['OK'] )
self.assert_( putFileRes['Value']['Successful'].has_key( remoteFile ) )
self.assert_( putFileRes['Value']['Successful'][remoteFile] )
self.assertEqual( putFileRes['Value']['Successful'][remoteFile], srcFileSize )
# Check the prestage file operation
self.assert_( prestageRes['OK'] )
self.assert_( prestageRes['Value']['Successful'].has_key( remoteFile ) )
self.assert_( prestageRes['Value']['Successful'][remoteFile] )
# Check the remove file operation
self.assert_( removeFileRes['OK'] )
self.assert_( removeFileRes['Value']['Successful'].has_key( remoteFile ) )
# Check that pre-staging non-existant file fails
self.assert_( deletedPrestageRes['OK'] )
self.assert_( deletedPrestageRes['Value']['Failed'].has_key( remoteFile ) )
expectedError = "No such file or directory"
self.assert_( expectedError in deletedPrestageRes['Value']['Failed'][remoteFile] )
def test_putFilegetTransportURL( self ):
print('\n\n#########################################################'
'################\n\n\t\t\tGet tURL test\n')
# First upload a file to the storage
srcFile = '/etc/group'
fileSize = getSize( srcFile )
testFileName = 'testFile.%s' % time.time()
destFile = self.storage.getCurrentURL( testFileName )['Value']
fileTuple = ( srcFile, destFile, fileSize )
putFileRes = self.storage.putFile( fileTuple )
#Check that we can get a turl
getTurlRes = self.storage.getTransportURL( destFile, ['dcap', 'gsidcap'] )
# Clean up the remote mess
removeFileRes = self.storage.removeFile( destFile )
# Try and get a turl for a non existant file
failedGetTurlRes = self.storage.getTransportURL( destFile, ['dcap', 'gsidcap'] )
# Check the put file operation
print(putFileRes)
self.assertTrue( putFileRes['OK'] )
self.assertTrue( destFile in putFileRes['Value']['Successful'] )
# check the get turl operation
print(getTurlRes, destFile)
self.assertTrue( getTurlRes['OK'] )
self.assertTrue( destFile in getTurlRes['Value']['Successful'] )
# check the remove file operation
self.assertTrue( removeFileRes['OK'] )
self.assertTrue( destFile in removeFileRes['Value']['Successful'] )
#Check the get turl with non existant file operation
self.assertTrue( failedGetTurlRes['OK'] )
self.assertTrue( destFile in failedGetTurlRes['Value']['Failed'] )
expectedError = "RFIOStorage.getTransportURL: File does not exist."
self.assertEqual( failedGetTurlRes['Value']['Failed'][destFile], expectedError )
def test_putExistsFile( self ):
print '\n\n#########################################################################\n\n\t\t\tExists test\n'
# First upload a file to the storage
srcFile = '/etc/group'
fileSize = getSize( srcFile )
testFileName = 'testFile.%s' % time.time()
destFile = self.storage.getCurrentURL( testFileName )['Value']
fileTuple = ( srcFile, destFile, fileSize )
putFileRes = self.storage.putFile( fileTuple )
# Then get the file's existance
existsFileRes = self.storage.exists( destFile )
# Now remove the file
removeFileRes = self.storage.removeFile( destFile )
# Check again that the file exists
failedExistRes = self.storage.exists( destFile )
# Check the put file operation
self.assert_( putFileRes['OK'] )
self.assert_( putFileRes['Value']['Successful'].has_key( destFile ) )
# Check the exists operation
self.assert_( existsFileRes['OK'] )
self.assert_( existsFileRes['Value']['Successful'].has_key( destFile ) )
self.assert_( existsFileRes['Value']['Successful'][destFile] )
# Check the removal operation
self.assert_( removeFileRes['OK'] )
self.assert_( removeFileRes['Value']['Successful'].has_key( destFile ) )
# Check the failed exists operation
self.assert_( failedExistRes['OK'] )
self.assert_( failedExistRes['Value']['Successful'].has_key( destFile ) )
self.assertFalse( failedExistRes['Value']['Successful'][destFile] )
def test_putPrestageFile(self):
print '\n\n#########################################################################\n\n\t\t\tFile prestage test\n'
# First upload a file to the storage
srcFile = '/etc/group'
srcFileSize = getSize(srcFile)
testFileName = 'testFile.%s' % time.time()
remoteFile = self.storage.getCurrentURL(testFileName)['Value']
fileDict = {remoteFile: srcFile}
putFileRes = self.storage.putFile(fileDict)
# Check that we can issue a stage request
prestageRes = self.storage.prestageFile(remoteFile)
# Clean up the remote mess
removeFileRes = self.storage.removeFile(remoteFile)
# Check what happens with deleted files
deletedPrestageRes = self.storage.prestageFile(remoteFile)
# Check the put file operation
self.assert_(putFileRes['OK'])
self.assert_(putFileRes['Value']['Successful'].has_key(remoteFile))
self.assert_(putFileRes['Value']['Successful'][remoteFile])
self.assertEqual(putFileRes['Value']['Successful'][remoteFile],
srcFileSize)
# Check the prestage file operation
self.assert_(prestageRes['OK'])
self.assert_(prestageRes['Value']['Successful'].has_key(remoteFile))
self.assert_(prestageRes['Value']['Successful'][remoteFile])
# Check the remove file operation
self.assert_(removeFileRes['OK'])
self.assert_(removeFileRes['Value']['Successful'].has_key(remoteFile))
# Check that pre-staging non-existant file fails
self.assert_(deletedPrestageRes['OK'])
self.assert_(deletedPrestageRes['Value']['Failed'].has_key(remoteFile))
expectedError = "No such file or directory"
self.assert_(
expectedError in deletedPrestageRes['Value']['Failed'][remoteFile])
def test_putGetFile(self):
print '\n\n#########################################################################\n\n\t\t\tPut and Get test\n'
# First upload a file to the storage
srcFile = '/etc/group'
srcFileSize = getSize(srcFile)
testFileName = 'testFile.%s' % time.time()
remoteFile = self.storage.getCurrentURL(testFileName)['Value']
fileDict = {remoteFile: srcFile}
putFileRes = self.storage.putFile(fileDict)
# Then make sure we can get a local copy of the file
getFileRes = self.storage.getFile(remoteFile)
# Cleanup the remote mess
removeFileRes = self.storage.removeFile(remoteFile)
#Cleanup the mess locally
os.remove(testFileName)
# Check the put operation
self.assert_(putFileRes['OK'])
self.assert_(putFileRes['Value']['Successful'].has_key(remoteFile))
self.assertEqual(putFileRes['Value']['Successful'][remoteFile],
srcFileSize)
# Check the get operation
self.assert_(getFileRes['OK'])
self.assert_(getFileRes['Value']['Successful'].has_key(remoteFile))
self.assertEqual(getFileRes['Value']['Successful'][remoteFile],
srcFileSize)
# Check the remove operation
self.assert_(removeFileRes['OK'])
self.assert_(removeFileRes['Value']['Successful'].has_key(remoteFile))
self.assert_(removeFileRes['Value']['Successful'][remoteFile])
def test_getDirectoryMetadata(self):
print '\n\n#########################################################################\n\n\t\t\tDirectory metadata test\n'
directory = "%s/%s" % (self.destDirectory, 'getDirectoryMetadataTest')
destDirectory = self.storageElement.getPfnForLfn(directory)['Value']
# Create a local directory to upload
localDir = '/tmp/unit-test'
srcFile = '/etc/group'
sizeOfLocalFile = getSize(srcFile)
if not os.path.exists(localDir):
os.mkdir(localDir)
for i in range(self.numberOfFiles):
shutil.copy(srcFile, '%s/testFile.%s' % (localDir, time.time()))
time.sleep(1)
# Check that we can successfully upload the directory to the storage element
dirDict = {destDirectory: localDir}
putDirRes = self.storageElement.putDirectory(dirDict,
singleDirectory=True)
# Get the directory metadata
metadataDirRes = self.storageElement.getDirectoryMetadata(
destDirectory, singleDirectory=True)
# Now remove the remove directory
removeDirRes = self.storageElement.removeDirectory(
destDirectory, recursive=True, singleDirectory=True)
#Clean up the locally created directory
shutil.rmtree(localDir)
# Perform the checks for the put dir operation
self.assert_(putDirRes['OK'])
self.assert_(putDirRes['Value'])
if putDirRes['Value']['Files']:
self.assertEqual(putDirRes['Value']['Files'], self.numberOfFiles)
self.assertEqual(putDirRes['Value']['Size'],
self.numberOfFiles * sizeOfLocalFile)
self.assert_(
type(putDirRes['Value']['Files']) in
[types.IntType, types.LongType])
self.assert_(
type(putDirRes['Value']['Size']) in
[types.IntType, types.LongType])
# Perform the checks for the list dir operation
self.assert_(metadataDirRes['OK'])
self.assert_(metadataDirRes['Value'])
self.assert_(metadataDirRes['Value']['Mode'])
self.assert_(type(metadataDirRes['Value']['Mode']) == types.IntType)
self.assert_(metadataDirRes['Value']['Directory'])
self.assertFalse(metadataDirRes['Value']['File'])
# Perform the checks for the remove directory operation
self.assert_(removeDirRes['OK'])
self.assert_(removeDirRes['Value'])
if removeDirRes['Value']['FilesRemoved']:
self.assertEqual(removeDirRes['Value']['FilesRemoved'],
self.numberOfFiles)
self.assertEqual(removeDirRes['Value']['SizeRemoved'],
self.numberOfFiles * sizeOfLocalFile)
self.assert_(
type(removeDirRes['Value']['FilesRemoved']) in
[types.IntType, types.LongType])
self.assert_(
type(removeDirRes['Value']['SizeRemoved']) in
[types.IntType, types.LongType])
def test_putExistsFile(self):
print("\n\n#########################################################"
"################\n\n\t\t\tExists test\n")
# First upload a file to the storage
srcFile = "/etc/group"
srcFileSize = getSize(srcFile)
testFileName = "testFile.%s" % time.time()
remoteFile = self.storage.getCurrentURL(testFileName)["Value"]
fileDict = {remoteFile: srcFile}
putFileRes = self.storage.putFile(fileDict)
# Then get the file's existance
existsFileRes = self.storage.exists(remoteFile)
# Now remove the file
removeFileRes = self.storage.removeFile(remoteFile)
# Check again that the file exists
failedExistRes = self.storage.exists(remoteFile)
# Check the put file operation
self.assertTrue(putFileRes["OK"])
self.assertTrue(remoteFile in putFileRes["Value"]["Successful"])
self.assertEqual(putFileRes["Value"]["Successful"][remoteFile],
srcFileSize)
# Check the exists operation
self.assertTrue(existsFileRes["OK"])
self.assertTrue(remoteFile in existsFileRes["Value"]["Successful"])
self.assertTrue(existsFileRes["Value"]["Successful"][remoteFile])
# Check the removal operation
self.assertTrue(removeFileRes["OK"])
self.assertTrue(remoteFile in removeFileRes["Value"]["Successful"])
# Check the failed exists operation
self.assertTrue(failedExistRes["OK"])
self.assertTrue(remoteFile in failedExistRes["Value"]["Successful"])
self.assertFalse(failedExistRes["Value"]["Successful"][remoteFile])
def test_putIsFile(self):
print("\n\n#########################################################"
"################\n\n\t\t\tIs file test\n")
# First upload a file to the storage
srcFile = "/etc/group"
srcFileSize = getSize(srcFile)
testFileName = "testFile.%s" % time.time()
remoteFile = self.storage.getCurrentURL(testFileName)["Value"]
fileDict = {remoteFile: srcFile}
putFileRes = self.storage.putFile(fileDict)
# Check we are able to determine that it is a file
isFileRes = self.storage.isFile(remoteFile)
# Clean up the remote mess
removeFileRes = self.storage.removeFile(remoteFile)
# Check that everything isn't a file
remoteDir = os.path.dirname(remoteFile)
failedIsFileRes = self.storage.isFile(remoteDir)
# Check the put file operation
self.assertTrue(putFileRes["OK"])
self.assertTrue(remoteFile in putFileRes["Value"]["Successful"])
self.assertTrue(putFileRes["Value"]["Successful"][remoteFile])
self.assertEqual(putFileRes["Value"]["Successful"][remoteFile],
srcFileSize)
# Check the is file operation
self.assertTrue(isFileRes["OK"])
self.assertTrue(remoteFile in isFileRes["Value"]["Successful"])
self.assertTrue(isFileRes["Value"]["Successful"][remoteFile])
# check the remove file operation
self.assertTrue(removeFileRes["OK"])
self.assertTrue(remoteFile in removeFileRes["Value"]["Successful"])
# Check that the directory is not a file
self.assertTrue(failedIsFileRes["OK"])
self.assertTrue(remoteDir in failedIsFileRes["Value"]["Successful"])
self.assertFalse(failedIsFileRes["Value"]["Successful"][remoteDir])
def __putFile( self, src_file, dest_url ):
res = pfnparse( src_file )
if not res['OK']:
return res
localCache = False
srcDict = res['Value']
if srcDict['Protocol'] in ['dips', 'dip']:
localCache = True
srcSEURL = srcDict['Protocol'] + '://' + srcDict['Host'] + ':' + srcDict['Port'] + srcDict['WSUrl']
transferClient = TransferClient( srcSEURL )
res = transferClient.receiveFile( srcDict['FileName'], os.path.join( srcDict['Path'], srcDict['FileName'] ) )
if not res['OK']:
return res
src_file = srcDict['FileName']
if not os.path.exists( src_file ):
errStr = "DIPStorage.__putFile: The source local file does not exist."
gLogger.error( errStr, src_file )
return S_ERROR( errStr )
sourceSize = getSize( src_file )
if sourceSize == -1:
errStr = "DIPStorage.__putFile: Failed to get file size."
gLogger.error( errStr, src_file )
return S_ERROR( errStr )
transferClient = TransferClient( self.url )
res = transferClient.sendFile( src_file, dest_url, token = self.checkSum )
if localCache:
os.unlink( src_file )
if res['OK']:
return S_OK( sourceSize )
else:
return res
def test_putGetDirectory( self ):
print('\n\n#########################################################'
'################\n\n\t\t\tGet Directory test\n')
# First clean the remote directory incase something was left there
remoteDir = self.storage.getCurrentURL( 'putDirTest' )['Value']
ignore = self.storage.removeDirectory( remoteDir )
# Create a local directory to upload
localDir = '/tmp/unit-test'
srcFile = '/etc/group'
numberOfFiles = 5
sizeOfLocalFile = getSize( srcFile )
print('Creating local directory: %s' % localDir)
if not os.path.exists( localDir ):
os.mkdir( localDir )
for i in range( numberOfFiles ):
shutil.copy( srcFile, '%s/testFile.%s' % ( localDir, time.time() ) )
time.sleep( 1 )
# Check that we can successfully upload the directory to the storage element
dirTuple = ( localDir, remoteDir )
putDirRes = self.storage.putDirectory( dirTuple )
#Clean up the locally created directory
print('Removing local directory: %s' % localDir)
localFiles = os.listdir( localDir )
for fileName in localFiles:
fullPath = '%s/%s' % ( localDir, fileName )
os.remove( fullPath )
os.removedirs( localDir )
# Check that we can get directories from the storage element
directoryTuple = ( remoteDir, localDir )
getDirRes = self.storage.getDirectory( directoryTuple )
# Now remove the remove directory
removeDirRes = self.storage.removeDirectory( remoteDir )
#Clean up the locally created directory
print('Removing local directory: %s' % localDir)
localFiles = os.listdir( localDir )
for fileName in localFiles:
fullPath = '%s/%s' % ( localDir, fileName )
os.remove( fullPath )
os.removedirs( localDir )
# Perform the checks for the put dir operation
self.assertTrue( putDirRes['OK'] )
self.assertTrue( remoteDir in putDirRes['Value']['Successful'] )
resDict = putDirRes['Value']['Successful'][remoteDir]
self.assertEqual( resDict['Files'], numberOfFiles )
self.assertEqual( resDict['Size'], numberOfFiles * sizeOfLocalFile )
# Perform the checks for the get dir operation
self.assertTrue( getDirRes['OK'] )
self.assertTrue( remoteDir in getDirRes['Value']['Successful'] )
resDict = getDirRes['Value']['Successful'][remoteDir]
self.assertEqual( resDict['Files'], numberOfFiles )
self.assertEqual( resDict['Size'], numberOfFiles * sizeOfLocalFile )
# Perform the checks for the remove directory operation
self.assertTrue( removeDirRes['OK'] )
self.assertTrue( remoteDir in removeDirRes['Value']['Successful'] )
resDict = removeDirRes['Value']['Successful'][remoteDir]
self.assertEqual( resDict['Files'], numberOfFiles )
self.assertEqual( resDict['Size'], numberOfFiles * sizeOfLocalFile )
def test_putRemoveDirectory(self):
print("\n\n#########################################################"
"################\n\n\t\t\tPut Directory test\n")
# First clean the remote directory incase something was left there
remoteDir = self.storage.getCurrentURL("")["Value"]
_ = self.storage.removeDirectory(remoteDir, True)
# Create a local directory to upload
localDir = "/tmp/unit-test"
srcFile = "/etc/group"
sizeOfLocalFile = getSize(srcFile)
if not os.path.exists(localDir):
os.mkdir(localDir)
for i in range(self.numberOfFiles):
shutil.copy(srcFile, "%s/testFile.%s" % (localDir, time.time()))
time.sleep(1)
# Check that we can successfully upload the directory to the storage element
dirDict = {remoteDir: localDir}
putDirRes = self.storage.putDirectory(dirDict)
# Now remove the remove directory
removeDirRes = self.storage.removeDirectory(remoteDir, True)
# Clean up the locally created directory
shutil.rmtree(localDir)
# Perform the checks for the put dir operation
self.assertTrue(putDirRes["OK"])
self.assertTrue(remoteDir in putDirRes["Value"]["Successful"])
if putDirRes["Value"]["Successful"][remoteDir]["Files"]:
self.assertEqual(
putDirRes["Value"]["Successful"][remoteDir]["Files"],
self.numberOfFiles)
self.assertEqual(
putDirRes["Value"]["Successful"][remoteDir]["Size"],
self.numberOfFiles * sizeOfLocalFile)
self.assertTrue(
isinstance(putDirRes["Value"]["Successful"][remoteDir]["Files"],
int))
self.assertTrue(
type(putDirRes["Value"]["Successful"][remoteDir]["Size"]) in
six.integer_types)
# Perform the checks for the remove dir operation
self.assertTrue(removeDirRes["OK"])
self.assertTrue(remoteDir in removeDirRes["Value"]["Successful"])
if removeDirRes["Value"]["Successful"][remoteDir]["FilesRemoved"]:
self.assertEqual(
removeDirRes["Value"]["Successful"][remoteDir]["FilesRemoved"],
self.numberOfFiles)
self.assertEqual(
removeDirRes["Value"]["Successful"][remoteDir]["SizeRemoved"],
self.numberOfFiles * sizeOfLocalFile)
self.assertTrue(
isinstance(
removeDirRes["Value"]["Successful"][remoteDir]["FilesRemoved"],
int))
self.assertTrue(
type(removeDirRes["Value"]["Successful"][remoteDir]["SizeRemoved"])
in six.integer_types)
def test_putGetDirectory( self ):
print '\n\n#########################################################################\n\n\t\t\tGet Directory test\n'
# First clean the remote directory incase something was left there
remoteDir = self.storage.getCurrentURL( 'putDirTest' )['Value']
ignore = self.storage.removeDirectory( remoteDir )
# Create a local directory to upload
localDir = '/tmp/unit-test'
srcFile = '/etc/group'
numberOfFiles = 5
sizeOfLocalFile = getSize( srcFile )
print 'Creating local directory: %s' % localDir
if not os.path.exists( localDir ):
os.mkdir( localDir )
for i in range( numberOfFiles ):
shutil.copy( srcFile, '%s/testFile.%s' % ( localDir, time.time() ) )
time.sleep( 1 )
# Check that we can successfully upload the directory to the storage element
dirTuple = ( localDir, remoteDir )
putDirRes = self.storage.putDirectory( dirTuple )
#Clean up the locally created directory
print 'Removing local directory: %s' % localDir
localFiles = os.listdir( localDir )
for fileName in localFiles:
fullPath = '%s/%s' % ( localDir, fileName )
os.remove( fullPath )
os.removedirs( localDir )
# Check that we can get directories from the storage element
directoryTuple = ( remoteDir, localDir )
getDirRes = self.storage.getDirectory( directoryTuple )
# Now remove the remove directory
removeDirRes = self.storage.removeDirectory( remoteDir )
#Clean up the locally created directory
print 'Removing local directory: %s' % localDir
localFiles = os.listdir( localDir )
for fileName in localFiles:
fullPath = '%s/%s' % ( localDir, fileName )
os.remove( fullPath )
os.removedirs( localDir )
# Perform the checks for the put dir operation
self.assert_( putDirRes['OK'] )
self.assert_( putDirRes['Value']['Successful'].has_key( remoteDir ) )
resDict = putDirRes['Value']['Successful'][remoteDir]
self.assertEqual( resDict['Files'], numberOfFiles )
self.assertEqual( resDict['Size'], numberOfFiles * sizeOfLocalFile )
# Perform the checks for the get dir operation
self.assert_( getDirRes['OK'] )
self.assert_( getDirRes['Value']['Successful'].has_key( remoteDir ) )
resDict = getDirRes['Value']['Successful'][remoteDir]
self.assertEqual( resDict['Files'], numberOfFiles )
self.assertEqual( resDict['Size'], numberOfFiles * sizeOfLocalFile )
# Perform the checks for the remove directory operation
self.assert_( removeDirRes['OK'] )
self.assert_( removeDirRes['Value']['Successful'].has_key( remoteDir ) )
resDict = removeDirRes['Value']['Successful'][remoteDir]
self.assertEqual( resDict['Files'], numberOfFiles )
self.assertEqual( resDict['Size'], numberOfFiles * sizeOfLocalFile )
def setUp( self ): self.numberOfFiles = 1 self.storageElement = StorageElement( storageElementToTest ) self.localSourceFile = "/etc/group" self.localFileSize = getSize( self.localSourceFile ) self.destDirectory = "/lhcb/test/unit-test/TestStorageElement" destinationDir = self.storageElement.getPfnForLfn( self.destDirectory )['Value'] res = self.storageElement.createDirectory( destinationDir, singleDirectory = True ) self.assert_( res['OK'] )
def test_putRemoveDirectory(self):
print('\n\n#########################################################'
'################\n\n\t\t\tPut Directory test\n')
# First clean the remote directory incase something was left there
remoteDir = self.storage.getCurrentURL('')['Value']
ignore = self.storage.removeDirectory(remoteDir, True)
# Create a local directory to upload
localDir = '/tmp/unit-test'
srcFile = '/etc/group'
sizeOfLocalFile = getSize(srcFile)
if not os.path.exists(localDir):
os.mkdir(localDir)
for i in range(self.numberOfFiles):
shutil.copy(srcFile, '%s/testFile.%s' % (localDir, time.time()))
time.sleep(1)
# Check that we can successfully upload the directory to the storage element
dirDict = {remoteDir: localDir}
putDirRes = self.storage.putDirectory(dirDict)
# Now remove the remove directory
removeDirRes = self.storage.removeDirectory(remoteDir, True)
#Clean up the locally created directory
shutil.rmtree(localDir)
# Perform the checks for the put dir operation
self.assertTrue(putDirRes['OK'])
self.assertTrue(putDirRes['Value']['Successful'].has_key(remoteDir))
if putDirRes['Value']['Successful'][remoteDir]['Files']:
self.assertEqual(
putDirRes['Value']['Successful'][remoteDir]['Files'],
self.numberOfFiles)
self.assertEqual(
putDirRes['Value']['Successful'][remoteDir]['Size'],
self.numberOfFiles * sizeOfLocalFile)
self.assertTrue(
type(putDirRes['Value']['Successful'][remoteDir]['Files']) ==
IntType)
self.assertTrue(
type(putDirRes['Value']['Successful'][remoteDir]['Size']) in
[LongType, IntType])
# Perform the checks for the remove dir operation
self.assertTrue(removeDirRes['OK'])
self.assertTrue(removeDirRes['Value']['Successful'].has_key(remoteDir))
if removeDirRes['Value']['Successful'][remoteDir]['FilesRemoved']:
self.assertEqual(
removeDirRes['Value']['Successful'][remoteDir]['FilesRemoved'],
self.numberOfFiles)
self.assertEqual(
removeDirRes['Value']['Successful'][remoteDir]['SizeRemoved'],
self.numberOfFiles * sizeOfLocalFile)
self.assertTrue(
type(removeDirRes['Value']['Successful'][remoteDir]
['FilesRemoved']) == IntType)
self.assertTrue(
type(removeDirRes['Value']['Successful'][remoteDir]['SizeRemoved'])
in [LongType, IntType])
def test_removeDirectory(self):
print('\n\n#########################################################'
'################\n\n\t\t\tRemove directory test\n')
destDirectory = "%s/%s" % (self.destDirectory, 'removeDirectoryTest')
# destDirectory = returnSingleResult( self.storageElement.getURL( directory ) )['Value']
# Create a local directory to upload
localDir = '/tmp/unit-test'
srcFile = '/etc/group'
sizeOfLocalFile = getSize(srcFile)
if not os.path.exists(localDir):
os.mkdir(localDir)
for i in range(self.numberOfFiles):
shutil.copy(srcFile, '%s/testFile.%s' % (localDir, time.time()))
time.sleep(1)
# Check that we can successfully upload the directory to the storage element
dirDict = {destDirectory: localDir}
putDirRes = self.storageElement.putDirectory(dirDict)
# Get the directory metadata
# Now remove the remove directory
removeDirRes = self.storageElement.removeDirectory(destDirectory,
recursive=True)
# Clean up the locally created directory
shutil.rmtree(localDir)
# Perform the checks for the put dir operation
self.assertTrue(putDirRes['OK'])
self.assertTrue(putDirRes['Value'])
if putDirRes['Value']['Successful'][destDirectory]['Files']:
self.assertEqual(
putDirRes['Value']['Successful'][destDirectory]['Files'],
self.numberOfFiles)
self.assertEqual(
putDirRes['Value']['Successful'][destDirectory]['Size'],
self.numberOfFiles * sizeOfLocalFile)
self.assertTrue(
type(putDirRes['Value']['Successful'][destDirectory]['Files'])
in six.integer_types)
self.assertTrue(
type(putDirRes['Value']['Successful'][destDirectory]['Size'])
in six.integer_types)
# Perform the checks for the remove directory operation
self.assertTrue(removeDirRes['OK'])
self.assertTrue(removeDirRes['Value'])
if removeDirRes['Value']['Successful'][destDirectory]['FilesRemoved']:
self.assertEqual(
removeDirRes['Value']['Successful'][destDirectory]
['FilesRemoved'], self.numberOfFiles)
self.assertEqual(
removeDirRes['Value']['Successful'][destDirectory]
['SizeRemoved'], self.numberOfFiles * sizeOfLocalFile)
self.assertTrue(
type(removeDirRes['Value']['Successful'][destDirectory]
['FilesRemoved']) in six.integer_types)
self.assertTrue(
type(removeDirRes['Value']['Successful'][destDirectory]
['SizeRemoved']) in six.integer_types)
def setUp( self ): self.numberOfFiles = 1 self.storageElement = StorageElement( storageElementToTest ) self.localSourceFile = fileToTest self.localFileSize = getSize( self.localSourceFile ) self.destDirectory = lfnDirToTest # destinationDir = returnSingleResult( self.storageElement.getURL( self.destDirectory ) )['Value'] destinationDir = self.destDirectory res = self.storageElement.createDirectory( destinationDir ) self.assert_( res['OK'] )
def setUp(self):
self.numberOfFiles = 1
self.storageElement = StorageElement(storageElementToTest)
self.localSourceFile = fileToTest
self.localFileSize = getSize(self.localSourceFile)
self.destDirectory = lfnDirToTest
# destinationDir = returnSingleResult( self.storageElement.getURL( self.destDirectory ) )['Value']
destinationDir = self.destDirectory
res = self.storageElement.createDirectory(destinationDir)
self.assertTrue(res['OK'])
def setUp(self):
self.numberOfFiles = 1
self.storageElement = StorageElement(storageElementToTest)
self.localSourceFile = "/etc/group"
self.localFileSize = getSize(self.localSourceFile)
self.destDirectory = "/lhcb/test/unit-test/TestStorageElement"
destinationDir = returnSingleResult(
self.storageElement.getPfnForLfn(self.destDirectory))
res = self.storageElement.createDirectory(destinationDir,
singleDirectory=True)
self.assert_(res['OK'])
def test_putGetDirectory( self ):
print '\n\n#########################################################################\n\n\t\t\tGet Directory test\n'
# First clean the remote directory incase something was left there
remoteDir = self.storage.getCurrentURL( '' )['Value']
ignore = self.storage.removeDirectory( remoteDir, True )
# Create a local directory to upload
localDir = '/tmp/unit-test'
srcFile = '/etc/group'
sizeOfLocalFile = getSize( srcFile )
if not os.path.exists( localDir ):
os.mkdir( localDir )
for i in range( self.numberOfFiles ):
shutil.copy( srcFile, '%s/testFile.%s' % ( localDir, time.time() ) )
time.sleep( 1 )
# Check that we can successfully upload the directory to the storage element
dirDict = {remoteDir:localDir}
putDirRes = self.storage.putDirectory( dirDict )
#Clean up the locally created directory
shutil.rmtree( localDir )
# Check that we can get directories from the storage element
getDirRes = self.storage.getDirectory( remoteDir, localPath = localDir )
# Now remove the remove directory
removeDirRes = self.storage.removeDirectory( remoteDir, True )
#Clean up the locally created directory
shutil.rmtree( localDir )
# Perform the checks for the put dir operation
self.assert_( putDirRes['OK'] )
self.assert_( putDirRes['Value']['Successful'].has_key( remoteDir ) )
if putDirRes['Value']['Successful'][remoteDir]['Files']:
self.assertEqual( putDirRes['Value']['Successful'][remoteDir]['Files'], self.numberOfFiles )
self.assertEqual( putDirRes['Value']['Successful'][remoteDir]['Size'], self.numberOfFiles * sizeOfLocalFile )
self.assert_( type( putDirRes['Value']['Successful'][remoteDir]['Files'] ) == IntType )
self.assert_( type( putDirRes['Value']['Successful'][remoteDir]['Size'] ) in [LongType, IntType] )
# Perform the checks for the get dir operation
self.assert_( getDirRes['OK'] )
self.assert_( getDirRes['Value']['Successful'].has_key( remoteDir ) )
resDict = getDirRes['Value']['Successful'][remoteDir]
if resDict['Files']:
self.assertEqual( resDict['Files'], self.numberOfFiles )
self.assertEqual( resDict['Size'], self.numberOfFiles * sizeOfLocalFile )
self.assert_( type( resDict['Files'] ) == IntType )
self.assert_( type( resDict['Size'] ) in [LongType, IntType] )
# Perform the checks for the remove directory operation
self.assert_( removeDirRes['OK'] )
self.assert_( removeDirRes['Value']['Successful'].has_key( remoteDir ) )
if removeDirRes['Value']['Successful'][remoteDir]['FilesRemoved']:
self.assertEqual( removeDirRes['Value']['Successful'][remoteDir]['FilesRemoved'], self.numberOfFiles )
self.assertEqual( removeDirRes['Value']['Successful'][remoteDir]['SizeRemoved'], self.numberOfFiles * sizeOfLocalFile )
self.assert_( type( removeDirRes['Value']['Successful'][remoteDir]['FilesRemoved'] ) == IntType )
self.assert_( type( removeDirRes['Value']['Successful'][remoteDir]['SizeRemoved'] ) in [LongType, IntType] )
def test_getDirectory( self ):
print '\n\n#########################################################################\n\n\t\t\tGet directory test\n'
destDirectory = "%s/%s" % ( self.destDirectory, 'getDirectoryTest' )
# destDirectory = returnSingleResult( self.storageElement.getURL( directory ) )['Value']
# Create a local directory to upload
localDir = '/tmp/unit-test'
srcFile = '/etc/group'
sizeOfLocalFile = getSize( srcFile )
if not os.path.exists( localDir ):
os.mkdir( localDir )
for i in range( self.numberOfFiles ):
shutil.copy( srcFile, '%s/testFile.%s' % ( localDir, time.time() ) )
time.sleep( 1 )
# Check that we can successfully upload the directory to the storage element
dirDict = {destDirectory:localDir}
putDirRes = self.storageElement.putDirectory( dirDict )
# Get the directory metadata
#Clean up the locally created directory
shutil.rmtree( localDir )
getDirRes = self.storageElement.getDirectory( destDirectory, localPath = localDir )
# Now remove the remove directory
removeDirRes = self.storageElement.removeDirectory( destDirectory, recursive = True )
#Clean up the locally created directory
if os.path.exists( localDir ):
shutil.rmtree( localDir )
# Perform the checks for the put dir operation
self.assert_( putDirRes['OK'] )
self.assert_( putDirRes['Value'] )
for _dir in dirDict:
if putDirRes['Value']['Successful'][_dir]['Files']:
self.assertEqual( putDirRes['Value']['Successful'][_dir]['Files'], self.numberOfFiles )
self.assertEqual( putDirRes['Value']['Successful'][_dir]['Size'], self.numberOfFiles * sizeOfLocalFile )
self.assert_( type( putDirRes['Value']['Successful'][_dir]['Files'] ) in [types.IntType, types.LongType] )
self.assert_( type( putDirRes['Value']['Successful'][_dir]['Size'] ) in [types.IntType, types.LongType] )
# Perform the checks for the get directory operation
self.assert_( getDirRes['OK'] )
self.assert_( getDirRes['Value'] )
for _dir in dirDict:
if getDirRes['Value']['Successful'][_dir]['Files']:
self.assertEqual( getDirRes['Value']['Successful'][_dir]['Files'], self.numberOfFiles )
self.assertEqual( getDirRes['Value']['Successful'][_dir]['Size'], self.numberOfFiles * sizeOfLocalFile )
self.assert_( type( getDirRes['Value']['Successful'][_dir]['Files'] ) in [types.IntType, types.LongType] )
self.assert_( type( getDirRes['Value']['Successful'][_dir]['Size'] ) in [types.IntType, types.LongType] )
# Perform the checks for the remove directory operation
self.assert_( removeDirRes['OK'] )
self.assert_( removeDirRes['Value'] )
if removeDirRes['Value']['Successful'][destDirectory]['FilesRemoved']:
self.assertEqual( removeDirRes['Value']['Successful'][destDirectory]['FilesRemoved'], self.numberOfFiles )
self.assertEqual( removeDirRes['Value']['Successful'][destDirectory]['SizeRemoved'], self.numberOfFiles * sizeOfLocalFile )
self.assert_( type( removeDirRes['Value']['Successful'][destDirectory]['FilesRemoved'] ) in [types.IntType, types.LongType] )
self.assert_( type( removeDirRes['Value']['Successful'][destDirectory]['SizeRemoved'] ) in [types.IntType, types.LongType] )
def test_putListDirectory(self):
print("\n\n#########################################################"
"################\n\n\t\t\tList Directory test\n")
# First clean the remote directory incase something was left there
remoteDir = self.storage.getCurrentURL("putDirTest")["Value"]
ignore = self.storage.removeDirectory(remoteDir)
# Create a local directory to upload
localDir = "/tmp/unit-test"
srcFile = "/etc/group"
numberOfFiles = 5
sizeOfLocalFile = getSize(srcFile)
print("Creating local directory: %s" % localDir)
if not os.path.exists(localDir):
os.mkdir(localDir)
for i in range(numberOfFiles):
shutil.copy(srcFile, "%s/testFile.%s" % (localDir, time.time()))
time.sleep(1)
# Check that we can successfully upload the directory to the storage element
dirTuple = (localDir, remoteDir)
putDirRes = self.storage.putDirectory(dirTuple)
# List the remote directory
listDirRes = self.storage.listDirectory(remoteDir)
# Now remove the remove directory
removeDirRes = self.storage.removeDirectory(remoteDir)
# Clean up the locally created directory
print("Removing local directory: %s" % localDir)
localFiles = os.listdir(localDir)
for fileName in localFiles:
fullPath = "%s/%s" % (localDir, fileName)
os.remove(fullPath)
os.removedirs(localDir)
# Perform the checks for the put dir operation
self.assertTrue(putDirRes["OK"])
self.assertTrue(remoteDir in putDirRes["Value"]["Successful"])
resDict = putDirRes["Value"]["Successful"][remoteDir]
self.assertEqual(resDict["Files"], numberOfFiles)
self.assertEqual(resDict["Size"], numberOfFiles * sizeOfLocalFile)
# Perform the checks for the list dir operation
self.assertTrue(listDirRes["OK"])
self.assertTrue(remoteDir in listDirRes["Value"]["Successful"])
resDict = listDirRes["Value"]["Successful"][remoteDir]
self.assertTrue("SubDirs" in resDict)
self.assertTrue("Files" in resDict)
self.assertEqual(len(resDict["Files"]), numberOfFiles)
# Perform the checks for the remove directory operation
self.assertTrue(removeDirRes["OK"])
self.assertTrue(remoteDir in removeDirRes["Value"]["Successful"])
resDict = removeDirRes["Value"]["Successful"][remoteDir]
self.assertEqual(resDict["Files"], numberOfFiles)
self.assertEqual(resDict["Size"], numberOfFiles * sizeOfLocalFile)
def test_putGetFileMetaData(self):
print('\n\n#########################################################'
'################\n\n\t\t\tGet file metadata test\n')
# First upload a file to the storage
srcFile = '/etc/group'
srcFileSize = getSize(srcFile)
testFileName = 'testFile.%s' % time.time()
remoteFile = self.storage.getCurrentURL(testFileName)['Value']
fileDict = {remoteFile: srcFile}
putFileRes = self.storage.putFile(fileDict)
# Check that we can get the file metadata
getMetadataRes = self.storage.getFileMetadata(remoteFile)
# Clean up the remote mess
removeFileRes = self.storage.removeFile(remoteFile)
# See what happens with non existant files
failedMetadataRes = self.storage.getFileMetadata(remoteFile)
# Check directories are handled properly
remoteDir = os.path.dirname(remoteFile)
directoryMetadataRes = self.storage.getFileMetadata(remoteDir)
# Check the put file operation
self.assertTrue(putFileRes['OK'])
self.assertTrue(putFileRes['Value']['Successful'].has_key(remoteFile))
self.assertTrue(putFileRes['Value']['Successful'][remoteFile])
self.assertEqual(putFileRes['Value']['Successful'][remoteFile],
srcFileSize)
# Check the get metadata operation
self.assertTrue(getMetadataRes['OK'])
self.assertTrue(
getMetadataRes['Value']['Successful'].has_key(remoteFile))
fileMetaData = getMetadataRes['Value']['Successful'][remoteFile]
self.assertTrue(fileMetaData['Cached'])
self.assertFalse(fileMetaData['Migrated'])
self.assertEqual(fileMetaData['Size'], srcFileSize)
# check the remove file operation
self.assertTrue(removeFileRes['OK'])
self.assertTrue(
removeFileRes['Value']['Successful'].has_key(remoteFile))
# Check the get metadata for non existant file
self.assertTrue(failedMetadataRes['OK'])
self.assertTrue(
failedMetadataRes['Value']['Failed'].has_key(remoteFile))
expectedError = "File does not exist"
self.assertTrue(
expectedError in failedMetadataRes['Value']['Failed'][remoteFile])
# Check that metadata operation with a directory
self.assertTrue(directoryMetadataRes['OK'])
self.assertTrue(
directoryMetadataRes['Value']['Failed'].has_key(remoteDir))
expectedError = "Supplied path is not a file"
self.assertTrue(expectedError in directoryMetadataRes['Value']
['Failed'][remoteDir])
def test_putListDirectory(self):
print "\n\n#########################################################################\n\n\t\t\tList Directory test\n"
# First clean the remote directory incase something was left there
remoteDir = self.storage.getCurrentURL("putDirTest")["Value"]
ignore = self.storage.removeDirectory(remoteDir)
# Create a local directory to upload
localDir = "/tmp/unit-test"
srcFile = "/etc/group"
numberOfFiles = 5
sizeOfLocalFile = getSize(srcFile)
print "Creating local directory: %s" % localDir
if not os.path.exists(localDir):
os.mkdir(localDir)
for i in range(numberOfFiles):
shutil.copy(srcFile, "%s/testFile.%s" % (localDir, time.time()))
time.sleep(1)
# Check that we can successfully upload the directory to the storage element
dirTuple = (localDir, remoteDir)
putDirRes = self.storage.putDirectory(dirTuple)
# List the remote directory
listDirRes = self.storage.listDirectory(remoteDir)
# Now remove the remove directory
removeDirRes = self.storage.removeDirectory(remoteDir)
# Clean up the locally created directory
print "Removing local directory: %s" % localDir
localFiles = os.listdir(localDir)
for fileName in localFiles:
fullPath = "%s/%s" % (localDir, fileName)
os.remove(fullPath)
os.removedirs(localDir)
# Perform the checks for the put dir operation
self.assert_(putDirRes["OK"])
self.assert_(putDirRes["Value"]["Successful"].has_key(remoteDir))
resDict = putDirRes["Value"]["Successful"][remoteDir]
self.assertEqual(resDict["Files"], numberOfFiles)
self.assertEqual(resDict["Size"], numberOfFiles * sizeOfLocalFile)
# Perform the checks for the list dir operation
self.assert_(listDirRes["OK"])
self.assert_(listDirRes["Value"]["Successful"].has_key(remoteDir))
resDict = listDirRes["Value"]["Successful"][remoteDir]
self.assert_(resDict.has_key("SubDirs"))
self.assert_(resDict.has_key("Files"))
self.assertEqual(len(resDict["Files"].keys()), numberOfFiles)
# Perform the checks for the remove directory operation
self.assert_(removeDirRes["OK"])
self.assert_(removeDirRes["Value"]["Successful"].has_key(remoteDir))
resDict = removeDirRes["Value"]["Successful"][remoteDir]
self.assertEqual(resDict["Files"], numberOfFiles)
self.assertEqual(resDict["Size"], numberOfFiles * sizeOfLocalFile)
def test_getDirectoryMetadata( self ):
print '\n\n#########################################################################\n\n\t\t\tDirectory metadata test\n'
directory = "%s/%s" % ( self.destDirectory, 'getDirectoryMetadataTest' )
destDirectory = returnSingleResult( self.storageElement.getPfnForLfn( directory ) )['Value']
# Create a local directory to upload
localDir = '/tmp/unit-test'
srcFile = '/etc/group'
sizeOfLocalFile = getSize( srcFile )
if not os.path.exists( localDir ):
os.mkdir( localDir )
for i in range( self.numberOfFiles ):
shutil.copy( srcFile, '%s/testFile.%s' % ( localDir, time.time() ) )
time.sleep( 1 )
# Check that we can successfully upload the directory to the storage element
dirDict = {destDirectory:localDir}
putDirRes = self.storageElement.putDirectory( dirDict, singleDirectory = True )
# Get the directory metadata
metadataDirRes = self.storageElement.getDirectoryMetadata( destDirectory, singleDirectory = True )
# Now remove the remove directory
removeDirRes = self.storageElement.removeDirectory( destDirectory, recursive = True, singleDirectory = True )
#Clean up the locally created directory
shutil.rmtree( localDir )
# Perform the checks for the put dir operation
self.assert_( putDirRes['OK'] )
self.assert_( putDirRes['Value'] )
if putDirRes['Value']['Files']:
self.assertEqual( putDirRes['Value']['Files'], self.numberOfFiles )
self.assertEqual( putDirRes['Value']['Size'], self.numberOfFiles * sizeOfLocalFile )
self.assert_( type( putDirRes['Value']['Files'] ) in [types.IntType, types.LongType] )
self.assert_( type( putDirRes['Value']['Size'] ) in [types.IntType, types.LongType] )
# Perform the checks for the list dir operation
self.assert_( metadataDirRes['OK'] )
self.assert_( metadataDirRes['Value'] )
# Works only for the SRM2 plugin
# self.assert_( metadataDirRes['Value']['Mode'] )
# self.assert_( type( metadataDirRes['Value']['Mode'] ) == types.IntType )
self.assert_( metadataDirRes['Value']['Directory'] )
self.assertFalse( metadataDirRes['Value']['File'] )
# Perform the checks for the remove directory operation
self.assert_( removeDirRes['OK'] )
self.assert_( removeDirRes['Value'] )
if removeDirRes['Value']['FilesRemoved']:
self.assertEqual( removeDirRes['Value']['FilesRemoved'], self.numberOfFiles )
self.assertEqual( removeDirRes['Value']['SizeRemoved'], self.numberOfFiles * sizeOfLocalFile )
self.assert_( type( removeDirRes['Value']['FilesRemoved'] ) in [types.IntType, types.LongType] )
self.assert_( type( removeDirRes['Value']['SizeRemoved'] ) in [types.IntType, types.LongType] )
def test_putGetDirectoryMetadata(self):
print('\n\n#########################################################'
'################\n\n\t\t\tGet Directory Metadata test\n')
# First clean the remote directory incase something was left there
remoteDir = self.storage.getCurrentURL('')['Value']
_ = self.storage.removeDirectory(remoteDir, True)
# Create a local directory to upload
localDir = '/tmp/unit-test'
srcFile = '/etc/group'
sizeOfLocalFile = getSize(srcFile)
if not os.path.exists(localDir):
os.mkdir(localDir)
for i in range(self.numberOfFiles):
shutil.copy(srcFile, '%s/testFile.%s' % (localDir, time.time()))
time.sleep(1)
# Check that we can successfully upload the directory to the storage element
dirDict = {remoteDir: localDir}
putDirRes = self.storage.putDirectory(dirDict)
# Get the directory metadata
getMetadataRes = self.storage.getDirectoryMetadata(remoteDir)
# Now remove the remove directory
removeDirRes = self.storage.removeDirectory(remoteDir, True)
# Clean up the locally created directory
shutil.rmtree(localDir)
# Perform the checks for the put dir operation
self.assertTrue(putDirRes['OK'])
self.assertTrue(remoteDir in putDirRes['Value']['Successful'])
if putDirRes['Value']['Successful'][remoteDir]['Files']:
self.assertEqual(putDirRes['Value']['Successful'][remoteDir]['Files'], self.numberOfFiles)
self.assertEqual(putDirRes['Value']['Successful'][remoteDir]['Size'], self.numberOfFiles * sizeOfLocalFile)
self.assertTrue(isinstance(putDirRes['Value']['Successful'][remoteDir]['Files'], int))
self.assertTrue(type(putDirRes['Value']['Successful'][remoteDir]['Size']) in six.integer_types)
# Perform the checks for the get metadata operation
self.assertTrue(getMetadataRes['OK'])
self.assertTrue(remoteDir in getMetadataRes['Value']['Successful'])
resDict = getMetadataRes['Value']['Successful'][remoteDir]
self.assertTrue('Mode' in resDict)
self.assertTrue(isinstance(resDict['Mode'], int))
# Perform the checks for the remove directory operation
if removeDirRes['Value']['Successful'][remoteDir]['FilesRemoved']:
self.assertEqual(removeDirRes['Value']['Successful'][remoteDir]['FilesRemoved'], self.numberOfFiles)
self.assertEqual(removeDirRes['Value']['Successful'][remoteDir]
['SizeRemoved'], self.numberOfFiles * sizeOfLocalFile)
self.assertTrue(isinstance(removeDirRes['Value']['Successful'][remoteDir]['FilesRemoved'], int))
self.assertTrue(type(removeDirRes['Value']['Successful'][remoteDir]['SizeRemoved']) in six.integer_types)
def __getFile( self, src_url, dest_file ):
transferClient = TransferClient( self.url )
res = transferClient.receiveFile( dest_file, src_url, token = self.checkSum )
if not res['OK']:
return res
if not os.path.exists( dest_file ):
errStr = "DIPStorage.__getFile: The destination local file does not exist."
gLogger.error( errStr, dest_file )
return S_ERROR( errStr )
destSize = getSize( dest_file )
if destSize == -1:
errStr = "DIPStorage.__getFile: Failed to get the local file size."
gLogger.error( errStr, dest_file )
return S_ERROR( errStr )
return S_OK( destSize )
def __getFile( self, src_url, dest_file ):
transferClient = TransferClient( self.url )
res = transferClient.receiveFile( dest_file, src_url, token = self.checkSum )
if not res['OK']:
return res
if not os.path.exists( dest_file ):
errStr = "DIPStorage.__getFile: The destination local file does not exist."
gLogger.error( errStr, dest_file )
return S_ERROR( errStr )
destSize = getSize( dest_file )
if destSize == -1:
errStr = "DIPStorage.__getFile: Failed to get the local file size."
gLogger.error( errStr, dest_file )
return S_ERROR( errStr )
return S_OK( destSize )
def test_putGetFileMetaData(self):
print("\n\n#########################################################"
"################\n\n\t\t\tGet file metadata test\n")
# First upload a file to the storage
srcFile = "/etc/group"
srcFileSize = getSize(srcFile)
testFileName = "testFile.%s" % time.time()
remoteFile = self.storage.getCurrentURL(testFileName)["Value"]
fileDict = {remoteFile: srcFile}
putFileRes = self.storage.putFile(fileDict)
# Check that we can get the file metadata
getMetadataRes = self.storage.getFileMetadata(remoteFile)
# Clean up the remote mess
removeFileRes = self.storage.removeFile(remoteFile)
# See what happens with non existant files
failedMetadataRes = self.storage.getFileMetadata(remoteFile)
# Check directories are handled properly
remoteDir = os.path.dirname(remoteFile)
directoryMetadataRes = self.storage.getFileMetadata(remoteDir)
# Check the put file operation
self.assertTrue(putFileRes["OK"])
self.assertTrue(remoteFile in putFileRes["Value"]["Successful"])
self.assertTrue(putFileRes["Value"]["Successful"][remoteFile])
self.assertEqual(putFileRes["Value"]["Successful"][remoteFile],
srcFileSize)
# Check the get metadata operation
self.assertTrue(getMetadataRes["OK"])
self.assertTrue(remoteFile in getMetadataRes["Value"]["Successful"])
fileMetaData = getMetadataRes["Value"]["Successful"][remoteFile]
self.assertTrue(fileMetaData["Cached"])
self.assertFalse(fileMetaData["Migrated"])
self.assertEqual(fileMetaData["Size"], srcFileSize)
# check the remove file operation
self.assertTrue(removeFileRes["OK"])
self.assertTrue(remoteFile in removeFileRes["Value"]["Successful"])
# Check the get metadata for non existant file
self.assertTrue(failedMetadataRes["OK"])
self.assertTrue(remoteFile in failedMetadataRes["Value"]["Failed"])
expectedError = "File does not exist"
self.assertTrue(
expectedError in failedMetadataRes["Value"]["Failed"][remoteFile])
# Check that metadata operation with a directory
self.assertTrue(directoryMetadataRes["OK"])
self.assertTrue(remoteDir in directoryMetadataRes["Value"]["Failed"])
expectedError = "Supplied path is not a file"
self.assertTrue(expectedError in directoryMetadataRes["Value"]
["Failed"][remoteDir])
def test_putPrestageStatus(self):
print('\n\n#########################################################'
'################\n\n\t\t\tPrestage status test\n')
# First upload a file to the storage
srcFile = '/etc/group'
srcFileSize = getSize(srcFile)
testFileName = 'testFile.%s' % time.time()
remoteFile = self.storage.getCurrentURL(testFileName)['Value']
fileDict = {remoteFile: srcFile}
putFileRes = self.storage.putFile(fileDict)
# Check that we can issue a stage request
prestageRes = self.storage.prestageFile(remoteFile)
srmID = ''
if prestageRes['OK']:
if prestageRes['Value']['Successful'].has_key(remoteFile):
srmID = prestageRes['Value']['Successful'][remoteFile]
# Take a quick break to allow the SRM to realise the file is available
sleepTime = 10
print('Sleeping for %s seconds' % sleepTime)
time.sleep(sleepTime)
# Check that we can monitor the stage request
prestageStatusRes = self.storage.prestageFileStatus(
{remoteFile: srmID})
# Clean up the remote mess
removeFileRes = self.storage.removeFile(remoteFile)
# Check the put file operation
self.assertTrue(putFileRes['OK'])
self.assertTrue(putFileRes['Value']['Successful'].has_key(remoteFile))
self.assertTrue(putFileRes['Value']['Successful'][remoteFile])
self.assertEqual(putFileRes['Value']['Successful'][remoteFile],
srcFileSize)
# Check the prestage file operation
self.assertTrue(prestageRes['OK'])
self.assertTrue(prestageRes['Value']['Successful'].has_key(remoteFile))
self.assertTrue(prestageRes['Value']['Successful'][remoteFile])
self.assertTrue(
type(prestageRes['Value']['Successful'][remoteFile]) in
StringTypes)
# Check the prestage status operation
self.assertTrue(prestageStatusRes['OK'])
self.assertTrue(
prestageStatusRes['Value']['Successful'].has_key(remoteFile))
self.assertTrue(prestageStatusRes['Value']['Successful'][remoteFile])
# Check the remove file operation
self.assertTrue(removeFileRes['OK'])
self.assertTrue(
removeFileRes['Value']['Successful'].has_key(remoteFile))
def test_listDirectory( self ):
print '\n\n#########################################################################\n\n\t\t\tList directory test\n'
directory = "%s/%s" % ( self.destDirectory, 'listDirectoryTest' )
destDirectory = self.storageElement.getPfnForLfn( directory )['Value']
# Create a local directory to upload
localDir = '/tmp/unit-test'
srcFile = '/etc/group'
sizeOfLocalFile = getSize( srcFile )
if not os.path.exists( localDir ):
os.mkdir( localDir )
for i in range( self.numberOfFiles ):
shutil.copy( srcFile, '%s/testFile.%s' % ( localDir, time.time() ) )
time.sleep( 1 )
# Check that we can successfully upload the directory to the storage element
dirDict = {destDirectory:localDir}
putDirRes = self.storageElement.putDirectory( dirDict, singleDirectory = True )
print putDirRes
# List the remote directory
listDirRes = self.storageElement.listDirectory( destDirectory, singleDirectory = True )
# Now remove the remove directory
removeDirRes = self.storageElement.removeDirectory( destDirectory, recursive = True, singleDirectory = True )
print removeDirRes
#Clean up the locally created directory
shutil.rmtree( localDir )
# Perform the checks for the put dir operation
self.assert_( putDirRes['OK'] )
self.assert_( putDirRes['Value'] )
if putDirRes['Value']['Files']:
self.assertEqual( putDirRes['Value']['Files'], self.numberOfFiles )
self.assertEqual( putDirRes['Value']['Size'], self.numberOfFiles * sizeOfLocalFile )
self.assert_( type( putDirRes['Value']['Files'] ) in [types.IntType, types.LongType] )
self.assert_( type( putDirRes['Value']['Size'] ) in [types.IntType, types.LongType] )
# Perform the checks for the list dir operation
self.assert_( listDirRes['OK'] )
self.assert_( listDirRes['Value'] )
self.assert_( listDirRes.has_key( 'SubDirs' ) )
self.assert_( listDirRes.has_key( 'Files' ) )
self.assertEqual( len( listDirRes['Files'].keys() ), self.numberOfFiles )
# Perform the checks for the remove directory operation
self.assert_( removeDirRes['OK'] )
self.assert_( removeDirRes['Value'] )
if removeDirRes['Value']['FilesRemoved']:
self.assertEqual( removeDirRes['Value']['FilesRemoved'], self.numberOfFiles )
self.assertEqual( removeDirRes['Value']['SizeRemoved'], self.numberOfFiles * sizeOfLocalFile )
self.assert_( type( removeDirRes['Value']['FilesRemoved'] ) in [types.IntType, types.LongType] )
self.assert_( type( removeDirRes['Value']['SizeRemoved'] ) in [types.IntType, types.LongType] )
def test_putGetFileMetaData( self ):
print '\n\n#########################################################################\n\n\t\t\tGet file metadata test\n'
# First upload a file to the storage
srcFile = '/etc/group'
srcFileSize = getSize( srcFile )
testFileName = 'testFile.%s' % time.time()
remoteFile = self.storage.getCurrentURL( testFileName )['Value']
fileDict = {remoteFile:srcFile}
putFileRes = self.storage.putFile( fileDict )
# Check that we can get the file metadata
getMetadataRes = self.storage.getFileMetadata( remoteFile )
# Clean up the remote mess
removeFileRes = self.storage.removeFile( remoteFile )
# See what happens with non existant files
failedMetadataRes = self.storage.getFileMetadata( remoteFile )
# Check directories are handled properly
remoteDir = os.path.dirname( remoteFile )
directoryMetadataRes = self.storage.getFileMetadata( remoteDir )
# Check the put file operation
self.assert_( putFileRes['OK'] )
self.assert_( putFileRes['Value']['Successful'].has_key( remoteFile ) )
self.assert_( putFileRes['Value']['Successful'][remoteFile] )
self.assertEqual( putFileRes['Value']['Successful'][remoteFile], srcFileSize )
# Check the get metadata operation
self.assert_( getMetadataRes['OK'] )
self.assert_( getMetadataRes['Value']['Successful'].has_key( remoteFile ) )
fileMetaData = getMetadataRes['Value']['Successful'][remoteFile]
self.assert_( fileMetaData['Cached'] )
self.assertFalse( fileMetaData['Migrated'] )
self.assertEqual( fileMetaData['Size'], srcFileSize )
# check the remove file operation
self.assert_( removeFileRes['OK'] )
self.assert_( removeFileRes['Value']['Successful'].has_key( remoteFile ) )
# Check the get metadata for non existant file
self.assert_( failedMetadataRes['OK'] )
self.assert_( failedMetadataRes['Value']['Failed'].has_key( remoteFile ) )
expectedError = "File does not exist"
self.assert_( expectedError in failedMetadataRes['Value']['Failed'][remoteFile] )
# Check that metadata operation with a directory
self.assert_( directoryMetadataRes['OK'] )
self.assert_( directoryMetadataRes['Value']['Failed'].has_key( remoteDir ) )
expectedError = "Supplied path is not a file"
self.assert_( expectedError in directoryMetadataRes['Value']['Failed'][remoteDir] )
def test_putGetFileMetaData(self):
print "\n\n#########################################################################\n\n\t\t\tGet file metadata test\n"
# First upload a file to the storage
srcFile = "/etc/group"
fileSize = getSize(srcFile)
testFileName = "testFile.%s" % time.time()
destFile = self.storage.getCurrentURL(testFileName)["Value"]
fileTuple = (srcFile, destFile, fileSize)
putFileRes = self.storage.putFile(fileTuple)
# Check that we can get the file metadata
getMetadataRes = self.storage.getFileMetadata(destFile)
# Clean up the remote mess
removeFileRes = self.storage.removeFile(destFile)
# See what happens with non existant files
failedMetadataRes = self.storage.getFileMetadata(destFile)
# Check directories are handled properly
destDir = os.path.dirname(destFile)
directoryMetadataRes = self.storage.getFileMetadata(destDir)
# Check the put file operation
self.assert_(putFileRes["OK"])
self.assert_(putFileRes["Value"]["Successful"].has_key(destFile))
# Check the get metadata operation
self.assert_(getMetadataRes["OK"])
self.assert_(getMetadataRes["Value"]["Successful"].has_key(destFile))
fileMetaData = getMetadataRes["Value"]["Successful"][destFile]
# self.assert_(fileMetaData['Cached'])
# self.assertFalse(fileMetaData['Migrated'])
self.assertEqual(fileMetaData["Size"], fileSize)
# check the remove file operation
self.assert_(removeFileRes["OK"])
self.assert_(removeFileRes["Value"]["Successful"].has_key(destFile))
# Check the get metadata for non existant file
self.assert_(failedMetadataRes["OK"])
self.assert_(failedMetadataRes["Value"]["Failed"].has_key(destFile))
expectedError = "No such file or directory"
self.assertEqual(failedMetadataRes["Value"]["Failed"][destFile], expectedError)
# Check that metadata operation with a directory
self.assert_(directoryMetadataRes["OK"])
self.assert_(directoryMetadataRes["Value"]["Failed"].has_key(destDir))
expectedError = "RFIOStorage.getFileMetadata: Supplied path is not a file."
self.assertEqual(directoryMetadataRes["Value"]["Failed"][destDir], expectedError)
def test_putGetFileSize( self ):
print '\n\n#########################################################################\n\n\t\t\tGet file size test\n'
# First upload a file to the storage
srcFile = '/etc/group'
srcFileSize = getSize( srcFile )
testFileName = 'testFile.%s' % time.time()
remoteFile = self.storage.getCurrentURL( testFileName )['Value']
fileDict = {remoteFile:srcFile}
putFileRes = self.storage.putFile( fileDict )
# Check that we can get the file size
getSizeRes = self.storage.getFileSize( remoteFile )
# Clean up the remote mess
removeFileRes = self.storage.removeFile( remoteFile )
# Check non existant files
failedSizeRes = self.storage.getFileMetadata( remoteFile )
# Check directories are handled properly
remoteDir = os.path.dirname( remoteFile )
directorySizeRes = self.storage.getFileSize( remoteDir )
# Check the put file operation
self.assert_( putFileRes['OK'] )
self.assert_( putFileRes['Value']['Successful'].has_key( remoteFile ) )
self.assert_( putFileRes['Value']['Successful'][remoteFile] )
self.assertEqual( putFileRes['Value']['Successful'][remoteFile], srcFileSize )
# Check that we got the file size correctly
self.assert_( getSizeRes['OK'] )
self.assert_( getSizeRes['Value']['Successful'].has_key( remoteFile ) )
self.assertEqual( getSizeRes['Value']['Successful'][remoteFile], srcFileSize )
# check the remove file operation
self.assert_( removeFileRes['OK'] )
self.assert_( removeFileRes['Value']['Successful'].has_key( remoteFile ) )
# Check the get size with non existant file works properly
self.assert_( failedSizeRes['OK'] )
self.assert_( failedSizeRes['Value']['Failed'].has_key( remoteFile ) )
expectedError = "File does not exist"
self.assert_( expectedError in failedSizeRes['Value']['Failed'][remoteFile] )
# Check that the passing a directory is handled correctly
self.assert_( directorySizeRes['OK'] )
self.assert_( directorySizeRes['Value']['Failed'].has_key( remoteDir ) )
expectedError = "Supplied path is not a file"
self.assert_( expectedError in directorySizeRes['Value']['Failed'][remoteDir] )
def test_putPrestageStatus( self ):
print '\n\n#########################################################################\n\n\t\t\tPrestage status test\n'
# First upload a file to the storage
srcFile = '/etc/group'
srcFileSize = getSize( srcFile )
testFileName = 'testFile.%s' % time.time()
remoteFile = self.storage.getCurrentURL( testFileName )['Value']
fileDict = {remoteFile:srcFile}
putFileRes = self.storage.putFile( fileDict )
# Check that we can issue a stage request
prestageRes = self.storage.prestageFile( remoteFile )
srmID = ''
if prestageRes['OK']:
if prestageRes['Value']['Successful'].has_key( remoteFile ):
srmID = prestageRes['Value']['Successful'][remoteFile]
# Take a quick break to allow the SRM to realise the file is available
sleepTime = 10
print 'Sleeping for %s seconds' % sleepTime
time.sleep( sleepTime )
# Check that we can monitor the stage request
prestageStatusRes = self.storage.prestageFileStatus( {remoteFile:srmID} )
# Clean up the remote mess
removeFileRes = self.storage.removeFile( remoteFile )
# Check the put file operation
self.assert_( putFileRes['OK'] )
self.assert_( putFileRes['Value']['Successful'].has_key( remoteFile ) )
self.assert_( putFileRes['Value']['Successful'][remoteFile] )
self.assertEqual( putFileRes['Value']['Successful'][remoteFile], srcFileSize )
# Check the prestage file operation
self.assert_( prestageRes['OK'] )
self.assert_( prestageRes['Value']['Successful'].has_key( remoteFile ) )
self.assert_( prestageRes['Value']['Successful'][remoteFile] )
self.assert_( type( prestageRes['Value']['Successful'][remoteFile] ) in StringTypes )
# Check the prestage status operation
self.assert_( prestageStatusRes['OK'] )
self.assert_( prestageStatusRes['Value']['Successful'].has_key( remoteFile ) )
self.assert_( prestageStatusRes['Value']['Successful'][remoteFile] )
# Check the remove file operation
self.assert_( removeFileRes['OK'] )
self.assert_( removeFileRes['Value']['Successful'].has_key( remoteFile ) )
def __putFile(self, src_file, dest_url):
res = pfnparse(src_file)
if not res['OK']:
return res
localCache = False
srcDict = res['Value']
if srcDict['Protocol'] in ['dips', 'dip']:
# Make the service URL from the file URL by stripping off the file part
serviceDict = dict(srcDict)
serviceDict['Path'] = '/'.join(srcDict['Path'].split('/')[:3])
serviceDict['FileName'] = ''
res = pfnunparse(serviceDict)
if not res['OK']:
return res
srcSEURL = res['Value']
localCache = True
transferClient = TransferClient(srcSEURL)
res = transferClient.receiveFile(
srcDict['FileName'], os.path.join(
srcDict['Path'], srcDict['FileName']))
if not res['OK']:
return res
src_file = srcDict['FileName']
if not os.path.exists(src_file):
errStr = "DIPStorage.__putFile: The source local file does not exist."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
sourceSize = getSize(src_file)
if sourceSize == -1:
errStr = "DIPStorage.__putFile: Failed to get file size."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
transferClient = TransferClient(self.url)
res = transferClient.sendFile(src_file, dest_url, token=self.checkSum)
if localCache:
os.unlink(src_file)
if res['OK']:
return S_OK(sourceSize)
else:
return res
def test_putRemoveFile( self ):
print '\n\n#########################################################################\n\n\t\t\tPut and Remove test\n'
# Make sure that we can actually upload a file properly
srcFile = '/etc/group'
srcFileSize = getSize( srcFile )
testFileName = 'testFile.%s' % time.time()
destFile = self.storage.getCurrentURL( testFileName )['Value']
fileDict = {destFile:srcFile}
putFileRes = self.storage.putFile( fileDict )
# Make sure we are able to remove the file
removeFileRes = self.storage.removeFile( destFile )
# Check the successful put file operation
self.assert_( putFileRes['OK'] )
self.assert_( putFileRes['Value']['Successful'].has_key( destFile ) )
self.assertEqual( putFileRes['Value']['Successful'][destFile], srcFileSize )
# Check the remove file operation
self.assert_( removeFileRes['OK'] )
self.assert_( removeFileRes['Value']['Successful'].has_key( destFile ) )
self.assert_( removeFileRes['Value']['Successful'][destFile] )
def getFile( self, path, localPath = False ):
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
client = RPCClient( self.url )
# Make sure transferClient uses the same ProxyStorage instance.
# Only the this one holds the file we want to transfer.
transferClient = TransferClient( client.serviceURL )
for src_url in urls.keys():
res = client.prepareFile( self.name, src_url )
if not res['OK']:
gLogger.error( "ProxyStorage.getFile: Failed to prepare file on remote server.", res['Message'] )
failed[src_url] = res['Message']
else:
fileName = os.path.basename( src_url )
if localPath:
dest_file = "%s/%s" % ( localPath, fileName )
else:
dest_file = "%s/%s" % ( os.getcwd(), fileName )
res = transferClient.receiveFile( dest_file, 'getFile/%s' % fileName )
if not res['OK']:
gLogger.error( "ProxyStorage.getFile: Failed to recieve file from proxy server.", res['Message'] )
failed[src_url] = res['Message']
elif not os.path.exists( dest_file ):
errStr = "ProxyStorage.getFile: The destination local file does not exist."
gLogger.error( errStr, dest_file )
failed[src_url] = errStr
else:
destSize = getSize( dest_file )
if destSize == -1:
errStr = "ProxyStorage.getFile: Failed to get the local file size."
gLogger.error( errStr, dest_file )
failed[src_url] = errStr
else:
successful[src_url] = destSize
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def test_putGetFileSize(self):
print "\n\n#########################################################################\n\n\t\t\tGet file size test\n"
# First upload a file to the storage
srcFile = "/etc/group"
fileSize = getSize(srcFile)
testFileName = "testFile.%s" % time.time()
destFile = self.storage.getCurrentURL(testFileName)["Value"]
fileTuple = (srcFile, destFile, fileSize)
putFileRes = self.storage.putFile(fileTuple)
# Check that we can get the file size
getSizeRes = self.storage.getFileSize(destFile)
# Clean up the remote mess
removeFileRes = self.storage.removeFile(destFile)
# Check non existant files
failedSizeRes = self.storage.getFileMetadata(destFile)
# Check directories are handled properly
destDir = os.path.dirname(destFile)
directorySizeRes = self.storage.getFileSize(destDir)
# Check the put file operation
self.assert_(putFileRes["OK"])
self.assert_(putFileRes["Value"]["Successful"].has_key(destFile))
# Check that we got the file size correctly
self.assert_(getSizeRes["OK"])
self.assert_(getSizeRes["Value"]["Successful"].has_key(destFile))
self.assertEqual(getSizeRes["Value"]["Successful"][destFile], fileSize)
# check the remove file operation
self.assert_(removeFileRes["OK"])
self.assert_(removeFileRes["Value"]["Successful"].has_key(destFile))
# Check the get size with non existant file works properly
self.assert_(failedSizeRes["OK"])
self.assert_(failedSizeRes["Value"]["Failed"].has_key(destFile))
expectedError = "No such file or directory"
self.assertEqual(failedSizeRes["Value"]["Failed"][destFile], expectedError)
# Check that the passing a directory is handled correctly
self.assert_(directorySizeRes["OK"])
self.assert_(directorySizeRes["Value"]["Failed"].has_key(destDir))
expectedError = "RFIOStorage.getFileSize: Supplied path is not a file."
self.assertEqual(directorySizeRes["Value"]["Failed"][destDir], expectedError)
def __getFile( self, src_url, dest_file ):
"""Get a local copy in the current directory of a physical file specified by its path
"""
if not os.path.exists( os.path.dirname( dest_file ) ):
os.makedirs( os.path.dirname( dest_file ) )
if os.path.exists( dest_file ):
gLogger.debug( "RFIOStorage.getFile: Local file already exists %s. Removing..." % dest_file )
os.remove( dest_file )
res = self.__executeOperation( src_url, 'getFileSize' )
if not res['OK']:
return S_ERROR( res['Message'] )
remoteSize = res['Value']
MIN_BANDWIDTH = 1024 * 100 # 100 KB/s
timeout = remoteSize / MIN_BANDWIDTH + 300
gLogger.debug( "RFIOStorage.getFile: Executing transfer of %s to %s" % ( src_url, dest_file ) )
comm = "rfcp %s %s" % ( src_url, dest_file )
res = shellCall( timeout, comm )
if res['OK']:
returncode, _stdout, stderr = res['Value']
if returncode == 0:
gLogger.debug( 'RFIOStorage.__getFile: Got file from storage, performing post transfer check.' )
localSize = getSize( dest_file )
if localSize == remoteSize:
gLogger.debug( "RFIOStorage.getFile: Post transfer check successful." )
return S_OK( localSize )
errorMessage = "RFIOStorage.__getFile: Source and destination file sizes do not match."
gLogger.error( errorMessage, src_url )
else:
errStr = "RFIOStorage.__getFile: Failed to get local copy of file."
gLogger.error( errStr, stderr )
errorMessage = "%s %s" % ( errStr, stderr )
else:
errStr = "RFIOStorage.__getFile: Failed to get local copy of file."
gLogger.error( errStr, res['Message'] )
errorMessage = "%s %s" % ( errStr, res['Message'] )
if os.path.exists( dest_file ):
gLogger.debug( "RFIOStorage.getFile: Removing local file %s." % dest_file )
os.remove( dest_file )
return S_ERROR( errorMessage )
def test_putPrestageFile( self ):
print '\n\n#########################################################################\n\n\t\t\tFile prestage test\n'
# First upload a file to the storage
srcFile = '/etc/group'
fileSize = getSize( srcFile )
testFileName = 'testFile.%s' % time.time()
destFile = self.storage.getCurrentURL( testFileName )['Value']
fileTuple = ( srcFile, destFile, fileSize )
putFileRes = self.storage.putFile( fileTuple )
# Check that we can issue a stage request
prestageRes = self.storage.prestageFile( destFile )
# Clean up the remote mess
removeFileRes = self.storage.removeFile( destFile )
# Check the put file operation
self.assert_( putFileRes['OK'] )
self.assert_( putFileRes['Value']['Successful'].has_key( destFile ) )
# Check the prestage file operation
self.assert_( prestageRes['OK'] )
self.assert_( prestageRes['Value']['Successful'].has_key( destFile ) )
self.assert_( prestageRes['Value']['Successful'][destFile] )
# Check the remove file operation
self.assert_( removeFileRes['OK'] )
self.assert_( removeFileRes['Value']['Successful'].has_key( destFile ) )
def testGetSize( self ):
""" getSize tests """
# non existing file
self.assertEqual( getSize("/spam/eggs/eggs"), -1, "inexisting file" )
# file unreadable
self.assertEqual( getSize('/root/.login'), -1 , "unreadable file")
def __putFile( self, src_file, dest_url, sourceSize ):
"""Put a copy of the local file to the current directory on the physical storage
"""
# Pre-transfer check
res = self.__executeOperation( dest_url, 'exists' )
if not res['OK']:
gLogger.debug( "RFIOStorage.__putFile: Failed to find pre-existance of destination file." )
return res
if res['Value']:
res = self.__executeOperation( dest_url, 'removeFile' )
if not res['OK']:
gLogger.debug( "RFIOStorage.__putFile: Failed to remove remote file %s." % dest_url )
else:
gLogger.debug( "RFIOStorage.__putFile: Removed remote file %s." % dest_url )
if not os.path.exists( src_file ):
errStr = "RFIOStorage.__putFile: The source local file does not exist."
gLogger.error( errStr, src_file )
return S_ERROR( errStr )
sourceSize = getSize( src_file )
if sourceSize == -1:
errStr = "RFIOStorage.__putFile: Failed to get file size."
gLogger.error( errStr, src_file )
return S_ERROR( errStr )
res = self.__getTransportURL( dest_url )
if not res['OK']:
gLogger.debug( "RFIOStorage.__putFile: Failed to get transport URL for file." )
return res
turl = res['Value']
MIN_BANDWIDTH = 1024 * 100 # 100 KB/s
timeout = sourceSize / MIN_BANDWIDTH + 300
gLogger.debug( "RFIOStorage.putFile: Executing transfer of %s to %s" % ( src_file, turl ) )
comm = "rfcp %s '%s'" % ( src_file, turl )
res = shellCall( timeout, comm )
if res['OK']:
returncode, _stdout, stderr = res['Value']
if returncode == 0:
gLogger.debug( 'RFIOStorage.putFile: Put file to storage, performing post transfer check.' )
res = self.__executeOperation( dest_url, 'getFileSize' )
if res['OK']:
destinationSize = res['Value']
if sourceSize == destinationSize :
gLogger.debug( "RFIOStorage.__putFile: Post transfer check successful." )
return S_OK( destinationSize )
errorMessage = "RFIOStorage.__putFile: Source and destination file sizes do not match."
gLogger.error( errorMessage, dest_url )
else:
errStr = "RFIOStorage.__putFile: Failed to put file to remote storage."
gLogger.error( errStr, stderr )
errorMessage = "%s %s" % ( errStr, stderr )
else:
errStr = "RFIOStorage.__putFile: Failed to put file to remote storage."
gLogger.error( errStr, res['Message'] )
errorMessage = "%s %s" % ( errStr, res['Message'] )
res = self.__executeOperation( dest_url, 'removeFile' )
if res['OK']:
gLogger.debug( "RFIOStorage.__putFile: Removed remote file remnant %s." % dest_url )
else:
gLogger.debug( "RFIOStorage.__putFile: Unable to remove remote file remnant %s." % dest_url )
return S_ERROR( errorMessage )