"""
SRM_OUTPUT=`srmls -recursion_depth=0 -retry_num=1 %s 2>/dev/null`
SRM_SIZE=`echo $SRM_OUTPUT | grep '%s' | grep -v '%s' | awk '{print $1;}'`
echo "SRM Size is $SRM_SIZE"
if [[ $SRM_SIZE == $FILE_SIZE ]]; then
exit 0
else
echo $SRM_OUTPUT
echo "ERROR: Size Mismatch between local and SE. Cleaning up failed file..."
%s
exit 60311
fi
""" % (remotePFN, remotePath, remoteHost, self.createRemoveFileCommand(targetPFN))
result += metadataCheck
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided
"""
command = "srmrm -2 -retry_num=0 %s" % pfnToRemove
self.executeCommand(command)
registerStageOutImpl("srmv2", SRMV2Impl)
"""
_createStageOutCommand_
Build an cp command
"""
original_size = os.stat(sourcePFN)[6]
print "Local File Size is: %s" % original_size
result = " /bin/cp "
if options != None:
result += " %s " % options
result += " %s " % sourcePFN
result += " %s " % targetPFN
result += "; DEST_SIZE=`/bin/ls -l %s | awk '{print $5}'` ; if [ $DEST_SIZE ] && [ '%s' -eq $DEST_SIZE ]; then exit 0; else echo \"Error: Size Mismatch between local and SE\"; exit 60311 ; fi " % (targetPFN,original_size)
print result
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided
"""
command = "/bin/rm %s" % pfnToRemove
self.executeCommand(command)
registerStageOutImpl("testFallbackToOldBackend", TestFallbackToOldBackendImpl)
echo "Cleaning up failed file:"
%s
exit 60311
fi
else
sleep 2
fi
done
echo "Cleaning up failed file:"
%s
exit 60311
""" % (remotePFN, checksumCommand, removeCommand, removeCommand)
result += metadataCheck
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided
"""
command = "%s lcg-del -b -l -D srmv2 --vo cms %s" % (self.setups,
pfnToRemove)
self.executeCommand(command)
registerStageOutImpl("srmv2-lcg", LCGImpl)
def createStageOutCommand(self, sourcePFN, targetPFN, options = None, checksums = None):
"""
_createStageOutCommand_
Build a dccp command
"""
optionsStr = ""
if options != None:
optionsStr = str(options)
dirname = os.path.dirname(targetPFN)
result = "#!/bin/sh\n"
result += "dccp %s %s %s" % ( optionsStr, sourcePFN, targetPFN)
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided:
Cannot implement this, since there isnt a dcrm command
"""
pass
registerStageOutImpl("dccp-generic", DCCPGenericImpl)
"""
result = "rfcp "
if options != None:
result += " %s " % options
result += " %s " % sourcePFN
result += " %s " % targetPFN
if self.stageIn:
remotePFN, localPFN = sourcePFN, targetPFN
else:
remotePFN, localPFN = targetPFN, sourcePFN
result += "\nFILE_SIZE=`stat -c %s"
result += " %s `;\n" % localPFN
result += " echo \"Local File Size is: $FILE_SIZE\"; DEST_SIZE=`rfstat %s | grep Size | cut -f2 -d:` ; if [ $DEST_SIZE ] && [ $FILE_SIZE == $DEST_SIZE ]; then exit 0; else echo \"Error: Size Mismatch between local and SE\"; exit 60311 ; fi " % (remotePFN)
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided: specific for Castor-1
"""
command = "stager_rm -M %s ; nsrm %s" %(pfnToRemove,pfnToRemove)
self.executeCommand(command)
registerStageOutImpl("rfcp-2", RFCP2Impl)
host = pfn.split('/')[2]
thisList = pfn.replace( '%s://%s/' % ( protocol, host ),'' ).split( '?' )
path = thisList[0]
opaque = ""
# If we have any opaque info keep it
if len( thisList ) == 2:
opaque = "?%s" % thisList[1]
# check for the path to actually be in the opaque information
if opaque.startswith( "?path=" ):
elements = opaque.split( '&' )
path = elements[0].replace('?path=','')
buildingOpaque = '?'
for element in elements[1:]:
buildingOpaque += element
buildingOpaque += '&'
opaque = buildingOpaque.rstrip( '&' )
elif opaque.find( "&path=" ) != -1:
elements = opaque.split( '&' )
buildingOpaque = elements[0]
for element in elements[1:]:
if element.startswith( 'path=' ):
path = element.replace( 'path=','' )
else:
buildingOpaque += '&' + element
opaque = buildingOpaque
return protocol, host, path, opaque
registerStageOutImpl("rfcp-CERN", RFCPCERNImpl)
def createStageOutCommand(self, sourcePFN, targetPFN, options = None, checksums = None):
"""
_createStageOutCommand_
Build a dccp command
"""
optionsStr = ""
if options != None:
optionsStr = str(options)
dirname = os.path.dirname(targetPFN)
result = "#!/bin/sh\n"
result += "dccp %s %s %s" % ( optionsStr, sourcePFN, targetPFN)
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided:
Cannot implement this, since there isnt a dcrm command
"""
pass
registerStageOutImpl("dccp-generic", DCCPGenericImpl)
def createStageOutCommand(self, sourcePFN, targetPFN, options = None, checksums = None):
"""
_createStageOutCommand_
Build an xrdcp command
"""
original_size = os.stat(sourcePFN)[6]
print "Local File Size is: %s" % original_size
result = ". /afs/cern.ch/user/c/cmsprod/scratch1/releases/CMSSW_1_8_0_pre0/src/runtime.sh ; xrdcp -d 3"
if options != None:
result += " %s " % options
result += " %s " % sourcePFN
result += " root://lxgate39.cern.ch/%s " % targetPFN
result += "; DEST_SIZE=`rfstat %s | grep Size | cut -f2 -d:` ; if [ $DEST_SIZE ] && [ '%s' == $DEST_SIZE ]; then exit 0; else echo \"Error: Size Mismatch between local and SE\"; exit 60311 ; fi " % (targetPFN,original_size)
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided: specific for Castor-2
"""
command = "stager_rm -M %s ; nsrm %s" %(pfnToRemove,pfnToRemove)
self.executeCommand(command)
registerStageOutImpl("xrdcp", XRDCPImpl)
def createStageOutCommand(self, sourcePFN, targetPFN, options = None, checksums = None):
"""
_createStageOutCommand_
Build an Hadoop put
"""
original_size = os.stat(sourcePFN)[6]
print "Local File Size is: %s" % original_size
result = "hadoop fs -put "
if options != None:
result += " %s " % options
result += " %s " % sourcePFN
result += " %s " % targetPFN
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided
"""
command = "hadoop fs -rm %s" % pfnToRemove
self.executeCommand(command)
registerStageOutImpl("hadoop", HadoopImpl)
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided
"""
command = ""
if pfnToRemove.startswith("s3://"):
command = "aws s3 rm %s" % pfnToRemove
elif pfnToRemove.startswith("/"):
command = "/bin/rm -f %s" % pfnToRemove
elif os.path.isfile(pfnToRemove):
command = "/bin/rm -f %s" % os.path.abspath(pfnToRemove)
self.executeCommand(command)
def createRemoveFileCommand(self, pfn):
"""
return the command to delete a file after a failed copy
"""
if pfn.startswith("s3://"):
return "aws s3 rm %s" % pfn
elif pfn.startswith("/"):
return "/bin/rm -f %s" % pfn
elif os.path.isfile(pfn):
return "/bin/rm -f %s" % os.path.abspath(pfn)
else:
return ""
registerStageOutImpl("awss3", AWSS3Impl)
if [[ $SRM_SIZE == $FILE_SIZE ]]; then
exit 0
else
echo "Error: Size Mismatch between local and SE"
echo "Cleaning up failed file:"
/bin/rm -f %s
exit 60311
fi
fi
echo "Cleaning up failed file:"
/bin/rm -f %s
exit 60311
""" % ( targetPnfsPath, targetPnfsPath, targetPnfsPath)
result += metadataCheck
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided
"""
command = "/bin/rm -f %s" % self.createPnfsPath(pfnToRemove)
self.executeCommand(command)
registerStageOutImpl("srm-fnal", SRMImpl)
dirName = os.path.dirname(targetPFN)
if not os.path.isdir(dirName):
os.makedirs(dirName)
return
def createSourceName(self, protocol, pfn):
"""
This should return the same PFN
"""
return pfn
def createStageOutCommand(self, sourcePFN, targetPFN, options = None, checksums = None):
command = "cp %s %s" %(sourcePFN, sourcePFN+'2')
return command
def removeFile(self, pfnToRemove):
command = "rm %s" %pfnToRemove
self.executeCommand(command)
return "WIN!!!"
registerStageOutImpl("test-win", WinImpl)
registerStageOutImpl("test-fail", FailImpl)
registerStageOutImpl("test-copy", LocalCopyImpl)
# remove multi-slashes from path
while (simpleCastorPath.find('//') > -1):
simpleCastorPath = simpleCastorPath.replace('//', '/')
return simpleCastorPath
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided: specific for Castor-1
"""
# Extract the CASTOR filepath
fileToDelete = self.parseCastorPath(pfnToRemove)
# Does file need removing from cmsTemp, or cmsFarmRead?
command = ''
if (re.match('/+castor/ads\.rl\.ac\.uk/prod/cms/store/unmerged/.*',
fileToDelete)):
command = "stager_rm -S cmsTemp -M %s ; nsrm %s" % (fileToDelete,
fileToDelete)
else:
command = "stager_rm -M %s ; nsrm %s" % (fileToDelete,
fileToDelete)
self.executeCommand(command)
registerStageOutImpl("rfcp-RAL", RFCPRALImpl)
sourcePFN,
targetPFN,
options=None,
checksums=None):
"""
_createStageOutCommand_
Build a shell command that will transfer the sourcePFN to the
targetPFN.
"""
if self.stageIn:
return "%s %s %s" % (self._downloadScript, sourcePFN, targetPFN)
else:
return "%s %s %s" % (self._cpScript, sourcePFN, targetPFN)
def removeFile(self, pfnToRemove):
"""
_removeFile_
Removes the pfn.
"""
command = "%s %s" % (self._rmScript, pfnToRemove)
self.executeCommand(command)
# Registers the implementation
registerStageOutImpl("vandy", VandyImpl)
"""
start=0
path=""
if pfn.startswith( "rfio:" ):
if pfn.find( "path=" ) != -1:
# first form, everything after path= is the path
dummy,path = pfn.split("path=")
else:
if pfn.find( "?" ) != -1:
# second form, path is between the third slash and the ?
path,dummy = pfn.split("?")
else:
# illegal form that seems to work rfio:///<path>
path = pfn
start = path.find( "//" ) # find 1st two
start = path.find( "/", start+2 ) # find 3rd
if path.find( "/", start+1 ) == start+1:
# if there is a 4th next get rid of the third
start += 1
path = path[start:]
else:
path = pfn
return os.path.dirname( path )
registerStageOutImpl("rfcp", RFCPImpl)
echo "Error: Size Mismatch between local and SE"
echo "Cleaning up failed file:"
%s
exit 60311
fi
else
sleep 2
fi
done
echo "Cleaning up failed file:"
%s
exit 60311
""" % (remotePFN, self.createRemoveFileCommand(targetPFN), self.createRemoveFileCommand(targetPFN))
result += metadataCheck
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided
"""
command = "srm-advisory-delete %s" % pfnToRemove
self.executeCommand(command)
registerStageOutImpl("srm", SRMImpl)
echo "Cleaning up failed file:"
/bin/rm -f %s
exit 60311
fi
fi
echo "Cleaning up failed file:"
/bin/rm -f %s
exit 60311
""" % (
targetPnfsPath,
targetPnfsPath,
targetPnfsPath,
)
result += metadataCheck
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided
"""
command = "/bin/rm -f %s" % self.createPnfsPath(pfnToRemove)
self.executeCommand(command)
registerStageOutImpl("srm-fnal", SRMImpl)
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided
"""
command = ""
if pfnToRemove.startswith("s3://"):
command = "aws s3 rm %s" % pfnToRemove
elif pfnToRemove.startswith("/"):
command = "/bin/rm -f %s" % pfnToRemove
elif os.path.isfile(pfnToRemove):
command = "/bin/rm -f %s" % os.path.abspath(pfnToRemove)
self.executeCommand(command)
def createRemoveFileCommand(self, pfn):
"""
return the command to delete a file after a failed copy
"""
if pfn.startswith("s3://"):
return "aws s3 rm %s" % pfn
elif pfn.startswith("/"):
return "/bin/rm -f %s" % pfn
elif os.path.isfile(pfn):
return "/bin/rm -f %s" % os.path.abspath(pfn)
else:
return ""
registerStageOutImpl("awss3", AWSS3Impl)
if [[ $EXIT_STATUS != 0 ]]; then
echo "ERROR: xrdcp exited with $EXIT_STATUS"
fi
exit $EXIT_STATUS
"""
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided
"""
method = self.storageMethod(pfnToRemove)
if method == 'xrdcp':
(_, host, path, _) = self.splitPFN(pfnToRemove)
command = "xrd %s rm %s" % (host, path)
print("Executing: %s" % command)
self.executeCommand(command)
elif method == 'srm':
return self.srmImpl.removeFile(pfnToRemove)
else:
command = "/bin/rm %s" % stripPrefixTOUNIX(pfnToRemove)
print("Executing: %s" % command)
self.executeCommand(command)
registerStageOutImpl("stageout-xrdcp-fnal", FNALImpl)
_createStageOutCommand_
Build a dccp command with a pnfs mkdir to generate the directory
"""
try:
import dcap
except ImportError as ie:
raise StageOutError("Python dCap wrappers not found on this host.")
optionsStr = ""
if options != None:
optionsStr = str(options)
result = "#!/bin/sh\n"
result += "dc_stageout %s %s %s" % ( optionsStr, sourcePFN, targetPFN)
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided
TODO: Implement cleanup of file via this API
"""
pass
registerStageOutImpl("pydcap", PYDCCPImpl)
else
sleep 2
fi
done
echo "Cleaning up failed file:"
%s
exit 60311
""" % (remotePFN, checksumCommand, removeCommand, removeCommand)
result += metadataCheck
# close sub-shell for CVMFS use case
if useCVMFS:
result += ")\n"
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided
"""
command = "%s lcg-del -b -l -D srmv2 --vo cms %s" % (self.setups, pfnToRemove)
self.executeCommand(command)
registerStageOutImpl("srmv2-lcg", LCGImpl)
host = pfn.split('/')[2]
thisList = pfn.replace( '%s://%s/' % ( protocol, host ),'' ).split( '?' )
path = thisList[0]
opaque = ""
# If we have any opaque info keep it
if len( thisList ) == 2:
opaque = "?%s" % thisList[1]
# check for the path to actually be in the opaque information
if opaque.startswith( "?path=" ):
elements = opaque.split( '&' )
path = elements[0].replace('?path=','')
buildingOpaque = '?'
for element in elements[1:]:
buildingOpaque += element
buildingOpaque += '&'
opaque = buildingOpaque.rstrip( '&' )
elif opaque.find( "&path=" ) != -1:
elements = opaque.split( '&' )
buildingOpaque = elements[0]
for element in elements[1:]:
if element.startswith( 'path=' ):
path = element.replace( 'path=','' )
else:
buildOpaque += '&' + element
opaque = buildingOpaque
return protocol, host, path, opaque
registerStageOutImpl("rfcp-CERN", RFCPCERNImpl)
# if that does not work just use as-is
if simpleCastorPath == None:
simpleCastorPath = complexCastorPath
# remove multi-slashes from path
while ( simpleCastorPath.find('//') > -1 ):
simpleCastorPath = simpleCastorPath.replace('//','/')
return simpleCastorPath
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided: specific for Castor-1
"""
# Extract the CASTOR filepath
fileToDelete = self.parseCastorPath(pfnToRemove)
# Does file need removing from cmsTemp, or cmsFarmRead?
command = ''
if(re.match('/+castor/ads.rl.ac.uk/prod/cms/store/unmerged/.*', fileToDelete)):
command = "stager_rm -S cmsTemp -M %s ; nsrm %s" % (fileToDelete, fileToDelete)
else:
command = "stager_rm -M %s ; nsrm %s" %(fileToDelete, fileToDelete)
self.executeCommand(command)
registerStageOutImpl("rfcp-RAL", RFCPRALImpl)
"""
_createStageOutCommand_
Build an cp command
"""
original_size = os.stat(sourcePFN)[6]
print "Local File Size is: %s" % original_size
result = " /bin/cp "
if options != None:
result += " %s " % options
result += " %s " % sourcePFN
result += " %s " % targetPFN
result += "; DEST_SIZE=`/bin/ls -l %s | awk '{print $5}'` ; if [ $DEST_SIZE ] && [ '%s' -eq $DEST_SIZE ]; then exit 0; else echo \"Error: Size Mismatch between local and SE\"; exit 60311 ; fi " % (
targetPFN, original_size)
print result
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided
"""
command = "/bin/rm %s" % pfnToRemove
self.executeCommand(command)
registerStageOutImpl("testFallbackToOldBackend", TestFallbackToOldBackendImpl)
"""
_createStageOutCommand_
Build a shell command that will transfer the sourcePFN to the
targetPFN.
"""
if self.stageIn:
return "%s %s %s" % (self._downloadScript, sourcePFN, targetPFN)
else:
return "%s %s %s" % (self._cpScript, sourcePFN, targetPFN)
def removeFile(self, pfnToRemove):
"""
_removeFile_
Removes the pfn.
"""
command = "%s %s" % (self._rmScript, pfnToRemove)
self.executeCommand(command)
# Registers the implementation
registerStageOutImpl("vandy", VandyImpl)
"""
result = "rfcp "
if options != None:
result += " %s " % options
result += " %s " % sourcePFN
result += " %s " % targetPFN
if self.stageIn:
remotePFN, localPFN = sourcePFN, targetPFN
else:
remotePFN, localPFN = targetPFN, sourcePFN
result += "\nFILE_SIZE=`stat -c %s"
result += " %s ;`\n" % localPFN
result += " echo \"Local File Size is: $FILE_SIZE\"; DEST_SIZE=`rfstat %s | grep Size | cut -f2 -d:` ; if [ $DEST_SIZE ] && [ $FILE_SIZE == $DEST_SIZE ]; then exit 0; else echo \"Error: Size Mismatch between local and SE\"; exit 60311 ; fi " % (remotePFN)
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided: specific for Castor-1
"""
command = "stageclr -M %s ; nsrm %s" %(pfnToRemove,pfnToRemove)
self.executeCommand(command)
registerStageOutImpl("rfcp-1", RFCP1Impl)
echo "Cleaning up failed file:"
%s
exit 60311
fi
else
sleep 2
fi
done
echo "Cleaning up failed file:"
%s
exit 60311
""" % (remotePFN, remotePath, remoteHost, self.createRemoveFileCommand(targetPFN), self.createRemoveFileCommand(targetPFN))
result += metadataCheck
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided
"""
command = "srmrm %s" % pfnToRemove
self.executeCommand(command)
registerStageOutImpl("srmv2", SRMV2Impl)
"""
result = "rfcp "
if options != None:
result += " %s " % options
result += " %s " % sourcePFN
result += " %s " % targetPFN
if self.stageIn:
remotePFN, localPFN = sourcePFN, targetPFN
else:
remotePFN, localPFN = targetPFN, sourcePFN
result += "\nFILE_SIZE=`stat -c %s"
result += " %s `;\n" % localPFN
result += " echo \"Local File Size is: $FILE_SIZE\"; DEST_SIZE=`rfstat %s | grep Size | cut -f2 -d:` ; if [ $DEST_SIZE ] && [ $FILE_SIZE == $DEST_SIZE ]; then exit 0; else echo \"Error: Size Mismatch between local and SE\"; exit 60311 ; fi " % (
remotePFN)
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided: specific for Castor-1
"""
command = "stager_rm -M %s ; nsrm %s" % (pfnToRemove, pfnToRemove)
self.executeCommand(command)
registerStageOutImpl("rfcp-2", RFCP2Impl)
EXIT_STATUS=$?
echo "gfal-copy exit status: $EXIT_STATUS"
if [[ $EXIT_STATUS != 0 ]]; then
echo "Non-zero gfal-copy Exit status!!!"
echo "Cleaning up failed file:"
%s
exit 60311
fi
exit 0
""" % self.createRemoveFileCommand(targetPFN)
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided
"""
command = ""
if os.path.isfile(pfnToRemove):
command = "/bin/rm -f %s" % os.path.abspath(pfnToRemove)
if pfnToRemove.startswith("file:"):
command = "env -i gfal-rm -vvv %s" % pfnToRemove
else:
command = "env -i gfal-rm -vvv %s" % pfnToRemove
self.executeCommand(command)
registerStageOutImpl("gfal2", GFAL2Impl)
def createStageOutCommand(self, sourcePFN, targetPFN, options = None, checksums = None):
"""
_createStageOutCommand_
Build an Hadoop put
"""
original_size = os.stat(sourcePFN)[6]
print("Local File Size is: %s" % original_size)
result = "hadoop fs -put "
if options != None:
result += " %s " % options
result += " %s " % sourcePFN
result += " %s " % targetPFN
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided
"""
command = "hadoop fs -rm %s" % pfnToRemove
self.executeCommand(command)
registerStageOutImpl("hadoop", HadoopImpl)
print("Local File Size is: %s" % original_size)
result = "/bin/cp "
if options != None:
result += " %s " % options
result += " %s " % sourcePFN
result += " %s " % targetPFN
result += "; DEST_SIZE=`/bin/ls -l %s | /bin/awk '{print $5}'` ; if [ $DEST_SIZE ] && [ '%s' == $DEST_SIZE ]; then exit 0; else echo \"Error: Size Mismatch between local and SE\"; exit 60311 ; fi " % (
targetPFN, original_size)
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided
"""
if pfnToRemove.find('/store/unmerged/') == -1:
pfnSplit = pfnToRemove.split("/store/", 1)[1]
filePath = "/pnfs/cms/WAX/11/store/%s" % pfnSplit
command = "rm -fv %s" % filePath
self.executeCommand(command)
else:
pfnSplit = pfnToRemove.split("/store/unmerged/", 1)[1]
pfnToRemove = "/lustre/unmerged/%s" % pfnSplit
command = "/bin/rm %s" % pfnToRemove
self.executeCommand(command)
registerStageOutImpl("dccp-fnal", DCCPFNALImpl)
"""
result = "rfcp "
if options != None:
result += " %s " % options
result += " %s " % sourcePFN
result += " %s " % targetPFN
if self.stageIn:
remotePFN, localPFN = sourcePFN, targetPFN
else:
remotePFN, localPFN = targetPFN, sourcePFN
result += "\nFILE_SIZE=`stat -c %s"
result += " %s ;`\n" % localPFN
result += " echo \"Local File Size is: $FILE_SIZE\"; DEST_SIZE=`rfstat %s | grep Size | cut -f2 -d:` ; if [ $DEST_SIZE ] && [ $FILE_SIZE == $DEST_SIZE ]; then exit 0; else echo \"Error: Size Mismatch between local and SE\"; exit 60311 ; fi " % (
remotePFN)
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided: specific for Castor-1
"""
command = "stageclr -M %s ; nsrm %s" % (pfnToRemove, pfnToRemove)
self.executeCommand(command)
registerStageOutImpl("rfcp-1", RFCP1Impl)
checksums=None):
"""
_createStageOutCommand_
Build an xrdcp command
"""
original_size = os.stat(sourcePFN)[6]
print "Local File Size is: %s" % original_size
result = ". /afs/cern.ch/user/c/cmsprod/scratch1/releases/CMSSW_1_8_0_pre0/src/runtime.sh ; xrdcp -d 3"
if options != None:
result += " %s " % options
result += " %s " % sourcePFN
result += " root://lxgate39.cern.ch/%s " % targetPFN
result += "; DEST_SIZE=`rfstat %s | grep Size | cut -f2 -d:` ; if [ $DEST_SIZE ] && [ '%s' == $DEST_SIZE ]; then exit 0; else echo \"Error: Size Mismatch between local and SE\"; exit 60311 ; fi " % (
targetPFN, original_size)
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided: specific for Castor-2
"""
command = "stager_rm -M %s ; nsrm %s" % (pfnToRemove, pfnToRemove)
self.executeCommand(command)
registerStageOutImpl("xrdcp", XRDCPImpl)
dirName = os.path.dirname(targetPFN)
if not os.path.isdir(dirName):
os.makedirs(dirName)
return
def createSourceName(self, protocol, pfn):
"""
This should return the same PFN
"""
return pfn
def createStageOutCommand(self,
sourcePFN,
targetPFN,
options=None,
checksums=None):
command = "cp %s %s" % (sourcePFN, sourcePFN + '2')
return command
def removeFile(self, pfnToRemove):
command = "rm %s" % pfnToRemove
self.executeCommand(command)
return "WIN!!!"
registerStageOutImpl("test-win", WinImpl)
registerStageOutImpl("test-fail", FailImpl)
registerStageOutImpl("test-copy", LocalCopyImpl)
"""
_createStageOutCommand_
Build an cp command
"""
original_size = os.stat(sourcePFN)[6]
print("Local File Size is: %s" % original_size)
result = " /bin/cp "
if options != None:
result += " %s " % options
result += " %s " % sourcePFN
result += " %s " % targetPFN
result += "; DEST_SIZE=`/bin/ls -l %s | awk '{print $5}'` ; if [ $DEST_SIZE ] && [ '%s' -eq $DEST_SIZE ]; then exit 0; else echo \"Error: Size Mismatch between local and SE\"; exit 60311 ; fi " % (
targetPFN, original_size)
print(result)
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided
"""
command = "/bin/rm %s" % pfnToRemove
self.executeCommand(command)
registerStageOutImpl("cp", CPImpl)
"""
_createStageOutCommand_
Build an cp command
"""
original_size = os.stat(sourcePFN)[6]
print "Local File Size is: %s" % original_size
result = " /bin/cp "
if options != None:
result += " %s " % options
result += " %s " % sourcePFN
result += " %s " % targetPFN
result += "; DEST_SIZE=`/bin/ls -l %s | awk '{print $5}'` ; if [ $DEST_SIZE ] && [ '%s' -eq $DEST_SIZE ]; then exit 0; else echo \"Error: Size Mismatch between local and SE\"; exit 60311 ; fi " % (targetPFN,original_size)
print result
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided
"""
command = "/bin/rm %s" % pfnToRemove
self.executeCommand(command)
registerStageOutImpl("cp", CPImpl)
fi
else
sleep 2
fi
done
echo "Cleaning up failed file:"
%s
exit 60311
""" % (
remotePFN,
self.createRemoveFileCommand(targetPFN),
self.createRemoveFileCommand(targetPFN),
)
result += metadataCheck
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided
"""
command = "srm-advisory-delete %s" % pfnToRemove
self.executeCommand(command)
registerStageOutImpl("srm", SRMImpl)
"""
start = 0
path = ""
if pfn.startswith("rfio:"):
if pfn.find("path=") != -1:
# first form, everything after path= is the path
dummy, path = pfn.split("path=")
else:
if pfn.find("?") != -1:
# second form, path is between the third slash and the ?
path, dummy = pfn.split("?")
else:
# illegal form that seems to work rfio:///<path>
path = pfn
start = path.find("//") # find 1st two
start = path.find("/", start + 2) # find 3rd
if path.find("/", start + 1) == start + 1:
# if there is a 4th next get rid of the third
start += 1
path = path[start:]
else:
path = pfn
return os.path.dirname(path)
registerStageOutImpl("rfcp", RFCPImpl)
print("Local File Size is: %s" % original_size)
result = "/bin/cp "
if options != None:
result += " %s " % options
result += " %s " % sourcePFN
result += " %s " % targetPFN
result += "; DEST_SIZE=`/bin/ls -l %s | /bin/awk '{print $5}'` ; if [ $DEST_SIZE ] && [ '%s' == $DEST_SIZE ]; then exit 0; else echo \"Error: Size Mismatch between local and SE\"; exit 60311 ; fi " % (
targetPFN, original_size)
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided
"""
if pfnToRemove.find('/store/unmerged/') == -1:
pfnSplit = pfnToRemove.split("/store/", 1)[1]
filePath = "/pnfs/cms/WAX/11/store/%s" % pfnSplit
command = "rm -fv %s" % filePath
self.executeCommand(command)
else:
pfnSplit = pfnToRemove.split("/store/unmerged/", 1)[1]
pfnToRemove = "/lustre/unmerged/%s" % pfnSplit
command = "/bin/rm %s" % pfnToRemove
self.executeCommand(command)
registerStageOutImpl("dccp-fnal", DCCPFNALImpl)
_createStageOutCommand_
Build a dccp command with a pnfs mkdir to generate the directory
"""
try:
import dcap
except ImportError, ie:
raise StageOutError("Python dCap wrappers not found on this host.")
optionsStr = ""
if options != None:
optionsStr = str(options)
result = "#!/bin/sh\n"
result += "dc_stageout %s %s %s" % (optionsStr, sourcePFN, targetPFN)
return result
def removeFile(self, pfnToRemove):
"""
_removeFile_
CleanUp pfn provided
TODO: Implement cleanup of file via this API
"""
pass
registerStageOutImpl("pydcap", PYDCCPImpl)
