def __init__(self):
StageOutImplV2.__init__(self)
# Create and hold onto a srm implementation in case we need it
self.srmImpl = LCGImpl()
self.cpImpl = CPImpl()
def __init__(self):
StageOutImplV2.__init__(self)
# Create and hold onto a srm implementation in case we need it
self.srmImpl = LCGImpl()
self.cpImpl = CPImpl()
class FNALImpl(StageOutImplV2):
"""
_FNALImpl_
Implement interface for dcache door based dccp command
"""
def __init__(self):
StageOutImplV2.__init__(self)
# Create and hold onto a srm implementation in case we need it
self.srmImpl = LCGImpl()
def storageMethod(self, PFN):
"""
Figure out which paths use DCAP, lustre, or SRM for access
"""
method = 'dccp'
for path in checkPaths:
if PFN.find(path) != -1:
method = 'lustre'
# Over ride a few paths for srm
for path in srmPaths:
if PFN.find(path) != -1:
method = 'srm'
# overrides for testing
for path in dccpPaths:
if PFN.find(path) != -1:
method = 'dccp'
if PFN.find('/store/user/meloam/lustre/') != -1:
method = 'lustre'
logging.debug("Using method %s for PFN %s", method, PFN)
return method
def substituteLustrePath(self, pfn):
for i in range(checkPathsCount):
if pfn.find(checkPaths[i]) != -1:
pfnSplit = pfn.split(checkPaths[i], 1)[1]
pfn = "%s%s" % (checkPathsReplace[i], pfnSplit)
return pfn
def dcapToPNFS(self, pfn):
if pfn.find('/pnfs/') == -1:
raise RuntimeError("dcapToPNFS called on a non-dcap/pnfs URL: %s" %
pfn)
pfnSplit = pfn.split("WAX/11/store/", 1)[1]
filePath = "/pnfs/cms/WAX/11/store/%s" % pfnSplit
return filePath
def createOutputDirectoryDCAP(self, targetPFN):
"""
_createOutputDirectory_
Create a dir for the target pfn by translating it to
a /pnfs name and calling mkdir
PFN will be of the form:
dcap://cmsdca.fnal.gov:22125/pnfs/fnal.gov/usr/cms/WAX/11/store/blah
We need to convert that into /pnfs/cms/WAX/11/store/blah, as it
will be seen from the worker node
Unmerged PFN will be of the form:
/lustre/unmerged
we don't need to convert it, just mkdir.
"""
logging.info("createOutputDirectoryDCAP called on %s", targetPFN)
# only create dir on remote storage
if targetPFN.find('/pnfs/') == -1:
return
filePath = self.dcapToPNFS(targetPFN)
directory = os.path.dirname(filePath)
logging.info("Creating path: %s", directory)
if not os.path.exists(directory):
os.makedirs(directory)
def createOutputDirectoryEOS(self, targetPFN):
logging.info("createOutputDirectoryEOS called on %s", targetPFN)
# only create dir on remote storage
if targetPFN.find('root://cmseos.fnal.gov') == -1:
return
filePath = targetPFN.split("root://cmseos.fnal.gov/", 1)[1]
directory = os.path.dirname(filePath)
logging.info("Creating path: %s", directory)
if not os.path.exists(directory):
os.makedirs(directory)
def createSourceName(self, protocol, pfn):
"""
createTargetName
generate the target PFN
"""
# if not pfn.startswith("srm"):
# return pfn
method = self.storageMethod(pfn)
if method == 'srm':
return self.srmImpl.createSourceName(protocol, pfn)
elif method == 'dccp':
logging.info("Translating PFN for dcache: %s", pfn)
pfn = pfn.split("/store/")[1]
dcacheDoor = getoutput(
". " + envScript + "\n" + \
"/opt/d-cache/dcap/bin/select_RdCapDoor.sh")
pfn = "%s%s" % (dcacheDoor, pfn)
logging.info(" Translated PFN: %s", pfn)
elif method == 'lustre':
logging.info("Translating PFN for lustre: %s", pfn)
pfn = self.substituteLustrePath(pfn)
logging.info(" Translated PFN: %s", pfn)
else:
raise RuntimeError("Unknown method found in createSourceName: %s" %
method)
return pfn
def doDelete(self, pfnToRemove, pnn, command, options, protocol):
"""
_removeFile_
CleanUp pfn provided
"""
method = self.storageMethod(pfnToRemove)
if method == 'srm':
return self.srmImpl.doDelete(pfnToRemove, pnn, command, options,
protocol)
elif method == 'dccp':
pfnSplit = pfnToRemove.split("/11/store/", 1)[1]
filePath = "/pnfs/cms/WAX/11/store/%s" % pfnSplit
if os.path.exists(filePath):
logging.info("Removing file: %s", filePath)
os.unlink(filePath)
elif method == 'lustre':
pfnToRemove = self.substituteLustrePath(pfnToRemove)
filePath = pfnToRemove.split("root://cmseos.fnal.gov/", 1)[1]
if os.path.exists(filePath):
logging.info("Removing file: %s", filePath)
os.unlink(filePath)
else:
raise RuntimeError("Unsupported storage method in doDelete: %s" %
method)
def doTransfer(self, fromPfn, toPfn, stageOut, pnn, command, options,
protocol, checksum):
"""
performs a transfer. stageOut tells you which way to go. returns the new pfn or
raises on failure. StageOutError (and inherited exceptions) are for expected errors
such as temporary connection failures. Anything else will be handled as an unexpected
error and skip retrying with this plugin
if stageOut is true:
The fromPfn is the LOCAL FILE NAME on the node, without file://
the toPfn is the target PFN, mapped from the LFN using the TFC or overrrides
if stageOut is false:
The toPfn is the LOCAL FILE NAME on the node, without file://
the fromPfn is the source PFN, mapped from the LFN using the TFC or overrrides
this behavior is because most transfer commands will switch their direction
simply by swapping the order of the arguments. the stageOut flag is provided
however, because sometimes you want to pass different command line args
"""
if stageOut:
localPFN = fromPfn
remotePFN = toPfn
else:
localPFN = toPfn
remotePFN = fromPfn
method = self.storageMethod(remotePFN)
sourceMethod = self.storageMethod(localPFN)
if method == 'srm' or sourceMethod == 'srm':
return self.srmImpl.doTransfer(fromPfn, toPfn, stageOut, \
pnn, command, options, protocol, checksum)
# transfer with lustre
elif method == 'lustre':
sourcePFN = self.substituteLustrePath(fromPfn)
targetPFN = self.substituteLustrePath(toPfn)
# there is no xrd module yet ...
if stageOut:
self.createOutputDirectoryEOS(targetPFN)
else:
self.createOutputDirectoryEOS(sourcePFN)
optionsStr = ""
if options != None:
optionsStr = str(options)
original_size = os.stat(sourcePFN)[6]
useChecksum = (checksum != None and 'adler32' in checksum
and not self.stageIn)
if useChecksum:
checksum['adler32'] = "%08x" % int(checksum['adler32'], 16)
# non-functional in 3.3.1 xrootd clients due to bug
# result += "-ODeos.targetsize=$LOCAL_SIZE\&eos.checksum=%s " % checksums['adler32']
# therefor embed information into target URL
targetPFN += "\?eos.targetsize=%s\&eos.checksum=%s" % (
original_size, checksum['adler32'])
result = "#!/bin/sh\n"
result += "xrdcp -d 0 %s %s %s\n" % (optionsStr, sourcePFN,
targetPFN)
try:
self.runCommandFailOnNonZero(result)
except:
logging.info("xrdcp failed, removing failed file")
if not stageOut and os.path.exists(pnfsPfn2(targetPFN)):
logging.info("unlinking %s", pnfsPfn2(targetPFN))
os.unlink(pnfsPfn2(targetPFN))
else:
self.doDelete(targetPFN, pnn, command, options, protocol)
raise
elif method == 'dccp':
if stageOut:
self.createOutputDirectoryDCAP(targetPFN)
targetPFN = self.createSourceName(protocol, targetPFN)
else:
self.createOutputDirectoryDCAP(sourcePFN)
sourcePFN = self.createSourceName(protocol, sourcePFN)
optionsStr = ""
if options != None:
optionsStr = str(options)
result = "#!/bin/bash\n"
result += ". " + envScript + "\n"
result += "dccp -o 86400 -d 0 -X -role=cmsprod %s %s %s" % (
optionsStr, sourcePFN, targetPFN)
try:
self.runCommandFailOnNonZero(result)
except:
logging.info("dccp failed, removing failed file")
if not stageOut and os.path.exists(pnfsPfn2(targetPFN)):
logging.info("unlinking %s", pnfsPfn2(targetPFN))
os.unlink(pnfsPfn2(targetPFN))
else:
self.doDelete(targetPFN, pnn, command, options, protocol)
raise
# //
# // CRC check
# //
try:
if stageOut:
self.runCommandFailOnNonZero("/opt/d-cache/dcap/bin/check_dCachefilecksum.sh %s %s" \
% (pnfsPfn2(targetPFN), sourcePFN))
else:
self.runCommandFailOnNonZero("/opt/d-cache/dcap/bin/check_dCachefilecksum.sh %s %s" \
% (pnfsPfn2(sourcePFN), targetPFN))
except:
logging.info("DCCP CRC Check failed, removing failed file")
self.doDelete(targetPFN, pnn, command, options, protocol)
raise
return
class FNALImpl(StageOutImplV2):
"""
_FNALImpl_
Implement interface for dcache door based dccp command
"""
def __init__(self):
StageOutImplV2.__init__(self)
# Create and hold onto a srm implementation in case we need it
self.srmImpl = LCGImpl()
self.cpImpl = CPImpl()
def storageMethod(self, PFN):
"""
Figure out which paths use DCAP, lustre, or SRM for access
"""
method = 'dccp'
for path in checkPaths:
if PFN.find(path) != -1:
method = 'lustre'
# Over ride a few paths for srm
for path in srmPaths:
if PFN.find(path) != -1:
method = 'srm'
# overrides for testing
for path in dccpPaths:
if PFN.find(path) != -1:
method = 'dccp'
if PFN.find('/store/user/meloam/lustre/') != -1:
method = 'lustre'
logging.debug("Using method %s for PFN %s" % (method, PFN))
return method
def substituteLustrePath(self, pfn):
for i in range(checkPathsCount):
if pfn.find(checkPaths[i]) != -1:
pfnSplit = pfn.split(checkPaths[i], 1)[1]
pfn = "%s%s" % (checkPathsReplace[i],pfnSplit)
return pfn
def dcapToPNFS(self, pfn):
if pfn.find('/pnfs/') == -1:
raise RuntimeError, "dcapToPNFS called on a non-dcap/pnfs URL: %s" % pfn
pfnSplit = pfn.split("WAX/11/store/", 1)[1]
filePath = "/pnfs/cms/WAX/11/store/%s" % pfnSplit
return filePath
def createOutputDirectoryDCAP(self, targetPFN):
"""
_createOutputDirectory_
Create a dir for the target pfn by translating it to
a /pnfs name and calling mkdir
PFN will be of the form:
dcap://cmsdca.fnal.gov:22125/pnfs/fnal.gov/usr/cms/WAX/11/store/blah
We need to convert that into /pnfs/cms/WAX/11/store/blah, as it
will be seen from the worker node
Unmerged PFN will be of the form:
/lustre/unmerged
we don't need to convert it, just mkdir.
"""
logging.info("createOutputDirectoryDCAP called on %s" % targetPFN)
# only create dir on remote storage
if targetPFN.find('/pnfs/') == -1:
return
filePath = self.dcapToPNFS( targetPFN )
directory = os.path.dirname(filePath)
logging.info("Creating path: %s" % directory)
if not os.path.exists( directory ):
os.makedirs( directory )
def createOutputDirectoryEOS(self, targetPFN):
logging.info("createOutputDirectoryEOS called on %s" % targetPFN)
# only create dir on remote storage
if targetPFN.find('root://cmseos.fnal.gov') == -1:
return
filePath = targetPFN.split("root://cmseos.fnal.gov/", 1)[1]
directory = os.path.dirname(filePath)
logging.info("Creating path: %s" % directory)
if not os.path.exists( directory ):
os.makedirs( directory )
def createSourceName(self, protocol, pfn):
"""
createTargetName
generate the target PFN
"""
#if not pfn.startswith("srm"):
# return pfn
method = self.storageMethod(pfn)
if method == 'srm':
return self.srmImpl.createSourceName(protocol, pfn)
elif method == 'dccp':
logging.info("Translating PFN for dcache: %s" % pfn)
pfn = pfn.split("/store/")[1]
dcacheDoor = commands.getoutput(
". " + envScript + "\n" +\
"/opt/d-cache/dcap/bin/select_RdCapDoor.sh")
pfn = "%s%s" % (dcacheDoor, pfn)
logging.info(" Translated PFN: %s" % pfn )
elif method == 'lustre':
logging.info("Translating PFN for lustre: %s" % pfn)
pfn = self.substituteLustrePath( pfn )
logging.info(" Translated PFN: %s" % pfn)
else:
raise RuntimeError, "Unknown method found in createSourceName: %s" % method
return pfn
def doDelete(self, pfnToRemove, seName, command, options, protocol ):
"""
_removeFile_
CleanUp pfn provided
"""
method = self.storageMethod(pfnToRemove)
if method == 'srm':
return self.srmImpl.doDelete(pfnToRemove, seName, command, options, protocol)
elif method == 'dccp':
pfnSplit = pfnToRemove.split("/11/store/", 1)[1]
filePath = "/pnfs/cms/WAX/11/store/%s" % pfnSplit
if os.path.exists( filePath ):
logging.info( "Removing file: " + filePath )
os.unlink( filePath )
elif method == 'lustre':
pfnToRemove = self.substituteLustrePath( pfnToRemove )
filePath = pfnToRemove.split("root://cmseos.fnal.gov/",1 )[1]
if os.path.exists( filePath ):
logging.info( "Removing file: " + filePath )
os.unlink( filePath )
else:
raise RuntimeError, "Unsupported storage method in doDelete: %s" % method
def doTransfer( self, fromPfn, toPfn, stageOut, seName, command, options, protocol, checksum ):
"""
performs a transfer. stageOut tells you which way to go. returns the new pfn or
raises on failure. StageOutError (and inherited exceptions) are for expected errors
such as temporary connection failures. Anything else will be handled as an unexpected
error and skip retrying with this plugin
if stageOut is true:
The fromPfn is the LOCAL FILE NAME on the node, without file://
the toPfn is the target PFN, mapped from the LFN using the TFC or overrrides
if stageOut is false:
The toPfn is the LOCAL FILE NAME on the node, without file://
the fromPfn is the source PFN, mapped from the LFN using the TFC or overrrides
this behavior is because most transfer commands will switch their direction
simply by swapping the order of the arguments. the stageOut flag is provided
however, because sometimes you want to pass different command line args
"""
if stageOut:
localPFN = fromPfn
remotePFN = toPfn
else:
localPFN = toPfn
remotePFN = fromPfn
method = self.storageMethod( remotePFN )
sourceMethod = self.storageMethod( localPFN )
if method == 'srm' or sourceMethod == 'srm':
return self.srmImpl.doTransfer( fromPfn, toPfn, stageOut,\
seName, command, options, protocol, checksum)
# transfer with lustre
elif method == 'lustre':
sourcePFN = self.substituteLustrePath( fromPfn )
targetPFN = self.substituteLustrePath( toPfn )
# there is no xrd module yet ...
if stageOut:
self.createOutputDirectoryEOS( targetPFN )
else:
self.createOutputDirectoryEOS( sourcePFN )
optionsStr = ""
if options != None:
optionsStr = str(options)
original_size = os.stat(sourcePFN)[6]
useChecksum = (checksum != None and 'adler32' in checksum and not self.stageIn)
if useChecksum:
checksum['adler32'] = "%08x" % int(checksum['adler32'], 16)
# non-functional in 3.3.1 xrootd clients due to bug
#result += "-ODeos.targetsize=$LOCAL_SIZE\&eos.checksum=%s " % checksums['adler32']
# therefor embed information into target URL
targetPFN += "\?eos.targetsize=%s\&eos.checksum=%s" % (original_size, checksum['adler32'])
result = "#!/bin/sh\n"
result += "xrdcp -d 0 %s %s %s\n" % (optionsStr, sourcePFN, targetPFN)
try:
self.runCommandFailOnNonZero( result )
except:
logging.info("dccp failed, removing failed file")
if not stageOut and os.path.exists( pnfsPfn2(targetPFN) ):
logging.info("unlinking %s" % pnfsPfn2(targetPFN))
os.unlink( pnfsPfn2(targetPFN) )
else:
self.doDelete( targetPFN, seName, command, options, protocol )
raise
elif method == 'dccp':
if stageOut:
self.createOutputDirectoryDCAP( targetPFN )
targetPFN = self.createSourceName( protocol, targetPFN )
else:
self.createOutputDirectoryDCAP( sourcePFN )
sourcePFN = self.createSourceName( protocol, sourcePFN )
optionsStr = ""
if options != None:
optionsStr = str(options)
dirname = os.path.dirname(targetPFN)
result = "#!/bin/bash\n"
result += ". " + envScript + "\n"
result += "dccp -o 86400 -d 0 -X -role=cmsprod %s %s %s" % ( optionsStr, sourcePFN, targetPFN)
try:
self.runCommandFailOnNonZero( result )
except:
logging.info("dccp failed, removing failed file")
if not stageOut and os.path.exists( pnfsPfn2(targetPFN) ):
logging.info("unlinking %s" % pnfsPfn2(targetPFN))
os.unlink( pnfsPfn2(targetPFN) )
else:
self.doDelete( targetPFN, seName, command, options, protocol )
raise
# //
# // CRC check
#//
try:
if stageOut:
self.runCommandFailOnNonZero( "/opt/d-cache/dcap/bin/check_dCachefilecksum.sh %s %s" \
% (pnfsPfn2(targetPFN), sourcePFN))
else:
self.runCommandFailOnNonZero( "/opt/d-cache/dcap/bin/check_dCachefilecksum.sh %s %s" \
% (pnfsPfn2(sourcePFN), targetPFN))
except:
logging.info("DCCP CRC Check failed, removing failed file")
self.doDelete( targetPFN, seName, command, options, protocol )
raise
return