def doDelete(self, pfn, seName, command, options, protocol ):
"""
deletes a file, raises on error
StageOutError (and inherited exceptions) are for expected errors
such as temporary connection failures. Anything else will be handled as an unexpected
error and skip retrying with this plugin
"""
runCommand(["hadoop", "fs", "-rm" ,pfn])
def doDelete(self, pfn, seName, command, options, protocol):
"""
deletes a file, raises on error
StageOutError (and inherited exceptions) are for expected errors
such as temporary connection failures. Anything else will be handled as an unexpected
error and skip retrying with this plugin
"""
runCommand(["hadoop", "fs", "-rm", pfn])
def testRunCommand_results(self):
self.assertEqual(
1, runCommand("%s %s -exit 0" % (self.python_runtime, self.base)))
self.assertEqual(
1, runCommand("%s %s -exit a" % (self.python_runtime, self.base)))
self.assertEqual(
2, runCommand("%s %s -text" % (self.python_runtime, self.base)))
self.assertEqual(
0, runCommand("%s %s -text a" % (self.python_runtime, self.base)))
def runCommandFailOnNonZero(self, command):
(exitCode, output) = runCommand(command)
if not exitCode:
logging.error("Error in file transfer:")
logging.error(output)
raise StageOutError, "Transfer failure"
return (exitCode, output)
def doTransfer(self, fromPfn, toPfn, stageOut, seName, command, options, protocol):
# Figures out the src and dst files
if stageOut:
srcPath = fromPfn
dstPath = toPfn
else:
srcPath = toPfn
dstPath = fromPfn
# Creates the directory
self.createOutputDirectory(os.path.dirname(dstPath))
# Does the copy
command = "%s %s %s" % (self._cpScript, srcPath, dstPath)
exitCode, output = runCommand(command)
print(output)
if exitCode != 0:
logging.error("Error in file transfer:")
logging.error(output)
raise StageOutError, "Transfer failure"
# Returns the path
return dstPath
def runCommandWarnOnNonZero(self, command):
logging.info("Executing %s", command)
(exitCode, output) = runCommand(command)
if exitCode:
logging.error("Error in file transfer..ignoring:")
logging.error(output)
return (exitCode, output)
def runCommandWarnOnNonZero(self, command):
logging.info("Executing %s" % command)
(exitCode, output) = runCommand(command)
if exitCode:
logging.error("Error in file transfer..ignoring:")
logging.error(output)
return (exitCode, output)
def doWrapped(self, commandArgs):
wrapperPath = os.path.join(getWMBASE(),'src','python','WMCore','Storage','Plugins','DCCPFNAL','wrapenv.sh')
commandArgs.insert(0, wrapperPath)
exitCode, output = runCommand(commandArgs)
if exitCode != 0:
logging.info("Non zero exit code: %s" % repr(exitCode))
return (exitCode, output)
def doDelete(self, pfnToRemove, seName, command, options, protocol):
"""
deletes a file, raises on error
StageOutError (and inherited exceptions) are for expected errors
such as temporary connection failures. Anything else will be handled as an unexpected
error and skip retrying with this plugin
"""
if pfnToRemove.find('/store/unmerged/lustre/') == -1:
pfnSplit = pfnToRemove.split("/store/", 1)[1]
filePath = "/pnfs/cms/WAX/11/store/%s" % pfnSplit
command = ["rm", "-fv", filePath]
runCommand(command)
else:
pfnSplit = pfnToRemove.split("/store/unmerged/lustre/", 1)[1]
pfnToRemove = "/lustre/unmerged/%s" % pfnSplit
command = ["/bin/rm", pfnToRemove]
runCommand(command)
def doDelete(self, pfnToRemove, pnn, command, options, protocol ):
"""
deletes a file, raises on error
StageOutError (and inherited exceptions) are for expected errors
such as temporary connection failures. Anything else will be handled as an unexpected
error and skip retrying with this plugin
"""
if pfnToRemove.find('/store/unmerged/lustre/') == -1:
pfnSplit = pfnToRemove.split("/store/", 1)[1]
filePath = "/pnfs/cms/WAX/11/store/%s" % pfnSplit
command = ["rm","-fv",filePath]
runCommand(command)
else:
pfnSplit = pfnToRemove.split("/store/unmerged/lustre/", 1)[1]
pfnToRemove = "/lustre/unmerged/%s" % pfnSplit
command = ["/bin/rm", pfnToRemove]
runCommand(command)
def runCommandFailOnNonZero(self, command):
logging.info("Executing %s", command)
(exitCode, output) = runCommand(command)
if exitCode:
logging.error("Error in file transfer:")
logging.error(" Command executed was: %s", command)
logging.error(" Output was: %s", output)
logging.error(" Exit code was: %s", exitCode)
raise StageOutError("Transfer failure")
return (exitCode, output)
def runCommandFailOnNonZero(self, command):
logging.info("Executing %s" % command)
(exitCode, output) = runCommand(command)
if exitCode:
logging.error("Error in file transfer:")
logging.error(" Command executed was: %s" % command )
logging.error(" Output was: %s" % output )
logging.error(" Exit code was: %s" % exitCode )
raise StageOutError, "Transfer failure"
return (exitCode, output)
def doWrapped(self, commandArgs):
wrapperPath = os.path.join(getWMBASE(), 'src', 'python', 'WMCore',
'Storage', 'Plugins', 'DCCPFNAL',
'wrapenv.sh')
commandArgs.insert(0, wrapperPath)
exitCode, output = runCommand(commandArgs)
if exitCode != 0:
logging.info("Non zero exit code: %s" % repr(exitCode))
return (exitCode, output)
def executeCommand(self, command):
"""
_execute_
Execute the command provided, throw a StageOutError if it exits
non zero
"""
try:
exitCode = runCommand(command)
msg = "Command exited with status: %s" % (exitCode)
print msg
except Exception, ex:
raise StageOutError(str(ex), Command = command, ExitCode = 60311)
def executeCommand(self, command):
"""
_execute_
Execute the command provided, throw a StageOutError if it exits
non zero
"""
try:
exitCode = runCommand(command)
msg = "Command exited with status: %s" % (exitCode)
print msg
except Exception, ex:
raise StageOutError(str(ex), Command = command, ExitCode = 60311)
def doDelete(self, pfn, seName, command, options, protocol):
"""
_removeFile_
Removes the pfn.
"""
command = "%s %s" % (self._rmScript, pfn)
exitCode, output = runCommand(command)
if exitCode != 0:
logging.error("Error removing file from LStore")
logging.error(output)
raise StageOutError, "remove file failure command %s, error %s" % (command, output)
def doDelete(self, pfn, seName, command, options, protocol):
"""
_removeFile_
Removes the pfn.
"""
command = "%s %s" % (self._rmScript, pfn)
exitCode, output = runCommand(command)
if exitCode != 0:
logging.error("Error removing file from LStore")
logging.error(output)
raise StageOutError("remove file failure command %s, error %s" %
(command, output))
def createOutputDirectory(self, targetPFN):
"""
_createOutputDirectory_
Creates the directory for vanderbilt
"""
command = "%s %s" % (self._mkdirScript, os.path.dirname(targetPFN))
# Calls the parent execute command to invoke the script which should
# throw a stage out error
exitCode, output = runCommand(command)
if exitCode != 0:
logging.error("Error creating directory")
logging.error(output)
def createOutputDirectory(self, targetPFN):
"""
_createOutputDirectory_
Creates the directory for vanderbilt
"""
command = "%s %s" % (self._mkdirScript, targetPFN)
# Calls the parent execute command to invoke the script which should
# throw a stage out error
exitCode, output = runCommand(command)
if exitCode != 0:
logging.error("Error creating directory in LStore")
logging.error("Command: %s" % command)
logging.error(output)
def doDelete(self, pfn, seName, command, options, protocol ):
"""
deletes a file, raises on error
StageOutError (and inherited exceptions) are for expected errors
such as temporary connection failures. Anything else will be handled as an unexpected
error and skip retrying with this plugin
"""
if pfn.startswith("srm://"):
runCommand( "lcg-del -b -l -D srmv2 --vo cms %s" % pfn )
elif pfn.startswith("file:"):
runCommand( "/bin/rm -f %s" % pfn.replace("file:", "", 1) )
else:
runCommand( StageOutImpl.createRemoveFileCommand(self, pfn) )
def doDelete(self, pfn, pnn, command, options, protocol ):
"""
deletes a file, raises on error
StageOutError (and inherited exceptions) are for expected errors
such as temporary connection failures. Anything else will be handled as an unexpected
error and skip retrying with this plugin
"""
if pfn.startswith("srm://"):
runCommand( "lcg-del -b -l -D srmv2 --vo cms %s" % pfn )
elif pfn.startswith("file:"):
runCommand( "/bin/rm -f %s" % pfn.replace("file:", "", 1) )
else:
runCommand( StageOutImpl.createRemoveFileCommand(self, pfn) )
def doDelete(self, pfn, pnn, command, options, protocol):
"""
deletes a file, raises on error
StageOutError (and inherited exceptions) are for expected errors
such as temporary connection failures. Anything else will be handled as an unexpected
error and skip retrying with this plugin
"""
if os.path.isfile(pfn):
runCommand("/bin/rm -f %s" % pfn)
elif pfn.startswith("file:"):
runCommand("env -i X509_USER_PROXY=$X509_USER_PROXY gfal-rm -vvv %s" % pfn)
else:
runCommand(StageOutImpl.createRemoveFileCommand(self, pfn))
def doDelete(self, pfn, seName, command, options, protocol):
"""
deletes a file, raises on error
StageOutError (and inherited exceptions) are for expected errors
such as temporary connection failures. Anything else will be handled as an unexpected
error and skip retrying with this plugin
"""
if os.path.isfile(pfn):
runCommand("/bin/rm -f %s" % pfn)
elif pfn.startswith("file:"):
runCommand("env -i gfal-rm -vvv %s" % pfn)
else:
runCommand(StageOutImpl.createRemoveFileCommand(self, pfn))
def executeCommand(self, command):
"""
_execute_
Execute the command provided, throw a StageOutError if it exits
non zero
"""
try:
exitCode = runCommand(command)
msg = "%s : Command exited with status: %s\n" % (time.strftime("%Y-%m-%dT%H:%M:%S"), exitCode)
print msg
except Exception as ex:
raise StageOutError(str(ex), Command=command, ExitCode=60311)
if exitCode:
msg = "%s : Command exited non-zero" % time.strftime("%Y-%m-%dT%H:%M:%S")
print "ERROR: Exception During Stage Out:\n"
print msg
raise StageOutError(msg, Command=command, ExitCode=exitCode)
return
def doTransfer(self, fromPfn, toPfn, stageOut, seName, command, options,
protocol, checksum):
"""
if stageOut is true:
The fromPfn is the LOCAL FILE NAME on the node, without file://
the toPfn is the target PFN, mapped from the LFN using the TFC or overrrides
if stageOut is false:
The toPfn is the LOCAL FILE NAME on the node, without file://
the fromPfn is the source PFN, mapped from the LFN using the TFC or overrrides
"""
# Figures out the src and dst files
if stageOut:
dstPath = toPfn
else:
dstPath = fromPfn
# Creates the directory
if stageOut:
self.createOutputDirectory(os.path.dirname(dstPath))
else:
os.makedirs(os.path.dirname(dstPath))
# Does the copy
if stageOut:
command = "%s %s %s" % (self._cpScript, fromPfn, toPfn)
else:
command = "%s %s %s" % (self._downloadScript, fromPfn, toPfn)
exitCode, output = runCommand(command)
print(output)
if exitCode != 0:
logging.error("Error in file transfer:")
logging.error(output)
raise StageOutError("Transfer failure, command %s, error %s" %
(command, output))
# Returns the path
return dstPath
def doTransfer(self, fromPfn, toPfn, stageOut, seName, command, options,
protocol, checksum):
"""
if stageOut is true:
The fromPfn is the LOCAL FILE NAME on the node, without file://
the toPfn is the target PFN, mapped from the LFN using the TFC or overrrides
if stageOut is false:
The toPfn is the LOCAL FILE NAME on the node, without file://
the fromPfn is the source PFN, mapped from the LFN using the TFC or overrrides
"""
# Figures out the src and dst files
if stageOut:
dstPath = toPfn
else:
dstPath = fromPfn
# Creates the directory
if stageOut:
self.createOutputDirectory(os.path.dirname(dstPath))
else:
os.makedirs(os.path.dirname(dstPath))
# Does the copy
if stageOut:
command = "%s %s %s" % (self._cpScript, fromPfn, toPfn)
else:
command = "%s %s %s" % (self._downloadScript, fromPfn, toPfn)
exitCode, output = runCommand(command)
print(output)
if exitCode != 0:
logging.error("Error in file transfer:")
logging.error(output)
raise StageOutError, "Transfer failure, command %s, error %s" % (command, output)
# Returns the path
return dstPath
def executeCommand(self, command):
"""
_execute_
Execute the command provided, throw a StageOutError if it exits
non zero
"""
try:
exitCode = runCommand(command)
msg = "%s : Command exited with status: %s\n" % (
time.strftime("%Y-%m-%dT%H:%M:%S"), exitCode)
print(msg)
except Exception as ex:
raise StageOutError(str(ex), Command=command, ExitCode=60311)
if exitCode:
msg = "%s : Command exited non-zero" % time.strftime(
"%Y-%m-%dT%H:%M:%S")
print("ERROR: Exception During Stage Out:\n")
print(msg)
raise StageOutError(msg, Command=command, ExitCode=exitCode)
return
def executeCommand(self, command):
"""
_execute_
Execute the command provided, throw a StageOutError if it exits
non zero
"""
try:
exitCode = runCommand(command)
msg = "Command exited with status: %s" % (exitCode)
print msg
except Exception as ex:
raise StageOutError(str(ex), Command=command, ExitCode=60311)
if exitCode in self.directoryErrorCodes:
raise StageOutInvalidPath()
elif exitCode:
msg = "Command exited non-zero"
print "ERROR: Exception During Stage Out:\n"
print msg
raise StageOutError(msg, Command=command, ExitCode=exitCode)
return
def doTransfer(self, fromPfn, toPfn, stageOut, seName, command, options, protocol):
"""
performs a transfer. stageOut tells you which way to go. returns the new pfn or
raises on failure. StageOutError (and inherited exceptions) are for expected errors
such as temporary connection failures. Anything else will be handled as an unexpected
error and skip retrying with this plugin
"""
localFileName = fromPfn
fromPfn = self.prependFileProtocol(fromPfn)
transferCommand = (
"lcg-cp -b -D srmv2 --vo cms --srm-timeout 2400 --sendreceive-timeout 2400 --connect-timeout 300 --verbose %s %s %s "
% (options, fromPfn, toPfn)
)
logging.info("Staging out with lcg-cp")
logging.info(" commandline: %s" % transferCommand)
(exitCode, output) = runCommand(transferCommand)
# riddle me this, the following line fails with:
# not all arguments converted during string formatting
# FIXME
logging.info(" output from lcg-cp: %s" % output)
logging.info(" complete. #") # exit code" is %s" % exitCode)
logging.info("Verifying file sizes")
localSize = os.path.getsize(localFileName)
remoteSize = subprocess.Popen(
["lcg-ls", "-l", "-b", "-D", "srmv2", toPfn], stdout=subprocess.PIPE
).communicate()[0]
logging.info("got the following from lcg-ls %s" % remoteSize)
remoteSize = remoteSize.split()[4]
logging.info("Localsize: %s Remotesize: %s" % (localSize, remoteSize))
if int(localSize) != int(remoteSize):
try:
self.doDelete(toPfn, None, None, None, None)
except:
pass
raise StageOutFailure, "File sizes don't match"
return toPfn
def executeCommand(self, command):
"""
_execute_
Execute the command provided, throw a StageOutError if it exits
non zero
"""
try:
exitCode = runCommand(command)
msg = "Command exited with status: %s" % (exitCode)
print msg
except Exception as ex:
raise StageOutError(str(ex), Command = command, ExitCode = 60311)
if exitCode in self.directoryErrorCodes:
raise StageOutInvalidPath()
elif exitCode:
msg = "Command exited non-zero"
print "ERROR: Exception During Stage Out:\n"
print msg
raise StageOutError(msg, Command = command, ExitCode = exitCode)
return
def testRunCommandWithOutput_error(self, mock_sys):
runCommand("python %s -exit 0" % self.base)
mock_sys.stdout.write.assert_called_with("")
mock_sys.stderr.write.assert_any_call("")
mock_sys.stderr.write.assert_any_call("0\n")
def testRunCommandWithOutput_noError(self, mock_sys):
runCommand("python %s -text a" % self.base)
mock_sys.stdout.write.assert_any_call("a\n")
mock_sys.stderr.write.assert_any_call("")
def testRunCommandWithOutput_errorAndText(self, mock_sys):
runCommand("python %s -exception test" % self.base)
assert "Exception: test\n" in mock_sys.stderr.write.call_args_list[0][
0][0]
mock_sys.stdout.write.assert_any_call("test\n")
def testRunCommandWithOutput_noError(self, mock_sys):
runCommand("python %s -text a" % self.base)
mock_sys.stdout.write.assert_any_call("a\n")
mock_sys.stderr.write.assert_any_call("")
def testRunCommandWithOutput_error(self, mock_sys):
runCommand("python %s -exit 0" % self.base)
mock_sys.stdout.write.assert_called_with("")
mock_sys.stderr.write.assert_any_call("")
mock_sys.stderr.write.assert_any_call("0\n")
def testRunCommand_results(self):
self.assertEqual(1, runCommand("python %s -exit 0" % self.base))
self.assertEqual(1, runCommand("python %s -exit a" % self.base))
self.assertEqual(2, runCommand("python %s -text" % self.base))
self.assertEqual(0, runCommand("python %s -text a" % self.base))
def testRunCommandWithOutput_errorAndText(self, mock_sys):
runCommand("python %s -exception test" % self.base)
assert "Exception: test\n" in mock_sys.stderr.write.call_args_list[0][0][0]
mock_sys.stdout.write.assert_any_call("test\n")
def runCommandWarnOnNonZero(self, command):
(exitCode, output) = runCommand(command)
if not exitCode:
logging.error("Error in file transfer..ignoring:")
logging.error(output)
return (exitCode, output)
def testRunCommand_results(self):
self.assertEqual(1, runCommand("python %s -exit 0" % self.base))
self.assertEqual(1, runCommand("python %s -exit a" % self.base))
self.assertEqual(2, runCommand("python %s -text" % self.base))
self.assertEqual(0, runCommand("python %s -text a" % self.base))
def doWrapped(self, commandArgs):
wrapperPath = os.path.join(getWMBASE(), 'src', 'python', 'WMCore',
'Storage', 'Plugins', 'XRDCP', 'wrapenv.sh')
commandArgs.insert(0, wrapperPath)
return runCommand(commandArgs)
def doWrapped(self, commandArgs):
wrapperPath = os.path.join(getWMBASE(),'src','python','WMCore','Storage','Plugins','XRDCP','wrapenv.sh')
commandArgs.insert(0,wrapperPath)
return runCommand(commandArgs)
