def createRemoveFileCommand(self, pfn):
"""
handle both srm and file pfn types
"""
if pfn.startswith("srm://"):
return "srmrm %s" % pfn
elif pfn.startswith("file:"):
return "/bin/rm -f %s" % pfn.replace("file://", "", 1)
else:
return StageOutImpl.createRemoveFileCommand(self, pfn)
def createRemoveFileCommand(self, pfn):
"""
handle both srm and file pfn types
"""
if pfn.startswith("srm://"):
return "srm-advisory-delete %s" % pfn
elif pfn.startswith("file:"):
return "/bin/rm -f %s" % pfn.replace("file://", "", 1)
else:
return StageOutImpl.createRemoveFileCommand(self, pfn)
def createRemoveFileCommand(self, pfn):
"""
handle both srm and file pfn types
"""
if pfn.startswith("srm://"):
return "%s lcg-del -b -l -D srmv2 --vo cms %s" % (self.setups, pfn)
elif pfn.startswith("file:"):
return "/bin/rm -f %s" % pfn.replace("file:", "", 1)
else:
return StageOutImpl.createRemoveFileCommand(self, pfn)
def createRemoveFileCommand(self, pfn):
"""
handle both srm and file pfn types
"""
if pfn.startswith("srm://"):
return "%s lcg-del -b -l -D srmv2 --vo cms %s" % (self.setups, pfn)
elif pfn.startswith("file:"):
return "/bin/rm -f %s" % pfn.replace("file:", "", 1)
else:
return StageOutImpl.createRemoveFileCommand(self, pfn)
def doDelete(self, pfn, seName, command, options, protocol ):
"""
deletes a file, raises on error
StageOutError (and inherited exceptions) are for expected errors
such as temporary connection failures. Anything else will be handled as an unexpected
error and skip retrying with this plugin
"""
if pfn.startswith("srm://"):
runCommand( "lcg-del -b -l -D srmv2 --vo cms %s" % pfn )
elif pfn.startswith("file:"):
runCommand( "/bin/rm -f %s" % pfn.replace("file:", "", 1) )
else:
runCommand( StageOutImpl.createRemoveFileCommand(self, pfn) )
def createRemoveFileCommand(self, pfn):
"""
_createRemoveFileCommand_
Alternate between EOS, CASTOR and local.
"""
if self.isEOS(pfn):
return "xrd eoscms rm %s" % pfn.replace("root://eoscms//eos/cms/", "/eos/cms/", 1)
try:
simplePFN = self.parseCastorPath(pfn)
return "stager_rm -a -M %s ; nsrm %s" % (simplePFN, simplePFN)
except StageOutError:
# Not castor
pass
return StageOutImpl.createRemoveFileCommand(self, pfn)
def createRemoveFileCommand(self, pfn):
"""
_createRemoveFileCommand_
Alternate between EOS, CASTOR and local.
"""
if self.isEOS(pfn):
(_,host,path,_) = self.splitPFN(pfn)
return "xrd %s rm %s" % (host, path)
try:
simplePFN = self.parseCastorPath(pfn)
return "stager_rm -a -M %s ; nsrm %s" % (simplePFN, simplePFN)
except StageOutError:
# Not castor
pass
return StageOutImpl.createRemoveFileCommand(self, pfn)
def createRemoveFileCommand(self, pfn):
"""
_createRemoveFileCommand_
Alternate between EOS, CASTOR and local.
"""
if self.isEOS(pfn):
(_,host,path,_) = self.splitPFN(pfn)
return "xrd %s rm %s" % (host, path)
try:
simplePFN = self.parseCastorPath(pfn)
return "stager_rm -a -M %s ; nsrm %s" % (simplePFN, simplePFN)
except StageOutError:
# Not castor
pass
return StageOutImpl.createRemoveFileCommand(self, pfn)
def createRemoveFileCommand(self, pfn):
"""
_createRemoveFileCommand_
Alternate between EOS, CASTOR and local.
"""
if self.isEOS(pfn):
return "xrd eoscms rm %s" % pfn.replace("root://eoscms//eos/cms/",
"/eos/cms/", 1)
try:
simplePFN = self.parseCastorPath(pfn)
return "stager_rm -a -M %s ; nsrm %s" % (simplePFN, simplePFN)
except StageOutError:
# Not castor
pass
return StageOutImpl.createRemoveFileCommand(self, pfn)
def createRemoveFileCommand(self, pfn):
"""
handle both srm and file pfn types
lcg-del options used:
-b don't make BDII calls to get SE type. SURLs must be fully provided
-l it means that the SURL is not registered in any LFC server, don't connect to the LFC server
-D specifies the default SE type we want to use
--srm-timeout Sets the SRM timeout, define the maximum waiting time for a SRM query.
The request will be aborted if it is still queued after srmtimeout seconds.
Default to 180 seconds.
--sendreceive-timeout Sets the send/receive timeout, maximum total time for the operation.
The operation will be aborted if the operation is not finished before X seconds.
Default to 3600 seconds.
--connect-timeout The connection will be aborted if the remote host doesn't
reply before X seconds. Default to 60 seconds.
"""
if pfn.startswith("srm://"):
return "%s lcg-del -b -l -D srmv2 %s --vo cms %s" % (self.setups, self.timeoutOptions, pfn)
elif pfn.startswith("file:"):
return "/bin/rm -f %s" % pfn.replace("file:", "", 1)
else:
return StageOutImpl.createRemoveFileCommand(self, pfn)
def createRemoveFileCommand(self, pfn):
"""
handle both srm and file pfn types
lcg-del options used:
-b don't make BDII calls to get SE type. SURLs must be fully provided
-l it means that the SURL is not registered in any LFC server, don't connect to the LFC server
-D specifies the default SE type we want to use
--srm-timeout Sets the SRM timeout, define the maximum waiting time for a SRM query.
The request will be aborted if it is still queued after srmtimeout seconds.
Default to 180 seconds.
--sendreceive-timeout Sets the send/receive timeout, maximum total time for the operation.
The operation will be aborted if the operation is not finished before X seconds.
Default to 3600 seconds.
--connect-timeout The connection will be aborted if the remote host doesn't
reply before X seconds. Default to 60 seconds.
"""
if pfn.startswith("srm://"):
return "%s lcg-del -b -l -D srmv2 %s --vo cms %s" % (
self.setups, self.timeoutOptions, pfn)
elif pfn.startswith("file:"):
return "/bin/rm -f %s" % pfn.replace("file:", "", 1)
else:
return StageOutImpl.createRemoveFileCommand(self, pfn)
class StageOutImplTest(unittest.TestCase):
def setUp(self):
self.StageOutImpl = StageOutImpl()
def testSplitPFN_noOpaque(self):
self.assertEqual(
("splitable/test/test1", "test1", "splitable/test/test1", ""),
StageOutImpl.splitPFN("splitable/test/test1"))
self.assertEqual(('//eoscms//eos/cms/store/', 'eoscms',
'//eoscms//eos/cms/store/', ""),
StageOutImpl.splitPFN("//eoscms//eos/cms/store/"))
def testSplitPFN_doubleSlashRoot(self):
self.assertEqual(
('root', 'eoscms', '/eos/cms/store/', ""),
StageOutImpl.splitPFN("root://eoscms//eos/cms/store/"))
def testSplitPFN_path(self):
self.assertEqual(
('root', 'eoscms', 'default', "?"),
StageOutImpl.splitPFN("root://eoscms//eos/cms/store?path=default"))
def testSplitPFN_pathConnector(self):
self.assertEqual(
('root', 'eoscms', 'default', "?default2"),
StageOutImpl.splitPFN(
"root://eoscms//eos/cms/store?path=default&default2"))
def testSplitPFN_path2(self):
self.assertEqual(
('root', 'eoscms', 'cms', "?cms2ss&path=default"),
StageOutImpl.splitPFN(
"root://eoscms//eos/cms/store?path=cms&cms2ss&path=default"))
def testSplitPFN_pathConnector2(self):
self.assertEqual(
('root', 'eoscms', 'cms', "?path=default&default2"),
StageOutImpl.splitPFN(
"root://eoscms//eos/cms/store?path=cms&path=default&default2"))
@mock.patch('WMCore.Storage.StageOutImpl.runCommandWithOutput')
def testExecuteCommand_stageOutError(self, mock_runCommand):
mock_runCommand.side_effect = Exception('BOOM!')
with self.assertRaises(Exception) as context:
self.StageOutImpl.executeCommand("command")
self.assertTrue('ErrorCode : 60311' in context.exception)
mock_runCommand.assert_called_with("command")
@mock.patch('WMCore.Storage.StageOutImpl.runCommandWithOutput')
def testExecuteCommand_exitCode(self, mock_runCommand):
mock_runCommand.return_value = 1, "Test Fake Error"
with self.assertRaises(Exception) as context:
self.StageOutImpl.executeCommand("command")
self.assertTrue('ErrorCode : 1' in context.exception)
mock_runCommand.assert_called_with("command")
@mock.patch('WMCore.Storage.StageOutImpl.runCommandWithOutput')
def testExecuteCommand_valid(self, mock_runCommand):
mock_runCommand.return_value = 0, "Test Success"
self.StageOutImpl.executeCommand("command")
mock_runCommand.assert_called_with("command")
@mock.patch('WMCore.Storage.StageOutImpl.os.path')
def testCreateRemoveFileCommand_isFile(self, mock_path):
mock_path.isfile.return_value = True
mock_path.abspath.return_value = "path"
self.assertEqual(
"/bin/rm -f path",
self.StageOutImpl.createRemoveFileCommand("test/path"))
mock_path.isfile.assert_called_with("test/path")
mock_path.abspath.assert_called_with("test/path")
@mock.patch('WMCore.Storage.StageOutImpl.os.path')
def testCreateRemoveFileCommand_notPath(self, mock_path):
mock_path.isfile.return_value = False
self.assertEqual(
"", self.StageOutImpl.createRemoveFileCommand("test/path"))
mock_path.isfile.assert_called_with("test/path")
def testCreateRemoveFileCommand_startsWithSlash(self):
self.assertEqual(
"/bin/rm -f /test/path",
self.StageOutImpl.createRemoveFileCommand("/test/path"))
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createSourceName')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createTargetName')
@mock.patch(
'WMCore.Storage.StageOutImpl.StageOutImpl.createOutputDirectory')
@mock.patch(
'WMCore.Storage.StageOutImpl.StageOutImpl.createStageOutCommand')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.executeCommand')
def testCallable(self, mock_executeCommand, mock_createStageOutCommand,
mock_createOutputDirectory, mock_createTargetName,
mock_createSourceName):
mock_createSourceName.return_value = "sourcePFN"
mock_createTargetName.return_value = "targetPFN"
mock_createStageOutCommand.return_value = "command"
self.StageOutImpl("protocol", "inputPFN", "targetPFN")
mock_createSourceName.assert_called_with("protocol", "inputPFN")
mock_createTargetName.assert_called_with("protocol", "targetPFN")
mock_createOutputDirectory.assert_called_with("targetPFN")
mock_createStageOutCommand.assert_called_with("sourcePFN", "targetPFN",
None, None)
mock_executeCommand.assert_called_with("command")
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createSourceName')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createTargetName')
@mock.patch(
'WMCore.Storage.StageOutImpl.StageOutImpl.createOutputDirectory')
@mock.patch(
'WMCore.Storage.StageOutImpl.StageOutImpl.createStageOutCommand')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.executeCommand')
@mock.patch('WMCore.Storage.StageOutImpl.time')
def testCallable_StageOutError(self, mock_time, mock_executeCommand,
mock_createStageOutCommand,
mock_createOutputDirectory,
mock_createTargetName,
mock_createSourceName):
mock_createSourceName.return_value = "sourcePFN"
mock_createTargetName.return_value = "targetPFN"
mock_createStageOutCommand.return_value = "command"
mock_createOutputDirectory.side_effect = [
StageOutError("error"),
StageOutError("error"), None
]
mock_executeCommand.side_effect = [
StageOutError("error"),
StageOutError("error"), None
]
self.StageOutImpl("protocol", "inputPFN", "targetPFN")
mock_createSourceName.assert_called_with("protocol", "inputPFN")
mock_createTargetName.assert_called_with("protocol", "targetPFN")
mock_createOutputDirectory.assert_called_with("targetPFN")
mock_createStageOutCommand.assert_called_with("sourcePFN", "targetPFN",
None, None)
mock_executeCommand.assert_called_with("command")
calls = [call(600), call(600), call(600), call(600)]
mock_time.sleep.assert_has_calls(calls)
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createSourceName')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createTargetName')
@mock.patch(
'WMCore.Storage.StageOutImpl.StageOutImpl.createOutputDirectory')
@mock.patch('WMCore.Storage.StageOutImpl.time')
def testCallable_StageOutErrorFail(self, mock_time,
mock_createOutputDirectory,
mock_createTargetName,
mock_createSourceName):
mock_createSourceName.return_value = "sourcePFN"
mock_createTargetName.return_value = "targetPFN"
mock_createOutputDirectory.side_effect = [
StageOutError("error"),
StageOutError("error"),
StageOutError("Last error")
]
with self.assertRaises(Exception) as context:
self.StageOutImpl("protocol", "inputPFN", "targetPFN")
self.assertTrue('Last error' in context.exception)
mock_createSourceName.assert_called_with("protocol", "inputPFN")
mock_createTargetName.assert_called_with("protocol", "targetPFN")
mock_createOutputDirectory.assert_called_with("targetPFN")
calls = [call(600), call(600)]
mock_time.sleep.assert_has_calls(calls)
class StageOutImplTest(unittest.TestCase):
def setUp(self):
self.StageOutImpl = StageOutImpl()
def testSplitPFN_noOpaque(self):
self.assertEqual(("splitable/test/test1", "test1", "splitable/test/test1", ""),
StageOutImpl.splitPFN("splitable/test/test1"))
self.assertEqual(('//eoscms//eos/cms/store/', 'eoscms', '//eoscms//eos/cms/store/', ""),
StageOutImpl.splitPFN("//eoscms//eos/cms/store/"))
def testSplitPFN_doubleSlashRoot(self):
self.assertEqual(('root', 'eoscms', '/eos/cms/store/', ""),
StageOutImpl.splitPFN("root://eoscms//eos/cms/store/"))
def testSplitPFN_path(self):
self.assertEqual(('root', 'eoscms', 'default', "?"),
StageOutImpl.splitPFN("root://eoscms//eos/cms/store?path=default"))
def testSplitPFN_pathConnector(self):
self.assertEqual(('root', 'eoscms', 'default', "?default2"),
StageOutImpl.splitPFN("root://eoscms//eos/cms/store?path=default&default2"))
def testSplitPFN_path2(self):
self.assertEqual(('root', 'eoscms', 'cms', "?cms2ss&path=default"),
StageOutImpl.splitPFN("root://eoscms//eos/cms/store?path=cms&cms2ss&path=default"))
def testSplitPFN_pathConnector2(self):
self.assertEqual(('root', 'eoscms', 'cms', "?path=default&default2"),
StageOutImpl.splitPFN("root://eoscms//eos/cms/store?path=cms&path=default&default2"))
@mock.patch('WMCore.Storage.StageOutImpl.runCommand')
def testExecuteCommand_stageOutError(self, mock_runCommand):
mock_runCommand.side_effect = Exception('BOOM!')
with self.assertRaises(Exception) as context:
self.StageOutImpl.executeCommand("command")
self.assertTrue('ErrorCode : 60311' in context.exception)
mock_runCommand.assert_called_with("command")
@mock.patch('WMCore.Storage.StageOutImpl.runCommand')
def testExecuteCommand_exitCode(self, mock_runCommand):
mock_runCommand.return_value = 1
with self.assertRaises(Exception) as context:
self.StageOutImpl.executeCommand("command")
self.assertTrue('ErrorCode : 1' in context.exception)
mock_runCommand.assert_called_with("command")
@mock.patch('WMCore.Storage.StageOutImpl.runCommand')
def testExecuteCommand_valid(self, mock_runCommand):
mock_runCommand.return_value = 0
self.StageOutImpl.executeCommand("command")
mock_runCommand.assert_called_with("command")
@mock.patch('WMCore.Storage.StageOutImpl.os.path')
def testCreateRemoveFileCommand_isFile(self, mock_path):
mock_path.isfile.return_value = True
mock_path.abspath.return_value = "path"
self.assertEqual("/bin/rm -f path", self.StageOutImpl.createRemoveFileCommand("test/path"))
mock_path.isfile.assert_called_with("test/path")
mock_path.abspath.assert_called_with("test/path")
@mock.patch('WMCore.Storage.StageOutImpl.os.path')
def testCreateRemoveFileCommand_notPath(self, mock_path):
mock_path.isfile.return_value = False
self.assertEqual("", self.StageOutImpl.createRemoveFileCommand("test/path"))
mock_path.isfile.assert_called_with("test/path")
def testCreateRemoveFileCommand_startsWithSlash(self):
self.assertEqual("/bin/rm -f /test/path", self.StageOutImpl.createRemoveFileCommand("/test/path"))
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createSourceName')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createTargetName')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createOutputDirectory')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createStageOutCommand')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.executeCommand')
def testCallable(self, mock_executeCommand, mock_createStageOutCommand, mock_createOutputDirectory,
mock_createTargetName, mock_createSourceName):
mock_createSourceName.return_value = "sourcePFN"
mock_createTargetName.return_value = "targetPFN"
mock_createStageOutCommand.return_value = "command"
self.StageOutImpl("protocol", "inputPFN", "targetPFN")
mock_createSourceName.assert_called_with("protocol", "inputPFN")
mock_createTargetName.assert_called_with("protocol", "targetPFN")
mock_createOutputDirectory.assert_called_with("targetPFN")
mock_createStageOutCommand.assert_called_with("sourcePFN", "targetPFN", None, None)
mock_executeCommand.assert_called_with("command")
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createSourceName')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createTargetName')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createOutputDirectory')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createStageOutCommand')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.executeCommand')
@mock.patch('WMCore.Storage.StageOutImpl.time')
def testCallable_StageOutError(self, mock_time, mock_executeCommand, mock_createStageOutCommand,
mock_createOutputDirectory, mock_createTargetName, mock_createSourceName):
mock_createSourceName.return_value = "sourcePFN"
mock_createTargetName.return_value = "targetPFN"
mock_createStageOutCommand.return_value = "command"
mock_createOutputDirectory.side_effect = [StageOutError("error"), StageOutError("error"), None]
mock_executeCommand.side_effect = [StageOutError("error"), StageOutError("error"), None]
self.StageOutImpl("protocol", "inputPFN", "targetPFN")
mock_createSourceName.assert_called_with("protocol", "inputPFN")
mock_createTargetName.assert_called_with("protocol", "targetPFN")
mock_createOutputDirectory.assert_called_with("targetPFN")
mock_createStageOutCommand.assert_called_with("sourcePFN", "targetPFN", None, None)
mock_executeCommand.assert_called_with("command")
calls = [call(600), call(600), call(600), call(600)]
mock_time.sleep.assert_has_calls(calls)
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createSourceName')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createTargetName')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createOutputDirectory')
@mock.patch('WMCore.Storage.StageOutImpl.time')
def testCallable_StageOutErrorFail(self, mock_time, mock_createOutputDirectory, mock_createTargetName,
mock_createSourceName):
mock_createSourceName.return_value = "sourcePFN"
mock_createTargetName.return_value = "targetPFN"
mock_createOutputDirectory.side_effect = [StageOutError("error"), StageOutError("error"),
StageOutError("Last error")]
with self.assertRaises(Exception) as context:
self.StageOutImpl("protocol", "inputPFN", "targetPFN")
self.assertTrue('Last error' in context.exception)
mock_createSourceName.assert_called_with("protocol", "inputPFN")
mock_createTargetName.assert_called_with("protocol", "targetPFN")
mock_createOutputDirectory.assert_called_with("targetPFN")
calls = [call(600), call(600)]
mock_time.sleep.assert_has_calls(calls)
