def setUp(self):
self.RFCPCERNImpl = RFCPCERNImpl()
def setUp(self):
self.RFCPCERNImpl = RFCPCERNImpl()
class RFCPCERNImplTest(unittest.TestCase):
def setUp(self):
self.RFCPCERNImpl = RFCPCERNImpl()
def testCreateSourceName(self):
self.assertEqual("name", self.RFCPCERNImpl.createSourceName("", "name"))
self.assertEqual("file:////name", self.RFCPCERNImpl.createSourceName("", "file:////name"))
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.checkDirExists')
def testCreateOutputDirectory_isEOS(self, mock_checkDirExists, mock_isEOS):
mock_isEOS.return_value = True
self.RFCPCERNImpl.createOutputDirectory("name")
mock_checkDirExists.assert_not_called()
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.checkDirExists')
def testCreateOutputDirectory_stageIn(self, mock_checkDirExists, mock_isEOS):
mock_isEOS.return_value = False
self.RFCPCERNImpl.stageIn = True
self.RFCPCERNImpl.createOutputDirectory("name")
mock_checkDirExists.assert_not_called()
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.checkDirExists')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.parseCastorPath')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.os.environ')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.createDir')
def testCreateOutputDirectory_environ(self, mock_createDir, mock_environ, mock_parseCastorPath, mock_checkDirExists,
mock_isEOS):
mock_environ.get.return_value = "t0export2"
mock_isEOS.return_value = False
mock_checkDirExists.return_value = False
mock_parseCastorPath.return_value = "path/testPath/file"
self.RFCPCERNImpl.createOutputDirectory("name")
mock_parseCastorPath.assert_called_once_with("name")
mock_checkDirExists.assert_called_once_with("path/testPath")
mock_createDir.assert_called_once_with("path/testPath")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.checkDirExists')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.parseCastorPath')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.os.environ')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.createDir')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.setFileClass')
def testCreateOutputDirectory_t0export(self, mock_setFileClass, mock_createDir, mock_environ, mock_parseCastorPath,
mock_checkDirExists, mock_isEOS):
mock_environ.get.return_value = "t0export"
mock_isEOS.return_value = False
mock_checkDirExists.return_value = False
mock_parseCastorPath.return_value = "/castor/cern.ch/cms/store/aa1data/aa2/aa3/aa4/aa5/file"
self.RFCPCERNImpl.createOutputDirectory("name")
mock_parseCastorPath.assert_called_once_with("name")
mock_checkDirExists.assert_has_calls([call("/castor/cern.ch/cms/store/aa1data/aa2/aa3/aa4/aa5"),
call("/castor/cern.ch/cms/store/aa1data/aa2/aa3/aa4")])
mock_createDir.assert_has_calls([call("/castor/cern.ch/cms/store/aa1data/aa2/aa3/aa4"),
call("/castor/cern.ch/cms/store/aa1data/aa2/aa3/aa4/aa5")])
mock_setFileClass.assert_called_once_with("/castor/cern.ch/cms/store/aa1data/aa2/aa3/aa4", "cms_production")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.checkDirExists')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.parseCastorPath')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.os.environ')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.createDir')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.setFileClass')
def testCreateOutputDirectory_RAW(self, mock_setFileClass, mock_createDir, mock_environ, mock_parseCastorPath,
mock_checkDirExists, mock_isEOS):
mock_environ.get.return_value = "t0export"
mock_isEOS.return_value = False
mock_checkDirExists.return_value = False
mock_parseCastorPath.return_value = "/castor/cern.ch/cms/store/aa1data/aa2/aa3/RAW/aa5/file"
self.RFCPCERNImpl.createOutputDirectory("name")
mock_parseCastorPath.assert_called_once_with("name")
mock_checkDirExists.assert_has_calls([call("/castor/cern.ch/cms/store/aa1data/aa2/aa3/RAW/aa5"),
call("/castor/cern.ch/cms/store/aa1data/aa2/aa3/RAW")])
mock_createDir.assert_has_calls([call("/castor/cern.ch/cms/store/aa1data/aa2/aa3/RAW"),
call("/castor/cern.ch/cms/store/aa1data/aa2/aa3/RAW/aa5")])
mock_setFileClass.assert_called_once_with("/castor/cern.ch/cms/store/aa1data/aa2/aa3/RAW", "cms_raw")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.createRemoveFileCommand')
def testCreateStageOutCommand_stageIn(self, mock_createRemoveFileCommand, mock_isEOS):
mock_createRemoveFileCommand.return_value = "remove file command"
mock_isEOS.return_value = False
self.RFCPCERNImpl.stageIn = True
result = self.RFCPCERNImpl.createStageOutCommand("sourcePFN", "targetPFN")
expectedResult = self.getStageOutCommandResult("sourcePFN", "targetPFN", True, False, "remove file command",
False, False, False)
mock_createRemoveFileCommand.assert_called_once_with("targetPFN")
self.assertEqual(expectedResult, result)
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.splitPFN')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.createRemoveFileCommand')
def testCreateStageOutCommand_isEOS(self, mock_createRemoveFileCommand, mock_isEOS, mock_splitPFN):
mock_createRemoveFileCommand.return_value = "remove file command"
mock_isEOS.return_value = True
mock_splitPFN.return_value = (None, "host", "path", None)
result = self.RFCPCERNImpl.createStageOutCommand("sourcePFN", "targetPFN")
expectedResult = self.getStageOutCommandResult("sourcePFN", "targetPFN", False, True, "remove file command",
"host", "path", False)
mock_createRemoveFileCommand.assert_called_once_with("targetPFN")
self.assertEqual(expectedResult, result)
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.splitPFN')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.createRemoveFileCommand')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.parseCastorPath')
def testCreateStageOutCommand_isEOSChecksums(self, mock_parseCastorPath, mock_createRemoveFileCommand, mock_isEOS,
mock_splitPFN):
mock_createRemoveFileCommand.return_value = "remove file command"
mock_isEOS.return_value = True
mock_parseCastorPath.return_value = "targetFile"
mock_splitPFN.return_value = (None, "host", "path", None)
result = self.RFCPCERNImpl.createStageOutCommand("sourcePFN", "targetPFN", checksums={"adler32": "32"})
expectedResult = self.getStageOutCommandResult("sourcePFN", "targetPFN", False, True, "remove file command",
"host", "path", "targetFile", useChecksum="00000032")
mock_createRemoveFileCommand.assert_called_once_with("targetPFN")
self.assertEqual(expectedResult, result)
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.splitPFN')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.createRemoveFileCommand')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.parseCastorPath')
def testCreateStageOutCommand_checksumsOptions(self, mock_parseCastorPath, mock_createRemoveFileCommand, mock_isEOS,
mock_splitPFN):
mock_createRemoveFileCommand.return_value = "remove file command"
mock_isEOS.return_value = False
mock_parseCastorPath.return_value = "targetFile"
mock_splitPFN.return_value = (None, "host", "path", None)
result = self.RFCPCERNImpl.createStageOutCommand("sourcePFN", "targetPFN", options="options",
checksums={"adler32": "32"})
expectedResult = self.getStageOutCommandResult("sourcePFN", "targetPFN", False, False, "remove file command",
"host", "path", "targetFile", options="options",
useChecksum="32")
mock_createRemoveFileCommand.assert_called_once_with("targetPFN")
self.assertEqual(expectedResult, result)
def getStageOutCommandResult(self, sourcePFN, targetPFN, stageIn, isEOS, removeCommand, host, path, targetFile,
options=None, useChecksum=None):
result = ""
if stageIn:
remotePFN, localPFN = sourcePFN, targetPFN
else:
remotePFN, localPFN = targetPFN, sourcePFN
result += "LOCAL_SIZE=`stat -c%%s \"%s\"`\n" % localPFN
result += "echo \"Local File Size is: $LOCAL_SIZE\"\n"
if isEOS:
result += "xrdcp -f -N "
if useChecksum:
targetPFN += "?eos.targetsize=$LOCAL_SIZE&eos.checksum=%s" % useChecksum
else:
if useChecksum:
result += "nstouch %s\n" % targetFile
result += "nssetchecksum -n adler32 -k %s %s\n" % (useChecksum, targetFile)
result += "rfcp "
if options != None:
result += " %s " % options
result += " \"%s\" " % sourcePFN
result += " \"%s\" \n" % targetPFN
if stageIn:
result += "LOCAL_SIZE=`stat -c%%s \"%s\"`\n" % localPFN
result += "echo \"Local File Size is: $LOCAL_SIZE\"\n"
if isEOS:
result += "REMOTE_SIZE=`xrd '%s' stat '%s' | sed -r 's/.* Size: ([0-9]+) .*/\\1/'`\n" % (host, path)
result += "echo \"Remote File Size is: $REMOTE_SIZE\"\n"
if useChecksum:
result += "echo \"Local File Checksum is: %s\"\n" % useChecksum
result += "REMOTE_XS=`xrd '%s' getchecksum '%s' | sed -r 's/.* adler32 ([0-9a-fA-F]{8}).*/\\1/'`\n" % (
host, path)
result += "echo \"Remote File Checksum is: $REMOTE_XS\"\n"
result += "if [ $REMOTE_SIZE ] && [ $REMOTE_XS ] && [ $LOCAL_SIZE == $REMOTE_SIZE ] &&" \
" [ '%s' == $REMOTE_XS ]; then exit 0; " % useChecksum
result += "else echo \"Error: Size or Checksum Mismatch between local and SE\";" \
" %s ; exit 60311 ; fi" % removeCommand
else:
result += "if [ $REMOTE_SIZE ] && [ $LOCAL_SIZE == $REMOTE_SIZE ]; then exit 0; "
result += "else echo \"Error: Size Mismatch between local and SE\"; %s ; exit 60311 ; fi" % removeCommand
else:
result += "REMOTE_SIZE=`rfstat '%s' | grep Size | cut -f2 -d: | tr -d ' '`\n" % remotePFN
result += "echo \"Remote File Size is: $REMOTE_SIZE\"\n"
result += "if [ $REMOTE_SIZE ] && [ $LOCAL_SIZE == $REMOTE_SIZE ]; then exit 0; else echo " \
"\"Error: Size Mismatch between local and SE\"; %s ; exit 60311 ; fi" % removeCommand
return result
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.splitPFN')
def testCreateRemoveFileCommand_isEos(self, mock_splitPFN, mock_isEOS):
mock_isEOS.return_value = True
mock_splitPFN.return_value = (None, "host", "path", None)
self.assertEqual("xrd host rm path", self.RFCPCERNImpl.createRemoveFileCommand("file/path"))
mock_isEOS.assert_called_once_with("file/path")
mock_splitPFN.assert_called_once_with("file/path")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.parseCastorPath')
def testCreateRemoveFileCommand_simplePFN(self, mock_parseCastorPath, mock_isEOS):
mock_isEOS.return_value = False
mock_parseCastorPath.return_value = "simplePFN"
self.assertEqual("stager_rm -a -M simplePFN ; nsrm simplePFN",
self.RFCPCERNImpl.createRemoveFileCommand("file/path"))
mock_isEOS.assert_called_once_with("file/path")
mock_parseCastorPath.assert_called_once_with("file/path")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.parseCastorPath')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.StageOutImpl.createRemoveFileCommand')
def testCreateRemoveFileCommand_StageOutError(self, mock_createRemoveFileCommand, mock_parseCastorPath, mock_isEOS):
mock_isEOS.return_value = False
mock_createRemoveFileCommand.return_value = "test"
mock_parseCastorPath.side_effect = StageOutError("Errors!")
self.assertEqual("test", self.RFCPCERNImpl.createRemoveFileCommand("file/path"))
mock_isEOS.assert_called_once_with("file/path")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.splitPFN')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.execute')
def testRemoveFile_isEOS(self, mock_executeCommand, mock_splitPFN, mock_isEOS):
mock_isEOS.return_value = True
mock_splitPFN.return_value = (None, "host", "path", None)
self.RFCPCERNImpl.removeFile("file/path")
mock_executeCommand.assert_called_with("xrd host rm path")
mock_isEOS.assert_called_with("file/path")
mock_splitPFN.assert_called_with("file/path")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.parseCastorPath')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.execute')
def testRemoveFile_simplePFN(self, mock_executeCommand, mock_parseCastorPath, mock_isEOS):
mock_isEOS.return_value = False
mock_parseCastorPath.return_value = "simplePFN"
self.RFCPCERNImpl.removeFile("file/path")
mock_executeCommand.assert_called_with("stager_rm -a -M simplePFN ; nsrm simplePFN")
mock_isEOS.assert_called_with("file/path")
mock_parseCastorPath.assert_called_with("file/path")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.runCommandWithOutput')
def testCheckDirExists_true(self, mock_runCommandWithOutput):
mock_runCommandWithOutput.return_value = (0, "Protection : d test")
self.assertTrue(self.RFCPCERNImpl.checkDirExists("file/path"))
mock_runCommandWithOutput.assert_called_with("rfstat file/path 2> /dev/null | grep Protection")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.runCommandWithOutput')
def testCheckDirExists_false(self, mock_runCommandWithOutput):
mock_runCommandWithOutput.return_value = (1, "Protection : d test")
self.assertFalse(self.RFCPCERNImpl.checkDirExists("file"))
mock_runCommandWithOutput.assert_called_with("rfstat file 2> /dev/null | grep Protection")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.runCommandWithOutput')
def testCheckDirExists_trueStageOutError(self, mock_runCommandWithOutput):
mock_runCommandWithOutput.return_value = (0, "Protection:test")
with self.assertRaises(StageOutError) as context:
self.RFCPCERNImpl.checkDirExists("file/test")
self.assertTrue('Output path is not a directory !' in context.exception)
mock_runCommandWithOutput.assert_called_with("rfstat file/test 2> /dev/null | grep Protection")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.runCommandWithOutput')
def testCheckDirExists_stageOutError(self, mock_runCommandWithOutput):
mock_runCommandWithOutput.return_value = StageOutError("Error")
with self.assertRaises(StageOutError) as context:
self.RFCPCERNImpl.checkDirExists("test2")
self.assertTrue('Error: Exception while invoking command:' in context.exception)
mock_runCommandWithOutput.assert_called_with("rfstat test2 2> /dev/null | grep Protection")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.execute')
def testCreateDir(self, mock_execute):
self.RFCPCERNImpl.createDir("dirName")
mock_execute.assert_called_with("nsmkdir -p \"dirName\"")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.execute')
def testSetFileClass(self, mock_execute):
self.RFCPCERNImpl.setFileClass("dir", "fileClass")
mock_execute.assert_called_with("nschclass fileClass dir")
def testParseCastorPath_stageOutError(self):
with self.assertRaises(StageOutError) as context:
self.RFCPCERNImpl.parseCastorPath("test2")
self.assertTrue("Can't parse castor path out of URL !" in context.exception)
def testParseCastorPath_xrootd(self):
self.assertEqual("/castor/cern.ch/testas",
self.RFCPCERNImpl.parseCastorPath("root:test//castor/cern.ch/testas"))
self.assertEqual("/castor/cern.ch/testas",
self.RFCPCERNImpl.parseCastorPath("root:test/castor/cern.ch/testas?as"))
self.assertEqual("/castor/cern.ch/testas/test",
self.RFCPCERNImpl.parseCastorPath("root://castor/cern.ch/testas//test?as"))
def testParseCastorPath_rfio(self):
self.assertEqual("/castor/cern.ch/a/a/",
self.RFCPCERNImpl.parseCastorPath("rfio:test//castor/cern.ch/a/a//?testas"))
self.assertEqual("/castor/cern.ch/testas",
self.RFCPCERNImpl.parseCastorPath("rfio:test//castor/cern.ch/testas?as"))
self.assertEqual("/castor/cern.ch/testas/test",
self.RFCPCERNImpl.parseCastorPath("rfio:test/castor/cern.ch/testas//test?as"))
def testParseCastorPath_castor(self):
self.assertEqual("/castor/cern.ch/a/a/?testas",
self.RFCPCERNImpl.parseCastorPath("//castor/cern.ch/a/a//?testas"))
self.assertEqual("/castor/cern.ch/testas?as", self.RFCPCERNImpl.parseCastorPath("//castor/cern.ch/testas?as"))
self.assertEqual("/castor/cern.ch/testas/test",
self.RFCPCERNImpl.parseCastorPath("/castor/cern.ch/testas//test"))
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.splitPFN')
def testIsEOS_rootFalse(self, mock_splitPFN):
mock_splitPFN.return_value = ("protocol", "host", None, None)
self.assertFalse(self.RFCPCERNImpl.isEOS("test"))
mock_splitPFN.assert_called_with("test")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.splitPFN')
def testIsEOS_rootTrue(self, mock_splitPFN):
mock_splitPFN.return_value = ("root", "host", None, None)
self.assertTrue(self.RFCPCERNImpl.isEOS("test"))
mock_splitPFN.assert_called_with("test")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.splitPFN')
def testIsEOS_rootCastorFalse(self, mock_splitPFN):
mock_splitPFN.return_value = ("root", "castorhost", None, None)
self.assertFalse(self.RFCPCERNImpl.isEOS("test"))
mock_splitPFN.assert_called_with("test")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.splitPFN')
def testIsEOS_rootCastorTrue(self, mock_splitPFN):
mock_splitPFN.return_value = ("root", "host:castor", None, None)
self.assertTrue(self.RFCPCERNImpl.isEOS("test"))
mock_splitPFN.assert_called_with("test")
class RFCPCERNImplTest(unittest.TestCase):
def setUp(self):
self.RFCPCERNImpl = RFCPCERNImpl()
def testCreateSourceName(self):
self.assertEqual("name",
self.RFCPCERNImpl.createSourceName("", "name"))
self.assertEqual(
"file:////name",
self.RFCPCERNImpl.createSourceName("", "file:////name"))
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch(
'WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.checkDirExists')
def testCreateOutputDirectory_isEOS(self, mock_checkDirExists, mock_isEOS):
mock_isEOS.return_value = True
self.RFCPCERNImpl.createOutputDirectory("name")
mock_checkDirExists.assert_not_called()
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch(
'WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.checkDirExists')
def testCreateOutputDirectory_stageIn(self, mock_checkDirExists,
mock_isEOS):
mock_isEOS.return_value = False
self.RFCPCERNImpl.stageIn = True
self.RFCPCERNImpl.createOutputDirectory("name")
mock_checkDirExists.assert_not_called()
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch(
'WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.checkDirExists')
@mock.patch(
'WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.parseCastorPath')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.os.environ')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.createDir')
def testCreateOutputDirectory_environ(self, mock_createDir, mock_environ,
mock_parseCastorPath,
mock_checkDirExists, mock_isEOS):
mock_environ.get.return_value = "t0export2"
mock_isEOS.return_value = False
mock_checkDirExists.return_value = False
mock_parseCastorPath.return_value = "path/testPath/file"
self.RFCPCERNImpl.createOutputDirectory("name")
mock_parseCastorPath.assert_called_once_with("name")
mock_checkDirExists.assert_called_once_with("path/testPath")
mock_createDir.assert_called_once_with("path/testPath")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch(
'WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.checkDirExists')
@mock.patch(
'WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.parseCastorPath')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.os.environ')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.createDir')
@mock.patch(
'WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.setFileClass')
def testCreateOutputDirectory_t0export(self, mock_setFileClass,
mock_createDir, mock_environ,
mock_parseCastorPath,
mock_checkDirExists, mock_isEOS):
mock_environ.get.return_value = "t0export"
mock_isEOS.return_value = False
mock_checkDirExists.return_value = False
mock_parseCastorPath.return_value = "/castor/cern.ch/cms/store/aa1data/aa2/aa3/aa4/aa5/file"
self.RFCPCERNImpl.createOutputDirectory("name")
mock_parseCastorPath.assert_called_once_with("name")
mock_checkDirExists.assert_has_calls([
call("/castor/cern.ch/cms/store/aa1data/aa2/aa3/aa4/aa5"),
call("/castor/cern.ch/cms/store/aa1data/aa2/aa3/aa4")
])
mock_createDir.assert_has_calls([
call("/castor/cern.ch/cms/store/aa1data/aa2/aa3/aa4"),
call("/castor/cern.ch/cms/store/aa1data/aa2/aa3/aa4/aa5")
])
mock_setFileClass.assert_called_once_with(
"/castor/cern.ch/cms/store/aa1data/aa2/aa3/aa4", "cms_production")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch(
'WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.checkDirExists')
@mock.patch(
'WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.parseCastorPath')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.os.environ')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.createDir')
@mock.patch(
'WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.setFileClass')
def testCreateOutputDirectory_RAW(self, mock_setFileClass, mock_createDir,
mock_environ, mock_parseCastorPath,
mock_checkDirExists, mock_isEOS):
mock_environ.get.return_value = "t0export"
mock_isEOS.return_value = False
mock_checkDirExists.return_value = False
mock_parseCastorPath.return_value = "/castor/cern.ch/cms/store/aa1data/aa2/aa3/RAW/aa5/file"
self.RFCPCERNImpl.createOutputDirectory("name")
mock_parseCastorPath.assert_called_once_with("name")
mock_checkDirExists.assert_has_calls([
call("/castor/cern.ch/cms/store/aa1data/aa2/aa3/RAW/aa5"),
call("/castor/cern.ch/cms/store/aa1data/aa2/aa3/RAW")
])
mock_createDir.assert_has_calls([
call("/castor/cern.ch/cms/store/aa1data/aa2/aa3/RAW"),
call("/castor/cern.ch/cms/store/aa1data/aa2/aa3/RAW/aa5")
])
mock_setFileClass.assert_called_once_with(
"/castor/cern.ch/cms/store/aa1data/aa2/aa3/RAW", "cms_raw")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch(
'WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.createRemoveFileCommand'
)
def testCreateStageOutCommand_stageIn(self, mock_createRemoveFileCommand,
mock_isEOS):
mock_createRemoveFileCommand.return_value = "remove file command"
mock_isEOS.return_value = False
self.RFCPCERNImpl.stageIn = True
result = self.RFCPCERNImpl.createStageOutCommand(
"sourcePFN", "targetPFN")
expectedResult = self.getStageOutCommandResult("sourcePFN",
"targetPFN", True,
False,
"remove file command",
False, False, False)
mock_createRemoveFileCommand.assert_called_once_with("targetPFN")
self.assertEqual(expectedResult, result)
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.splitPFN')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch(
'WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.createRemoveFileCommand'
)
def testCreateStageOutCommand_isEOS(self, mock_createRemoveFileCommand,
mock_isEOS, mock_splitPFN):
mock_createRemoveFileCommand.return_value = "remove file command"
mock_isEOS.return_value = True
mock_splitPFN.return_value = (None, "host", "path", None)
result = self.RFCPCERNImpl.createStageOutCommand(
"sourcePFN", "targetPFN")
expectedResult = self.getStageOutCommandResult("sourcePFN",
"targetPFN", False,
True,
"remove file command",
"host", "path", False)
mock_createRemoveFileCommand.assert_called_once_with("targetPFN")
self.assertEqual(expectedResult, result)
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.splitPFN')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch(
'WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.createRemoveFileCommand'
)
@mock.patch(
'WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.parseCastorPath')
def testCreateStageOutCommand_isEOSChecksums(self, mock_parseCastorPath,
mock_createRemoveFileCommand,
mock_isEOS, mock_splitPFN):
mock_createRemoveFileCommand.return_value = "remove file command"
mock_isEOS.return_value = True
mock_parseCastorPath.return_value = "targetFile"
mock_splitPFN.return_value = (None, "host", "path", None)
result = self.RFCPCERNImpl.createStageOutCommand(
"sourcePFN", "targetPFN", checksums={"adler32": "32"})
expectedResult = self.getStageOutCommandResult("sourcePFN",
"targetPFN",
False,
True,
"remove file command",
"host",
"path",
"targetFile",
useChecksum="00000032")
mock_createRemoveFileCommand.assert_called_once_with("targetPFN")
self.assertEqual(expectedResult, result)
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.splitPFN')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch(
'WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.createRemoveFileCommand'
)
@mock.patch(
'WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.parseCastorPath')
def testCreateStageOutCommand_checksumsOptions(
self, mock_parseCastorPath, mock_createRemoveFileCommand,
mock_isEOS, mock_splitPFN):
mock_createRemoveFileCommand.return_value = "remove file command"
mock_isEOS.return_value = False
mock_parseCastorPath.return_value = "targetFile"
mock_splitPFN.return_value = (None, "host", "path", None)
result = self.RFCPCERNImpl.createStageOutCommand(
"sourcePFN",
"targetPFN",
options="options",
checksums={"adler32": "32"})
expectedResult = self.getStageOutCommandResult("sourcePFN",
"targetPFN",
False,
False,
"remove file command",
"host",
"path",
"targetFile",
options="options",
useChecksum="32")
mock_createRemoveFileCommand.assert_called_once_with("targetPFN")
self.assertEqual(expectedResult, result)
def getStageOutCommandResult(self,
sourcePFN,
targetPFN,
stageIn,
isEOS,
removeCommand,
host,
path,
targetFile,
options=None,
useChecksum=None):
result = ""
if stageIn:
remotePFN, localPFN = sourcePFN, targetPFN
else:
remotePFN, localPFN = targetPFN, sourcePFN
result += "LOCAL_SIZE=`stat -c%%s \"%s\"`\n" % localPFN
result += "echo \"Local File Size is: $LOCAL_SIZE\"\n"
if isEOS:
result += "xrdcp -f -N "
if useChecksum:
targetPFN += "?eos.targetsize=$LOCAL_SIZE&eos.checksum=%s" % useChecksum
else:
if useChecksum:
result += "nstouch %s\n" % targetFile
result += "nssetchecksum -n adler32 -k %s %s\n" % (useChecksum,
targetFile)
result += "rfcp "
if options != None:
result += " %s " % options
result += " \"%s\" " % sourcePFN
result += " \"%s\" \n" % targetPFN
if stageIn:
result += "LOCAL_SIZE=`stat -c%%s \"%s\"`\n" % localPFN
result += "echo \"Local File Size is: $LOCAL_SIZE\"\n"
if isEOS:
result += "REMOTE_SIZE=`xrd '%s' stat '%s' | sed -r 's/.* Size: ([0-9]+) .*/\\1/'`\n" % (
host, path)
result += "echo \"Remote File Size is: $REMOTE_SIZE\"\n"
if useChecksum:
result += "echo \"Local File Checksum is: %s\"\n" % useChecksum
result += "REMOTE_XS=`xrd '%s' getchecksum '%s' | sed -r 's/.* adler32 ([0-9a-fA-F]{8}).*/\\1/'`\n" % (
host, path)
result += "echo \"Remote File Checksum is: $REMOTE_XS\"\n"
result += "if [ $REMOTE_SIZE ] && [ $REMOTE_XS ] && [ $LOCAL_SIZE == $REMOTE_SIZE ] &&" \
" [ '%s' == $REMOTE_XS ]; then exit 0; " % useChecksum
result += "else echo \"Error: Size or Checksum Mismatch between local and SE\";" \
" %s ; exit 60311 ; fi" % removeCommand
else:
result += "if [ $REMOTE_SIZE ] && [ $LOCAL_SIZE == $REMOTE_SIZE ]; then exit 0; "
result += "else echo \"Error: Size Mismatch between local and SE\"; %s ; exit 60311 ; fi" % removeCommand
else:
result += "REMOTE_SIZE=`rfstat '%s' | grep Size | cut -f2 -d: | tr -d ' '`\n" % remotePFN
result += "echo \"Remote File Size is: $REMOTE_SIZE\"\n"
result += "if [ $REMOTE_SIZE ] && [ $LOCAL_SIZE == $REMOTE_SIZE ]; then exit 0; else echo " \
"\"Error: Size Mismatch between local and SE\"; %s ; exit 60311 ; fi" % removeCommand
return result
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.splitPFN')
def testCreateRemoveFileCommand_isEos(self, mock_splitPFN, mock_isEOS):
mock_isEOS.return_value = True
mock_splitPFN.return_value = (None, "host", "path", None)
self.assertEqual(
"xrd host rm path",
self.RFCPCERNImpl.createRemoveFileCommand("file/path"))
mock_isEOS.assert_called_once_with("file/path")
mock_splitPFN.assert_called_once_with("file/path")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch(
'WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.parseCastorPath')
def testCreateRemoveFileCommand_simplePFN(self, mock_parseCastorPath,
mock_isEOS):
mock_isEOS.return_value = False
mock_parseCastorPath.return_value = "simplePFN"
self.assertEqual(
"stager_rm -a -M simplePFN ; nsrm simplePFN",
self.RFCPCERNImpl.createRemoveFileCommand("file/path"))
mock_isEOS.assert_called_once_with("file/path")
mock_parseCastorPath.assert_called_once_with("file/path")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch(
'WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.parseCastorPath')
@mock.patch(
'WMCore.Storage.Backends.RFCPCERNImpl.StageOutImpl.createRemoveFileCommand'
)
def testCreateRemoveFileCommand_StageOutError(self,
mock_createRemoveFileCommand,
mock_parseCastorPath,
mock_isEOS):
mock_isEOS.return_value = False
mock_createRemoveFileCommand.return_value = "test"
mock_parseCastorPath.side_effect = StageOutError("Errors!")
self.assertEqual(
"test", self.RFCPCERNImpl.createRemoveFileCommand("file/path"))
mock_isEOS.assert_called_once_with("file/path")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.splitPFN')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.execute')
def testRemoveFile_isEOS(self, mock_executeCommand, mock_splitPFN,
mock_isEOS):
mock_isEOS.return_value = True
mock_splitPFN.return_value = (None, "host", "path", None)
self.RFCPCERNImpl.removeFile("file/path")
mock_executeCommand.assert_called_with("xrd host rm path")
mock_isEOS.assert_called_with("file/path")
mock_splitPFN.assert_called_with("file/path")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.isEOS')
@mock.patch(
'WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.parseCastorPath')
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.execute')
def testRemoveFile_simplePFN(self, mock_executeCommand,
mock_parseCastorPath, mock_isEOS):
mock_isEOS.return_value = False
mock_parseCastorPath.return_value = "simplePFN"
self.RFCPCERNImpl.removeFile("file/path")
mock_executeCommand.assert_called_with(
"stager_rm -a -M simplePFN ; nsrm simplePFN")
mock_isEOS.assert_called_with("file/path")
mock_parseCastorPath.assert_called_with("file/path")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.runCommandWithOutput')
def testCheckDirExists_true(self, mock_runCommandWithOutput):
mock_runCommandWithOutput.return_value = (0, "Protection : d test")
self.assertTrue(self.RFCPCERNImpl.checkDirExists("file/path"))
mock_runCommandWithOutput.assert_called_with(
"rfstat file/path 2> /dev/null | grep Protection")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.runCommandWithOutput')
def testCheckDirExists_false(self, mock_runCommandWithOutput):
mock_runCommandWithOutput.return_value = (1, "Protection : d test")
self.assertFalse(self.RFCPCERNImpl.checkDirExists("file"))
mock_runCommandWithOutput.assert_called_with(
"rfstat file 2> /dev/null | grep Protection")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.runCommandWithOutput')
def testCheckDirExists_trueStageOutError(self, mock_runCommandWithOutput):
mock_runCommandWithOutput.return_value = (0, "Protection:test")
with self.assertRaises(StageOutError) as context:
self.RFCPCERNImpl.checkDirExists("file/test")
self.assertTrue(
'Output path is not a directory !' in context.exception)
mock_runCommandWithOutput.assert_called_with(
"rfstat file/test 2> /dev/null | grep Protection")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.runCommandWithOutput')
def testCheckDirExists_stageOutError(self, mock_runCommandWithOutput):
mock_runCommandWithOutput.return_value = StageOutError("Error")
with self.assertRaises(StageOutError) as context:
self.RFCPCERNImpl.checkDirExists("test2")
self.assertTrue('Error: Exception while invoking command:' in
context.exception)
mock_runCommandWithOutput.assert_called_with(
"rfstat test2 2> /dev/null | grep Protection")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.execute')
def testCreateDir(self, mock_execute):
self.RFCPCERNImpl.createDir("dirName")
mock_execute.assert_called_with("nsmkdir -p \"dirName\"")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.execute')
def testSetFileClass(self, mock_execute):
self.RFCPCERNImpl.setFileClass("dir", "fileClass")
mock_execute.assert_called_with("nschclass fileClass dir")
def testParseCastorPath_stageOutError(self):
with self.assertRaises(StageOutError) as context:
self.RFCPCERNImpl.parseCastorPath("test2")
self.assertTrue(
"Can't parse castor path out of URL !" in context.exception)
def testParseCastorPath_xrootd(self):
self.assertEqual(
"/castor/cern.ch/testas",
self.RFCPCERNImpl.parseCastorPath(
"root:test//castor/cern.ch/testas"))
self.assertEqual(
"/castor/cern.ch/testas",
self.RFCPCERNImpl.parseCastorPath(
"root:test/castor/cern.ch/testas?as"))
self.assertEqual(
"/castor/cern.ch/testas/test",
self.RFCPCERNImpl.parseCastorPath(
"root://castor/cern.ch/testas//test?as"))
def testParseCastorPath_rfio(self):
self.assertEqual(
"/castor/cern.ch/a/a/",
self.RFCPCERNImpl.parseCastorPath(
"rfio:test//castor/cern.ch/a/a//?testas"))
self.assertEqual(
"/castor/cern.ch/testas",
self.RFCPCERNImpl.parseCastorPath(
"rfio:test//castor/cern.ch/testas?as"))
self.assertEqual(
"/castor/cern.ch/testas/test",
self.RFCPCERNImpl.parseCastorPath(
"rfio:test/castor/cern.ch/testas//test?as"))
def testParseCastorPath_castor(self):
self.assertEqual(
"/castor/cern.ch/a/a/?testas",
self.RFCPCERNImpl.parseCastorPath("//castor/cern.ch/a/a//?testas"))
self.assertEqual(
"/castor/cern.ch/testas?as",
self.RFCPCERNImpl.parseCastorPath("//castor/cern.ch/testas?as"))
self.assertEqual(
"/castor/cern.ch/testas/test",
self.RFCPCERNImpl.parseCastorPath("/castor/cern.ch/testas//test"))
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.splitPFN')
def testIsEOS_rootFalse(self, mock_splitPFN):
mock_splitPFN.return_value = ("protocol", "host", None, None)
self.assertFalse(self.RFCPCERNImpl.isEOS("test"))
mock_splitPFN.assert_called_with("test")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.splitPFN')
def testIsEOS_rootTrue(self, mock_splitPFN):
mock_splitPFN.return_value = ("root", "host", None, None)
self.assertTrue(self.RFCPCERNImpl.isEOS("test"))
mock_splitPFN.assert_called_with("test")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.splitPFN')
def testIsEOS_rootCastorFalse(self, mock_splitPFN):
mock_splitPFN.return_value = ("root", "castorhost", None, None)
self.assertFalse(self.RFCPCERNImpl.isEOS("test"))
mock_splitPFN.assert_called_with("test")
@mock.patch('WMCore.Storage.Backends.RFCPCERNImpl.RFCPCERNImpl.splitPFN')
def testIsEOS_rootCastorTrue(self, mock_splitPFN):
mock_splitPFN.return_value = ("root", "host:castor", None, None)
self.assertTrue(self.RFCPCERNImpl.isEOS("test"))
mock_splitPFN.assert_called_with("test")
