def testSplitPFN_noOpaque(self):
self.assertEqual(
("splitable/test/test1", "test1", "splitable/test/test1", ""),
StageOutImpl.splitPFN("splitable/test/test1"))
self.assertEqual(('//eoscms//eos/cms/store/', 'eoscms',
'//eoscms//eos/cms/store/', ""),
StageOutImpl.splitPFN("//eoscms//eos/cms/store/"))
def __init__(self, stagein=False):
StageOutImpl.__init__(self, stagein)
self.numRetries = 5
self.retryPause = 300
# permissions for target directory
self.permissions = '775'
def __init__(self, stagein=False):
StageOutImpl.__init__(self, stagein)
self._mkdirScript = os.path.join(VandyImpl.BASEDIR, 'vandyMkdir.sh')
self._cpScript = os.path.join(VandyImpl.BASEDIR, 'vandyCp.sh')
self._rmScript = os.path.join(VandyImpl.BASEDIR, 'vandyRm.sh')
self._downloadScript = os.path.join(VandyImpl.BASEDIR, 'vandyDownload.sh')
def __init__(self, stagein=False):
StageOutImpl.__init__(self, stagein)
# If we want to execute commands in clean shell, we can`t separate them with ';'.
# Next commands after separation are executed without env -i and this leads us with
# mixed environment with COMP and system python.
# GFAL2 is not build under COMP environment and it had failures with mixed environment.
self.setups = "env -i X509_USER_PROXY=$X509_USER_PROXY JOBSTARTDIR=$JOBSTARTDIR bash -c '%s'"
self.removeCommand = self.setups % '. $JOBSTARTDIR/startup_environment.sh; date; gfal-rm -t 600 %s '
self.copyCommand = self.setups % '. $JOBSTARTDIR/startup_environment.sh; date; gfal-copy -t 2400 -T 2400 -p %(checksum)s %(options)s %(source)s %(destination)s'
def __init__(self, stagein=False):
StageOutImpl.__init__(self, stagein)
# If we want to execute commands in clean shell, we can`t separate them with ';'.
# Next commands after separation are executed without env -i and this leads us with
# mixed environment with COMP and system python.
# GFAL2 is not build under COMP environment and it had failures with mixed environment.
self.setups = "env -i X509_USER_PROXY=$X509_USER_PROXY JOBSTARTDIR=$JOBSTARTDIR bash -c '%s'"
self.removeCommand = self.setups % '. $JOBSTARTDIR/startup_environment.sh; printenv; date; gfal-rm -vvv -t 600 %s '
self.copyCommand = self.setups % '. $JOBSTARTDIR/startup_environment.sh; printenv; date; gfal-copy -vvv -t 2400 -T 2400 -p %(checksum)s %(options)s %(source)s %(destination)s'
def __init__(self, stagein=False):
StageOutImpl.__init__(self, stagein)
self._mkdirScript = os.path.join(VandyImpl.BASEDIR, 'vandyMkdir.sh')
self._cpScript = os.path.join(VandyImpl.BASEDIR, 'vandyCp.sh')
self._rmScript = os.path.join(VandyImpl.BASEDIR, 'vandyRm.sh')
self._downloadScript = os.path.join(VandyImpl.BASEDIR,
'vandyDownload.sh')
def __init__(self, stagein=False):
StageOutImpl.__init__(self, stagein)
self.setups = ""
setupScripts = {
"OSG_GRID": "/setup.sh",
"GLITE_WMS_LOCATION": "/etc/profile.d/glite-wmsui.sh",
"GLITE_LOCATION": "/../etc/profile.d/grid-env.sh",
"GRID_ENV_LOCATION": "/grid-env.sh",
}
for env, script in setupScripts.iteritems():
if os.environ.get(env):
fullScript = os.path.normpath(os.path.join(os.environ[env], script))
if os.path.isfile(fullScript):
self.setups += "source %s; " % fullScript
def __init__(self, stagein=False):
StageOutImpl.__init__(self, stagein)
self.setups = ''
setupScripts = {
'OSG_GRID' : '/setup.sh',
'GLITE_WMS_LOCATION' : '/etc/profile.d/glite-wmsui.sh',
'GLITE_LOCATION' : '/../etc/profile.d/grid-env.sh',
'GRID_ENV_LOCATION' : '/grid-env.sh',
}
for env, script in setupScripts.iteritems():
if os.environ.get(env):
fullScript = os.path.normpath(os.path.join(os.environ[env], script))
if os.path.isfile(fullScript):
self.setups += 'source %s; ' % fullScript
def __init__(self, stagein=False):
StageOutImpl.__init__(self, stagein)
self.setups = ''
setupScripts = {
'OSG_GRID': '/setup.sh',
'GLITE_WMS_LOCATION': '/etc/profile.d/glite-wmsui.sh',
'GLITE_LOCATION': '/../etc/profile.d/grid-env.sh',
'GRID_ENV_LOCATION': '/grid-env.sh',
}
for env, script in setupScripts.iteritems():
if os.environ.get(env):
fullScript = os.path.normpath(
os.path.join(os.environ[env], script))
if os.path.isfile(fullScript):
self.setups += 'source %s; ' % fullScript
def __init__(self, stagein=False):
StageOutImpl.__init__(self, stagein)
self.setups = ''
self.timeoutOptions = '--srm-timeout 600 --sendreceive-timeout 600 --connect-timeout 300'
setupScripts = {
'OSG_GRID': '/setup.sh',
'GLITE_WMS_LOCATION': '/etc/profile.d/glite-wmsui.sh',
'GLITE_LOCATION': '/../etc/profile.d/grid-env.sh',
'GRID_ENV_LOCATION': '/grid-env.sh',
}
for env, script in setupScripts.iteritems():
if os.environ.get(env):
fullScript = os.path.normpath(os.path.join(os.environ[env], script))
if os.path.isfile(fullScript):
self.setups += 'source %s; ' % fullScript
self.setups += 'date "+%Y-%m-%dT%H:%M:%S"; '
def createRemoveFileCommand(self, pfn):
"""
handle both srm and file pfn types
"""
if pfn.startswith("srm://"):
return "srm-advisory-delete %s" % pfn
elif pfn.startswith("file:"):
return "/bin/rm -f %s" % pfn.replace("file://", "", 1)
else:
return StageOutImpl.createRemoveFileCommand(self, pfn)
def createRemoveFileCommand(self, pfn):
"""
handle both srm and file pfn types
"""
if pfn.startswith("srm://"):
return "%s lcg-del -b -l -D srmv2 --vo cms %s" % (self.setups, pfn)
elif pfn.startswith("file:"):
return "/bin/rm -f %s" % pfn.replace("file:", "", 1)
else:
return StageOutImpl.createRemoveFileCommand(self, pfn)
def __init__(self, stagein=False):
StageOutImpl.__init__(self, stagein)
self.setups = ''
self.timeoutOptions = '--srm-timeout 600 --sendreceive-timeout 600 --connect-timeout 300'
setupScripts = {
'OSG_GRID': '/setup.sh',
'GLITE_WMS_LOCATION': '/etc/profile.d/glite-wmsui.sh',
'GLITE_LOCATION': '/../etc/profile.d/grid-env.sh',
'GRID_ENV_LOCATION': '/grid-env.sh',
}
for env, script in viewitems(setupScripts):
if os.environ.get(env):
fullScript = os.path.normpath(
os.path.join(os.environ[env], script))
if os.path.isfile(fullScript):
self.setups += 'source %s; ' % fullScript
self.setups += 'date "+%Y-%m-%dT%H:%M:%S"; '
def createRemoveFileCommand(self, pfn):
"""
handle both srm and file pfn types
"""
if pfn.startswith("srm://"):
return "%s lcg-del -b -l -D srmv2 --vo cms %s" % (self.setups, pfn)
elif pfn.startswith("file:"):
return "/bin/rm -f %s" % pfn.replace("file:", "", 1)
else:
return StageOutImpl.createRemoveFileCommand(self, pfn)
def createRemoveFileCommand(self, pfn):
"""
handle both srm and file pfn types
"""
if pfn.startswith("srm://"):
return "srmrm %s" % pfn
elif pfn.startswith("file:"):
return "/bin/rm -f %s" % pfn.replace("file://", "", 1)
else:
return StageOutImpl.createRemoveFileCommand(self, pfn)
def doDelete(self, pfn, seName, command, options, protocol ):
"""
deletes a file, raises on error
StageOutError (and inherited exceptions) are for expected errors
such as temporary connection failures. Anything else will be handled as an unexpected
error and skip retrying with this plugin
"""
if pfn.startswith("srm://"):
runCommand( "lcg-del -b -l -D srmv2 --vo cms %s" % pfn )
elif pfn.startswith("file:"):
runCommand( "/bin/rm -f %s" % pfn.replace("file:", "", 1) )
else:
runCommand( StageOutImpl.createRemoveFileCommand(self, pfn) )
def createRemoveFileCommand(self, pfn):
"""
_createRemoveFileCommand_
Alternate between EOS, CASTOR and local.
"""
if self.isEOS(pfn):
return "xrd eoscms rm %s" % pfn.replace("root://eoscms//eos/cms/", "/eos/cms/", 1)
try:
simplePFN = self.parseCastorPath(pfn)
return "stager_rm -a -M %s ; nsrm %s" % (simplePFN, simplePFN)
except StageOutError:
# Not castor
pass
return StageOutImpl.createRemoveFileCommand(self, pfn)
def createRemoveFileCommand(self, pfn):
"""
_createRemoveFileCommand_
Alternate between EOS, CASTOR and local.
"""
if self.isEOS(pfn):
return "xrd eoscms rm %s" % pfn.replace("root://eoscms//eos/cms/",
"/eos/cms/", 1)
try:
simplePFN = self.parseCastorPath(pfn)
return "stager_rm -a -M %s ; nsrm %s" % (simplePFN, simplePFN)
except StageOutError:
# Not castor
pass
return StageOutImpl.createRemoveFileCommand(self, pfn)
def createRemoveFileCommand(self, pfn):
"""
_createRemoveFileCommand_
Alternate between EOS, CASTOR and local.
"""
if self.isEOS(pfn):
(_,host,path,_) = self.splitPFN(pfn)
return "xrd %s rm %s" % (host, path)
try:
simplePFN = self.parseCastorPath(pfn)
return "stager_rm -a -M %s ; nsrm %s" % (simplePFN, simplePFN)
except StageOutError:
# Not castor
pass
return StageOutImpl.createRemoveFileCommand(self, pfn)
def createRemoveFileCommand(self, pfn):
"""
_createRemoveFileCommand_
Alternate between EOS, CASTOR and local.
"""
if self.isEOS(pfn):
(_,host,path,_) = self.splitPFN(pfn)
return "xrd %s rm %s" % (host, path)
try:
simplePFN = self.parseCastorPath(pfn)
return "stager_rm -a -M %s ; nsrm %s" % (simplePFN, simplePFN)
except StageOutError:
# Not castor
pass
return StageOutImpl.createRemoveFileCommand(self, pfn)
def createRemoveFileCommand(self, pfn):
"""
handle both srm and file pfn types
lcg-del options used:
-b don't make BDII calls to get SE type. SURLs must be fully provided
-l it means that the SURL is not registered in any LFC server, don't connect to the LFC server
-D specifies the default SE type we want to use
--srm-timeout Sets the SRM timeout, define the maximum waiting time for a SRM query.
The request will be aborted if it is still queued after srmtimeout seconds.
Default to 180 seconds.
--sendreceive-timeout Sets the send/receive timeout, maximum total time for the operation.
The operation will be aborted if the operation is not finished before X seconds.
Default to 3600 seconds.
--connect-timeout The connection will be aborted if the remote host doesn't
reply before X seconds. Default to 60 seconds.
"""
if pfn.startswith("srm://"):
return "%s lcg-del -b -l -D srmv2 %s --vo cms %s" % (self.setups, self.timeoutOptions, pfn)
elif pfn.startswith("file:"):
return "/bin/rm -f %s" % pfn.replace("file:", "", 1)
else:
return StageOutImpl.createRemoveFileCommand(self, pfn)
def createRemoveFileCommand(self, pfn):
"""
handle both srm and file pfn types
lcg-del options used:
-b don't make BDII calls to get SE type. SURLs must be fully provided
-l it means that the SURL is not registered in any LFC server, don't connect to the LFC server
-D specifies the default SE type we want to use
--srm-timeout Sets the SRM timeout, define the maximum waiting time for a SRM query.
The request will be aborted if it is still queued after srmtimeout seconds.
Default to 180 seconds.
--sendreceive-timeout Sets the send/receive timeout, maximum total time for the operation.
The operation will be aborted if the operation is not finished before X seconds.
Default to 3600 seconds.
--connect-timeout The connection will be aborted if the remote host doesn't
reply before X seconds. Default to 60 seconds.
"""
if pfn.startswith("srm://"):
return "%s lcg-del -b -l -D srmv2 %s --vo cms %s" % (
self.setups, self.timeoutOptions, pfn)
elif pfn.startswith("file:"):
return "/bin/rm -f %s" % pfn.replace("file:", "", 1)
else:
return StageOutImpl.createRemoveFileCommand(self, pfn)
def testSplitPFN_doubleSlashRoot(self):
self.assertEqual(('root', 'eoscms', '/eos/cms/store/', ""),
StageOutImpl.splitPFN("root://eoscms//eos/cms/store/"))
def testSplitPFN_path2(self):
self.assertEqual(('root', 'eoscms', 'cms', "?cms2ss&path=default"),
StageOutImpl.splitPFN("root://eoscms//eos/cms/store?path=cms&cms2ss&path=default"))
def __init__(self, stagein=False):
StageOutImpl.__init__(self, stagein)
self.directoryErrorCodes = (1,)
def testSplitPFN_pathConnector2(self):
self.assertEqual(('root', 'eoscms', 'cms', "?path=default&default2"),
StageOutImpl.splitPFN("root://eoscms//eos/cms/store?path=cms&path=default&default2"))
def __init__(self, stagein=False):
StageOutImpl.__init__(self, stagein)
def setUp(self):
self.StageOutImpl = StageOutImpl()
def testSplitPFN_doubleSlashRoot(self):
self.assertEqual(
('root', 'eoscms', '/eos/cms/store/', ""),
StageOutImpl.splitPFN("root://eoscms//eos/cms/store/"))
def testSplitPFN_path2(self):
self.assertEqual(
('root', 'eoscms', 'cms', "?cms2ss&path=default"),
StageOutImpl.splitPFN(
"root://eoscms//eos/cms/store?path=cms&cms2ss&path=default"))
class StageOutImplTest(unittest.TestCase):
def setUp(self):
self.StageOutImpl = StageOutImpl()
def testSplitPFN_noOpaque(self):
self.assertEqual(("splitable/test/test1", "test1", "splitable/test/test1", ""),
StageOutImpl.splitPFN("splitable/test/test1"))
self.assertEqual(('//eoscms//eos/cms/store/', 'eoscms', '//eoscms//eos/cms/store/', ""),
StageOutImpl.splitPFN("//eoscms//eos/cms/store/"))
def testSplitPFN_doubleSlashRoot(self):
self.assertEqual(('root', 'eoscms', '/eos/cms/store/', ""),
StageOutImpl.splitPFN("root://eoscms//eos/cms/store/"))
def testSplitPFN_path(self):
self.assertEqual(('root', 'eoscms', 'default', "?"),
StageOutImpl.splitPFN("root://eoscms//eos/cms/store?path=default"))
def testSplitPFN_pathConnector(self):
self.assertEqual(('root', 'eoscms', 'default', "?default2"),
StageOutImpl.splitPFN("root://eoscms//eos/cms/store?path=default&default2"))
def testSplitPFN_path2(self):
self.assertEqual(('root', 'eoscms', 'cms', "?cms2ss&path=default"),
StageOutImpl.splitPFN("root://eoscms//eos/cms/store?path=cms&cms2ss&path=default"))
def testSplitPFN_pathConnector2(self):
self.assertEqual(('root', 'eoscms', 'cms', "?path=default&default2"),
StageOutImpl.splitPFN("root://eoscms//eos/cms/store?path=cms&path=default&default2"))
@mock.patch('WMCore.Storage.StageOutImpl.runCommand')
def testExecuteCommand_stageOutError(self, mock_runCommand):
mock_runCommand.side_effect = Exception('BOOM!')
with self.assertRaises(Exception) as context:
self.StageOutImpl.executeCommand("command")
self.assertTrue('ErrorCode : 60311' in context.exception)
mock_runCommand.assert_called_with("command")
@mock.patch('WMCore.Storage.StageOutImpl.runCommand')
def testExecuteCommand_exitCode(self, mock_runCommand):
mock_runCommand.return_value = 1
with self.assertRaises(Exception) as context:
self.StageOutImpl.executeCommand("command")
self.assertTrue('ErrorCode : 1' in context.exception)
mock_runCommand.assert_called_with("command")
@mock.patch('WMCore.Storage.StageOutImpl.runCommand')
def testExecuteCommand_valid(self, mock_runCommand):
mock_runCommand.return_value = 0
self.StageOutImpl.executeCommand("command")
mock_runCommand.assert_called_with("command")
@mock.patch('WMCore.Storage.StageOutImpl.os.path')
def testCreateRemoveFileCommand_isFile(self, mock_path):
mock_path.isfile.return_value = True
mock_path.abspath.return_value = "path"
self.assertEqual("/bin/rm -f path", self.StageOutImpl.createRemoveFileCommand("test/path"))
mock_path.isfile.assert_called_with("test/path")
mock_path.abspath.assert_called_with("test/path")
@mock.patch('WMCore.Storage.StageOutImpl.os.path')
def testCreateRemoveFileCommand_notPath(self, mock_path):
mock_path.isfile.return_value = False
self.assertEqual("", self.StageOutImpl.createRemoveFileCommand("test/path"))
mock_path.isfile.assert_called_with("test/path")
def testCreateRemoveFileCommand_startsWithSlash(self):
self.assertEqual("/bin/rm -f /test/path", self.StageOutImpl.createRemoveFileCommand("/test/path"))
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createSourceName')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createTargetName')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createOutputDirectory')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createStageOutCommand')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.executeCommand')
def testCallable(self, mock_executeCommand, mock_createStageOutCommand, mock_createOutputDirectory,
mock_createTargetName, mock_createSourceName):
mock_createSourceName.return_value = "sourcePFN"
mock_createTargetName.return_value = "targetPFN"
mock_createStageOutCommand.return_value = "command"
self.StageOutImpl("protocol", "inputPFN", "targetPFN")
mock_createSourceName.assert_called_with("protocol", "inputPFN")
mock_createTargetName.assert_called_with("protocol", "targetPFN")
mock_createOutputDirectory.assert_called_with("targetPFN")
mock_createStageOutCommand.assert_called_with("sourcePFN", "targetPFN", None, None)
mock_executeCommand.assert_called_with("command")
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createSourceName')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createTargetName')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createOutputDirectory')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createStageOutCommand')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.executeCommand')
@mock.patch('WMCore.Storage.StageOutImpl.time')
def testCallable_StageOutError(self, mock_time, mock_executeCommand, mock_createStageOutCommand,
mock_createOutputDirectory, mock_createTargetName, mock_createSourceName):
mock_createSourceName.return_value = "sourcePFN"
mock_createTargetName.return_value = "targetPFN"
mock_createStageOutCommand.return_value = "command"
mock_createOutputDirectory.side_effect = [StageOutError("error"), StageOutError("error"), None]
mock_executeCommand.side_effect = [StageOutError("error"), StageOutError("error"), None]
self.StageOutImpl("protocol", "inputPFN", "targetPFN")
mock_createSourceName.assert_called_with("protocol", "inputPFN")
mock_createTargetName.assert_called_with("protocol", "targetPFN")
mock_createOutputDirectory.assert_called_with("targetPFN")
mock_createStageOutCommand.assert_called_with("sourcePFN", "targetPFN", None, None)
mock_executeCommand.assert_called_with("command")
calls = [call(600), call(600), call(600), call(600)]
mock_time.sleep.assert_has_calls(calls)
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createSourceName')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createTargetName')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createOutputDirectory')
@mock.patch('WMCore.Storage.StageOutImpl.time')
def testCallable_StageOutErrorFail(self, mock_time, mock_createOutputDirectory, mock_createTargetName,
mock_createSourceName):
mock_createSourceName.return_value = "sourcePFN"
mock_createTargetName.return_value = "targetPFN"
mock_createOutputDirectory.side_effect = [StageOutError("error"), StageOutError("error"),
StageOutError("Last error")]
with self.assertRaises(Exception) as context:
self.StageOutImpl("protocol", "inputPFN", "targetPFN")
self.assertTrue('Last error' in context.exception)
mock_createSourceName.assert_called_with("protocol", "inputPFN")
mock_createTargetName.assert_called_with("protocol", "targetPFN")
mock_createOutputDirectory.assert_called_with("targetPFN")
calls = [call(600), call(600)]
mock_time.sleep.assert_has_calls(calls)
def testSplitPFN_pathConnector2(self):
self.assertEqual(
('root', 'eoscms', 'cms', "?path=default&default2"),
StageOutImpl.splitPFN(
"root://eoscms//eos/cms/store?path=cms&path=default&default2"))
def testSplitPFN_noOpaque(self):
self.assertEqual(("splitable/test/test1", "test1", "splitable/test/test1", ""),
StageOutImpl.splitPFN("splitable/test/test1"))
self.assertEqual(('//eoscms//eos/cms/store/', 'eoscms', '//eoscms//eos/cms/store/', ""),
StageOutImpl.splitPFN("//eoscms//eos/cms/store/"))
def __init__(self, stagein=False):
StageOutImpl.__init__(self, stagein)
self.directoryErrorCodes = (1, )
def setUp(self):
self.StageOutImpl = StageOutImpl()
def __init__(self, stagein=False):
StageOutImpl.__init__(self, stagein)
self.numRetries = 5
self.retryPause = 300
def __init__(self, stagein=False):
StageOutImpl.__init__(self, stagein)
class StageOutImplTest(unittest.TestCase):
def setUp(self):
self.StageOutImpl = StageOutImpl()
def testSplitPFN_noOpaque(self):
self.assertEqual(
("splitable/test/test1", "test1", "splitable/test/test1", ""),
StageOutImpl.splitPFN("splitable/test/test1"))
self.assertEqual(('//eoscms//eos/cms/store/', 'eoscms',
'//eoscms//eos/cms/store/', ""),
StageOutImpl.splitPFN("//eoscms//eos/cms/store/"))
def testSplitPFN_doubleSlashRoot(self):
self.assertEqual(
('root', 'eoscms', '/eos/cms/store/', ""),
StageOutImpl.splitPFN("root://eoscms//eos/cms/store/"))
def testSplitPFN_path(self):
self.assertEqual(
('root', 'eoscms', 'default', "?"),
StageOutImpl.splitPFN("root://eoscms//eos/cms/store?path=default"))
def testSplitPFN_pathConnector(self):
self.assertEqual(
('root', 'eoscms', 'default', "?default2"),
StageOutImpl.splitPFN(
"root://eoscms//eos/cms/store?path=default&default2"))
def testSplitPFN_path2(self):
self.assertEqual(
('root', 'eoscms', 'cms', "?cms2ss&path=default"),
StageOutImpl.splitPFN(
"root://eoscms//eos/cms/store?path=cms&cms2ss&path=default"))
def testSplitPFN_pathConnector2(self):
self.assertEqual(
('root', 'eoscms', 'cms', "?path=default&default2"),
StageOutImpl.splitPFN(
"root://eoscms//eos/cms/store?path=cms&path=default&default2"))
@mock.patch('WMCore.Storage.StageOutImpl.runCommandWithOutput')
def testExecuteCommand_stageOutError(self, mock_runCommand):
mock_runCommand.side_effect = Exception('BOOM!')
with self.assertRaises(Exception) as context:
self.StageOutImpl.executeCommand("command")
self.assertTrue('ErrorCode : 60311' in context.exception)
mock_runCommand.assert_called_with("command")
@mock.patch('WMCore.Storage.StageOutImpl.runCommandWithOutput')
def testExecuteCommand_exitCode(self, mock_runCommand):
mock_runCommand.return_value = 1, "Test Fake Error"
with self.assertRaises(Exception) as context:
self.StageOutImpl.executeCommand("command")
self.assertTrue('ErrorCode : 1' in context.exception)
mock_runCommand.assert_called_with("command")
@mock.patch('WMCore.Storage.StageOutImpl.runCommandWithOutput')
def testExecuteCommand_valid(self, mock_runCommand):
mock_runCommand.return_value = 0, "Test Success"
self.StageOutImpl.executeCommand("command")
mock_runCommand.assert_called_with("command")
@mock.patch('WMCore.Storage.StageOutImpl.os.path')
def testCreateRemoveFileCommand_isFile(self, mock_path):
mock_path.isfile.return_value = True
mock_path.abspath.return_value = "path"
self.assertEqual(
"/bin/rm -f path",
self.StageOutImpl.createRemoveFileCommand("test/path"))
mock_path.isfile.assert_called_with("test/path")
mock_path.abspath.assert_called_with("test/path")
@mock.patch('WMCore.Storage.StageOutImpl.os.path')
def testCreateRemoveFileCommand_notPath(self, mock_path):
mock_path.isfile.return_value = False
self.assertEqual(
"", self.StageOutImpl.createRemoveFileCommand("test/path"))
mock_path.isfile.assert_called_with("test/path")
def testCreateRemoveFileCommand_startsWithSlash(self):
self.assertEqual(
"/bin/rm -f /test/path",
self.StageOutImpl.createRemoveFileCommand("/test/path"))
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createSourceName')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createTargetName')
@mock.patch(
'WMCore.Storage.StageOutImpl.StageOutImpl.createOutputDirectory')
@mock.patch(
'WMCore.Storage.StageOutImpl.StageOutImpl.createStageOutCommand')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.executeCommand')
def testCallable(self, mock_executeCommand, mock_createStageOutCommand,
mock_createOutputDirectory, mock_createTargetName,
mock_createSourceName):
mock_createSourceName.return_value = "sourcePFN"
mock_createTargetName.return_value = "targetPFN"
mock_createStageOutCommand.return_value = "command"
self.StageOutImpl("protocol", "inputPFN", "targetPFN")
mock_createSourceName.assert_called_with("protocol", "inputPFN")
mock_createTargetName.assert_called_with("protocol", "targetPFN")
mock_createOutputDirectory.assert_called_with("targetPFN")
mock_createStageOutCommand.assert_called_with("sourcePFN", "targetPFN",
None, None)
mock_executeCommand.assert_called_with("command")
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createSourceName')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createTargetName')
@mock.patch(
'WMCore.Storage.StageOutImpl.StageOutImpl.createOutputDirectory')
@mock.patch(
'WMCore.Storage.StageOutImpl.StageOutImpl.createStageOutCommand')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.executeCommand')
@mock.patch('WMCore.Storage.StageOutImpl.time')
def testCallable_StageOutError(self, mock_time, mock_executeCommand,
mock_createStageOutCommand,
mock_createOutputDirectory,
mock_createTargetName,
mock_createSourceName):
mock_createSourceName.return_value = "sourcePFN"
mock_createTargetName.return_value = "targetPFN"
mock_createStageOutCommand.return_value = "command"
mock_createOutputDirectory.side_effect = [
StageOutError("error"),
StageOutError("error"), None
]
mock_executeCommand.side_effect = [
StageOutError("error"),
StageOutError("error"), None
]
self.StageOutImpl("protocol", "inputPFN", "targetPFN")
mock_createSourceName.assert_called_with("protocol", "inputPFN")
mock_createTargetName.assert_called_with("protocol", "targetPFN")
mock_createOutputDirectory.assert_called_with("targetPFN")
mock_createStageOutCommand.assert_called_with("sourcePFN", "targetPFN",
None, None)
mock_executeCommand.assert_called_with("command")
calls = [call(600), call(600), call(600), call(600)]
mock_time.sleep.assert_has_calls(calls)
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createSourceName')
@mock.patch('WMCore.Storage.StageOutImpl.StageOutImpl.createTargetName')
@mock.patch(
'WMCore.Storage.StageOutImpl.StageOutImpl.createOutputDirectory')
@mock.patch('WMCore.Storage.StageOutImpl.time')
def testCallable_StageOutErrorFail(self, mock_time,
mock_createOutputDirectory,
mock_createTargetName,
mock_createSourceName):
mock_createSourceName.return_value = "sourcePFN"
mock_createTargetName.return_value = "targetPFN"
mock_createOutputDirectory.side_effect = [
StageOutError("error"),
StageOutError("error"),
StageOutError("Last error")
]
with self.assertRaises(Exception) as context:
self.StageOutImpl("protocol", "inputPFN", "targetPFN")
self.assertTrue('Last error' in context.exception)
mock_createSourceName.assert_called_with("protocol", "inputPFN")
mock_createTargetName.assert_called_with("protocol", "targetPFN")
mock_createOutputDirectory.assert_called_with("targetPFN")
calls = [call(600), call(600)]
mock_time.sleep.assert_has_calls(calls)
def __init__(self, stagein=False):
StageOutImpl.__init__(self, stagein)
self.numRetries = 5
self.retryPause = 300
def __init__(self, stagein=False):
StageOutImpl.__init__(self, stagein)
self.removeCommand = "env -i X509_USER_PROXY=$X509_USER_PROXY gfal-rm -vvv %s"
self.copyCommand = "env -i X509_USER_PROXY=$X509_USER_PROXY gfal-copy -t 2400 -T 2400 -p -vvv"
def __init__(self, stagein=False):
StageOutImpl.__init__(self, stagein)
# Create and hold onto a srm implementation in case we need it
self.srmImpl = LCGImpl(stagein)
def __init__(self, stagein=False):
StageOutImpl.__init__(self, stagein)
self.removeCommand = "env -i X509_USER_PROXY=$X509_USER_PROXY gfal-rm -vvv %s"
self.copyCommand = "env -i X509_USER_PROXY=$X509_USER_PROXY gfal-copy -t 2400 -T 2400 -p -vvv"
