def testRunCommandWithOutput_results(self):
err, text = runCommandWithOutput("python %s -text" % self.base)
self.assertEqual(2, err)
self.assertTrue("ExecutableCommands.py: error" in text)
self.assertEqual((1, 'stdout: \nstderr: 0\n'), runCommandWithOutput("python %s -exit 0" % self.base))
self.assertEqual((1, 'stdout: \nstderr: a\n'), runCommandWithOutput("python %s -exit a" % self.base))
self.assertEqual((0, 'stdout: a\n\nstderr: '), runCommandWithOutput("python %s -text a" % self.base))
def testRunCommandWithOutput_results(self):
err, text = runCommandWithOutput("python %s -text" % self.base)
self.assertEqual(2, err)
self.assertTrue("ExecutableCommands.py: error" in text)
self.assertEqual((1, "0\n"),
runCommandWithOutput("python %s -exit 0" % self.base))
self.assertEqual((1, "a\n"),
runCommandWithOutput("python %s -exit a" % self.base))
self.assertEqual((0, "a\n"),
runCommandWithOutput("python %s -text a" % self.base))
def checkDirExists(self, directory):
"""
_checkDirExists_
Check if directory exists (will throw if it exists as a file)
Only used for castor and local file
"""
command = "rfstat %s 2> /dev/null | grep Protection" % directory
print("Check dir existence : %s" % command)
try:
exitCode, output = runCommandWithOutput(command)
except Exception as ex:
msg = "Error: Exception while invoking command:\n"
msg += "%s\n" % command
msg += "Exception: %s\n" % str(ex)
msg += "Fatal error, abort stageout..."
raise StageOutError(msg)
if exitCode != 0:
return False
else:
regExpParser = re.compile('^Protection[ ]+: d')
if ( regExpParser.match(output) == None):
raise StageOutError("Output path is not a directory !")
else:
return True
def checkDirExists(self, directory):
"""
_checkDirExists_
Check if directory exists (will throw if it exists as a file)
Only used for castor and local file
"""
command = "rfstat %s 2> /dev/null | grep Protection" % directory
print("Check dir existence : %s" % command)
try:
exitCode, output = runCommandWithOutput(command)
except Exception as ex:
msg = "Error: Exception while invoking command:\n"
msg += "%s\n" % command
msg += "Exception: %s\n" % str(ex)
msg += "Fatal error, abort stageout..."
raise StageOutError(msg)
if exitCode != 0:
return False
else:
regExpParser = re.compile('^Protection[ ]+: d')
if regExpParser.match(output) is None:
raise StageOutError("Output path is not a directory !")
else:
return True
def testRunCommandWithOutput_results(self):
err, text = runCommandWithOutput("%s %s -text" %
(self.python_runtime, self.base))
self.assertEqual(2, err)
self.assertTrue("ExecutableCommands.py: error" in text)
self.assertEqual(
(1, 'stdout: \nstderr: 0\n'),
runCommandWithOutput("%s %s -exit 0" %
(self.python_runtime, self.base)))
self.assertEqual(
(1, 'stdout: \nstderr: a\n'),
runCommandWithOutput("%s %s -exit a" %
(self.python_runtime, self.base)))
self.assertEqual(
(0, 'stdout: a\n\nstderr: '),
runCommandWithOutput("%s %s -text a" %
(self.python_runtime, self.base)))
def createOutputDirectory(self, targetPFN):
"""
_createOutputDirectory_
create dir with group permission
"""
targetdir = os.path.dirname(targetPFN)
checkdircmd = "/bin/ls %s > /dev/null " % targetdir
print("Check dir existence : %s" % checkdircmd)
try:
checkdirexitCode, output = runCommandWithOutput(checkdircmd)
except Exception as ex:
msg = "Warning: Exception while invoking command:\n"
msg += "%s\n" % checkdircmd
msg += "Exception: %s\n" % str(ex)
msg += "Go on anyway..."
print(msg)
if checkdirexitCode:
mkdircmd = "/bin/mkdir -m 775 -p %s" % targetdir
print("=> creating the dir : %s" % mkdircmd)
try:
exitCode, output = runCommandWithOutput(mkdircmd)
except Exception as ex:
msg = "Warning: Exception while invoking command:\n"
msg += "%s\n" % mkdircmd
msg += "Exception: %s\n" % str(ex)
msg += "Go on anyway..."
print(msg)
if exitCode:
msg = "Warning: failed to create the dir %s with the following error:\n%s" % (
targetdir, output)
print(msg)
else:
print("=> dir %s correctly created" % targetdir)
else:
print("=> dir already exists... do nothing.")
def createOutputDirectory(self, targetPFN):
"""
_createOutputDirectory_
create dir with group permission
"""
targetdir = os.path.dirname(targetPFN)
checkdircmd = "/bin/ls %s > /dev/null " % targetdir
print("Check dir existence : %s" % checkdircmd)
try:
checkdirexitCode, output = runCommandWithOutput(checkdircmd)
except Exception as ex:
msg = "Warning: Exception while invoking command:\n"
msg += "%s\n" % checkdircmd
msg += "Exception: %s\n" % str(ex)
msg += "Go on anyway..."
print(msg)
if checkdirexitCode:
mkdircmd = "/bin/mkdir -m 775 -p %s" % targetdir
print("=> creating the dir : %s" % mkdircmd)
try:
exitCode, output = runCommandWithOutput(mkdircmd)
except Exception as ex:
msg = "Warning: Exception while invoking command:\n"
msg += "%s\n" % mkdircmd
msg += "Exception: %s\n" % str(ex)
msg += "Go on anyway..."
print(msg)
if exitCode:
msg = "Warning: failed to create the dir %s with the following error:\n%s" % (targetdir, output)
print(msg)
else:
print("=> dir %s correctly created" % targetdir)
else:
print("=> dir already exists... do nothing.")
def checkDirExists(self, directory):
"""
_checkDirExists_
Check if directory exists (will throw if it exists as a file)
Only used for castor and local file
"""
command = "rfstat %s 2> /dev/null | grep Protection" % directory
print "Check dir existence : %s" % command
try:
exitCode, output = runCommandWithOutput(command)
except Exception, ex:
msg = "Error: Exception while invoking command:\n"
msg += "%s\n" % command
msg += "Exception: %s\n" % str(ex)
msg += "Fatal error, abort stageout..."
raise StageOutError(msg)
def checkDirExists(self, directory):
"""
_checkDirExists_
Check if directory exists (will throw if it exists as a file)
Only used for castor and local file
"""
command = "rfstat %s 2> /dev/null | grep Protection" % directory
print "Check dir existence : %s" % command
try:
exitCode, output = runCommandWithOutput(command)
except Exception, ex:
msg = "Error: Exception while invoking command:\n"
msg += "%s\n" % command
msg += "Exception: %s\n" % str(ex)
msg += "Fatal error, abort stageout..."
raise StageOutError(msg)
def executeCommand(self, command):
"""
_execute_
Execute the command provided, throw a StageOutError if it exits
non zero
"""
try:
exitCode, output = runCommandWithOutput(command)
msg = "Command exited with status: %s\nOutput message: %s" % (exitCode, output)
logging.info(msg)
except Exception as ex:
raise StageOutError(str(ex), Command=command, ExitCode=60311)
if exitCode:
msg = "Command exited non-zero, ExitCode:%s\nOutput: %s " % (exitCode, output)
logging.error("Exception During Stage Out:\n%s", msg)
raise StageOutError(msg, Command=command, ExitCode=exitCode)
return
def executeCommand(self, command):
"""
_execute_
Execute the command provided, throw a StageOutError if it exits
non zero
"""
try:
exitCode, output = runCommandWithOutput(command)
msg = "%s : Command exited with status: %s\n Output message: %s" % (
time.strftime("%Y-%m-%dT%H:%M:%S"), exitCode, output)
print(msg)
except Exception as ex:
raise StageOutError(str(ex), Command=command, ExitCode=60311)
if exitCode:
msg = "%s : Command exited non-zero ExitCode:%s\nOutput: (%s) " % (
time.strftime("%Y-%m-%dT%H:%M:%S"), exitCode, output)
print("ERROR: Exception During Stage Out:\n%s" % msg)
raise StageOutError(msg, Command=command, ExitCode=exitCode)
return
def createOutputDirectory(self, targetPFN):
"""
_createOutputDirectory_
create dir with group permission
"""
# check how the targetPFN looks like and parse out the target dir
targetdir = None
if targetdir == None:
regExpParser = re.compile('/+castor/(.*)')
match = regExpParser.match(targetPFN)
if ( match != None ):
targetdir = os.path.dirname(targetPFN)
if targetdir == None:
regExpParser = re.compile('rfio:/+castor/(.*)')
match = regExpParser.match(targetPFN)
if ( match != None ):
targetdir = os.path.dirname('/castor/' + match.group(1))
if targetdir == None:
regExpParser = re.compile('rfio:.*path=/+castor/(.*)')
match = regExpParser.match(targetPFN)
if ( match != None ):
targetdir = os.path.dirname('/castor/' + match.group(1))
# raise exception if we have no rule that can parse the target dir
if targetdir == None:
raise StageOutError("Cannot parse directory out of targetPFN")
# remove multi-slashes from path
while ( targetdir.find('//') > -1 ):
targetdir = targetdir.replace('//','/')
#
# determine file class from LFN
#
fileclass = None
# temp or unmerged files use cms_no_tape
if ( fileclass == None ):
regExpParser = re.compile('.*/castor/cern.ch/cms/store/temp/')
if ( regExpParser.match(targetdir) != None ):
fileclass = 'cms_no_tape'
if ( fileclass == None ):
regExpParser = re.compile('.*/castor/cern.ch/cms/store/unmerged/')
if ( regExpParser.match(targetdir) != None ):
fileclass = 'cms_no_tape'
# RAW data files use cms_raw
if ( fileclass == None ):
regExpParser = re.compile('.*/castor/cern.ch/cms/store/data/[^/]+/[^/]+/RAW/')
if ( regExpParser.match(targetdir) != None ):
fileclass = 'cms_raw'
# otherwise we assume another type of production file
if ( fileclass == None ):
fileclass = 'cms_production'
print "Setting fileclass to : %s" % fileclass
# check if directory exists
rfstatCmd = "rfstat %s 2> /dev/null | grep Protection" % targetdir
print "Check dir existence : %s" % rfstatCmd
try:
rfstatExitCode, rfstatOutput = runCommandWithOutput(rfstatCmd)
except Exception as ex:
msg = "Error: Exception while invoking command:\n"
msg += "%s\n" % rfstatCmd
msg += "Exception: %s\n" % str(ex)
msg += "Fatal error, abort stageout..."
raise StageOutError(msg)
# does not exist => create it
if rfstatExitCode:
if ( fileclass != None ):
self.createDir(targetdir, '000')
self.setFileClass(targetdir,fileclass)
self.changeDirMode(targetdir, self.permissions)
else:
self.createDir(targetdir, self.permissions)
else:
# check if this is a directory
regExpParser = re.compile('Protection.*: d')
if ( regExpParser.match(rfstatOutput) == None):
raise StageOutError("Output path is not a directory !")
else:
# check if directory is writable
regExpParser = re.compile('Protection.*: d---------')
if ( regExpParser.match(rfstatOutput) != None and fileclass != None ):
self.setFileClass(targetdir,fileclass)
self.makeDirWritable(targetdir)
return
def createOutputDirectory(self, targetPFN):
"""
_createOutputDirectory_
create dir with group permission
"""
# check how the targetPFN looks like and parse out the target dir
targetdir = None
if targetdir == None:
regExpParser = re.compile('/+castor/(.*)')
match = regExpParser.match(targetPFN)
if (match != None):
targetdir = os.path.dirname(targetPFN)
if targetdir == None:
regExpParser = re.compile('rfio:/+castor/(.*)')
match = regExpParser.match(targetPFN)
if (match != None):
targetdir = os.path.dirname('/castor/' + match.group(1))
if targetdir == None:
regExpParser = re.compile('rfio:.*path=/+castor/(.*)')
match = regExpParser.match(targetPFN)
if (match != None):
targetdir = os.path.dirname('/castor/' + match.group(1))
# raise exception if we have no rule that can parse the target dir
if targetdir == None:
raise StageOutError("Cannot parse directory out of targetPFN")
# remove multi-slashes from path
while (targetdir.find('//') > -1):
targetdir = targetdir.replace('//', '/')
#
# determine file class from LFN
#
fileclass = None
# temp or unmerged files use cms_no_tape
if (fileclass == None):
regExpParser = re.compile('.*/castor/cern.ch/cms/store/temp/')
if (regExpParser.match(targetdir) != None):
fileclass = 'cms_no_tape'
if (fileclass == None):
regExpParser = re.compile('.*/castor/cern.ch/cms/store/unmerged/')
if (regExpParser.match(targetdir) != None):
fileclass = 'cms_no_tape'
# RAW data files use cms_raw
if (fileclass == None):
regExpParser = re.compile(
'.*/castor/cern.ch/cms/store/data/[^/]+/[^/]+/RAW/')
if (regExpParser.match(targetdir) != None):
fileclass = 'cms_raw'
# otherwise we assume another type of production file
if (fileclass == None):
fileclass = 'cms_production'
print "Setting fileclass to : %s" % fileclass
# check if directory exists
rfstatCmd = "rfstat %s 2> /dev/null | grep Protection" % targetdir
print "Check dir existence : %s" % rfstatCmd
try:
rfstatExitCode, rfstatOutput = runCommandWithOutput(rfstatCmd)
except Exception as ex:
msg = "Error: Exception while invoking command:\n"
msg += "%s\n" % rfstatCmd
msg += "Exception: %s\n" % str(ex)
msg += "Fatal error, abort stageout..."
raise StageOutError(msg)
# does not exist => create it
if rfstatExitCode:
if (fileclass != None):
self.createDir(targetdir, '000')
self.setFileClass(targetdir, fileclass)
self.changeDirMode(targetdir, self.permissions)
else:
self.createDir(targetdir, self.permissions)
else:
# check if this is a directory
regExpParser = re.compile('Protection.*: d')
if (regExpParser.match(rfstatOutput) == None):
raise StageOutError("Output path is not a directory !")
else:
# check if directory is writable
regExpParser = re.compile('Protection.*: d---------')
if (regExpParser.match(rfstatOutput) != None
and fileclass != None):
self.setFileClass(targetdir, fileclass)
self.makeDirWritable(targetdir)
return
