def testRunCommandWithOutput_results(self): err, text = runCommandWithOutput("python %s -text" % self.base) self.assertEqual(2, err) self.assertTrue("ExecutableCommands.py: error" in text) self.assertEqual((1, 'stdout: \nstderr: 0\n'), runCommandWithOutput("python %s -exit 0" % self.base)) self.assertEqual((1, 'stdout: \nstderr: a\n'), runCommandWithOutput("python %s -exit a" % self.base)) self.assertEqual((0, 'stdout: a\n\nstderr: '), runCommandWithOutput("python %s -text a" % self.base))
def testRunCommandWithOutput_results(self): err, text = runCommandWithOutput("python %s -text" % self.base) self.assertEqual(2, err) self.assertTrue("ExecutableCommands.py: error" in text) self.assertEqual((1, "0\n"), runCommandWithOutput("python %s -exit 0" % self.base)) self.assertEqual((1, "a\n"), runCommandWithOutput("python %s -exit a" % self.base)) self.assertEqual((0, "a\n"), runCommandWithOutput("python %s -text a" % self.base))
def checkDirExists(self, directory): """ _checkDirExists_ Check if directory exists (will throw if it exists as a file) Only used for castor and local file """ command = "rfstat %s 2> /dev/null | grep Protection" % directory print("Check dir existence : %s" % command) try: exitCode, output = runCommandWithOutput(command) except Exception as ex: msg = "Error: Exception while invoking command:\n" msg += "%s\n" % command msg += "Exception: %s\n" % str(ex) msg += "Fatal error, abort stageout..." raise StageOutError(msg) if exitCode != 0: return False else: regExpParser = re.compile('^Protection[ ]+: d') if ( regExpParser.match(output) == None): raise StageOutError("Output path is not a directory !") else: return True
def checkDirExists(self, directory): """ _checkDirExists_ Check if directory exists (will throw if it exists as a file) Only used for castor and local file """ command = "rfstat %s 2> /dev/null | grep Protection" % directory print("Check dir existence : %s" % command) try: exitCode, output = runCommandWithOutput(command) except Exception as ex: msg = "Error: Exception while invoking command:\n" msg += "%s\n" % command msg += "Exception: %s\n" % str(ex) msg += "Fatal error, abort stageout..." raise StageOutError(msg) if exitCode != 0: return False else: regExpParser = re.compile('^Protection[ ]+: d') if regExpParser.match(output) is None: raise StageOutError("Output path is not a directory !") else: return True
def testRunCommandWithOutput_results(self): err, text = runCommandWithOutput("%s %s -text" % (self.python_runtime, self.base)) self.assertEqual(2, err) self.assertTrue("ExecutableCommands.py: error" in text) self.assertEqual( (1, 'stdout: \nstderr: 0\n'), runCommandWithOutput("%s %s -exit 0" % (self.python_runtime, self.base))) self.assertEqual( (1, 'stdout: \nstderr: a\n'), runCommandWithOutput("%s %s -exit a" % (self.python_runtime, self.base))) self.assertEqual( (0, 'stdout: a\n\nstderr: '), runCommandWithOutput("%s %s -text a" % (self.python_runtime, self.base)))
def createOutputDirectory(self, targetPFN): """ _createOutputDirectory_ create dir with group permission """ targetdir = os.path.dirname(targetPFN) checkdircmd = "/bin/ls %s > /dev/null " % targetdir print("Check dir existence : %s" % checkdircmd) try: checkdirexitCode, output = runCommandWithOutput(checkdircmd) except Exception as ex: msg = "Warning: Exception while invoking command:\n" msg += "%s\n" % checkdircmd msg += "Exception: %s\n" % str(ex) msg += "Go on anyway..." print(msg) if checkdirexitCode: mkdircmd = "/bin/mkdir -m 775 -p %s" % targetdir print("=> creating the dir : %s" % mkdircmd) try: exitCode, output = runCommandWithOutput(mkdircmd) except Exception as ex: msg = "Warning: Exception while invoking command:\n" msg += "%s\n" % mkdircmd msg += "Exception: %s\n" % str(ex) msg += "Go on anyway..." print(msg) if exitCode: msg = "Warning: failed to create the dir %s with the following error:\n%s" % ( targetdir, output) print(msg) else: print("=> dir %s correctly created" % targetdir) else: print("=> dir already exists... do nothing.")
def createOutputDirectory(self, targetPFN): """ _createOutputDirectory_ create dir with group permission """ targetdir = os.path.dirname(targetPFN) checkdircmd = "/bin/ls %s > /dev/null " % targetdir print("Check dir existence : %s" % checkdircmd) try: checkdirexitCode, output = runCommandWithOutput(checkdircmd) except Exception as ex: msg = "Warning: Exception while invoking command:\n" msg += "%s\n" % checkdircmd msg += "Exception: %s\n" % str(ex) msg += "Go on anyway..." print(msg) if checkdirexitCode: mkdircmd = "/bin/mkdir -m 775 -p %s" % targetdir print("=> creating the dir : %s" % mkdircmd) try: exitCode, output = runCommandWithOutput(mkdircmd) except Exception as ex: msg = "Warning: Exception while invoking command:\n" msg += "%s\n" % mkdircmd msg += "Exception: %s\n" % str(ex) msg += "Go on anyway..." print(msg) if exitCode: msg = "Warning: failed to create the dir %s with the following error:\n%s" % (targetdir, output) print(msg) else: print("=> dir %s correctly created" % targetdir) else: print("=> dir already exists... do nothing.")
def checkDirExists(self, directory): """ _checkDirExists_ Check if directory exists (will throw if it exists as a file) Only used for castor and local file """ command = "rfstat %s 2> /dev/null | grep Protection" % directory print "Check dir existence : %s" % command try: exitCode, output = runCommandWithOutput(command) except Exception, ex: msg = "Error: Exception while invoking command:\n" msg += "%s\n" % command msg += "Exception: %s\n" % str(ex) msg += "Fatal error, abort stageout..." raise StageOutError(msg)
def executeCommand(self, command): """ _execute_ Execute the command provided, throw a StageOutError if it exits non zero """ try: exitCode, output = runCommandWithOutput(command) msg = "Command exited with status: %s\nOutput message: %s" % (exitCode, output) logging.info(msg) except Exception as ex: raise StageOutError(str(ex), Command=command, ExitCode=60311) if exitCode: msg = "Command exited non-zero, ExitCode:%s\nOutput: %s " % (exitCode, output) logging.error("Exception During Stage Out:\n%s", msg) raise StageOutError(msg, Command=command, ExitCode=exitCode) return
def executeCommand(self, command): """ _execute_ Execute the command provided, throw a StageOutError if it exits non zero """ try: exitCode, output = runCommandWithOutput(command) msg = "%s : Command exited with status: %s\n Output message: %s" % ( time.strftime("%Y-%m-%dT%H:%M:%S"), exitCode, output) print(msg) except Exception as ex: raise StageOutError(str(ex), Command=command, ExitCode=60311) if exitCode: msg = "%s : Command exited non-zero ExitCode:%s\nOutput: (%s) " % ( time.strftime("%Y-%m-%dT%H:%M:%S"), exitCode, output) print("ERROR: Exception During Stage Out:\n%s" % msg) raise StageOutError(msg, Command=command, ExitCode=exitCode) return
def createOutputDirectory(self, targetPFN): """ _createOutputDirectory_ create dir with group permission """ # check how the targetPFN looks like and parse out the target dir targetdir = None if targetdir == None: regExpParser = re.compile('/+castor/(.*)') match = regExpParser.match(targetPFN) if ( match != None ): targetdir = os.path.dirname(targetPFN) if targetdir == None: regExpParser = re.compile('rfio:/+castor/(.*)') match = regExpParser.match(targetPFN) if ( match != None ): targetdir = os.path.dirname('/castor/' + match.group(1)) if targetdir == None: regExpParser = re.compile('rfio:.*path=/+castor/(.*)') match = regExpParser.match(targetPFN) if ( match != None ): targetdir = os.path.dirname('/castor/' + match.group(1)) # raise exception if we have no rule that can parse the target dir if targetdir == None: raise StageOutError("Cannot parse directory out of targetPFN") # remove multi-slashes from path while ( targetdir.find('//') > -1 ): targetdir = targetdir.replace('//','/') # # determine file class from LFN # fileclass = None # temp or unmerged files use cms_no_tape if ( fileclass == None ): regExpParser = re.compile('.*/castor/cern.ch/cms/store/temp/') if ( regExpParser.match(targetdir) != None ): fileclass = 'cms_no_tape' if ( fileclass == None ): regExpParser = re.compile('.*/castor/cern.ch/cms/store/unmerged/') if ( regExpParser.match(targetdir) != None ): fileclass = 'cms_no_tape' # RAW data files use cms_raw if ( fileclass == None ): regExpParser = re.compile('.*/castor/cern.ch/cms/store/data/[^/]+/[^/]+/RAW/') if ( regExpParser.match(targetdir) != None ): fileclass = 'cms_raw' # otherwise we assume another type of production file if ( fileclass == None ): fileclass = 'cms_production' print "Setting fileclass to : %s" % fileclass # check if directory exists rfstatCmd = "rfstat %s 2> /dev/null | grep Protection" % targetdir print "Check dir existence : %s" % rfstatCmd try: rfstatExitCode, rfstatOutput = runCommandWithOutput(rfstatCmd) except Exception as ex: msg = "Error: Exception while invoking command:\n" msg += "%s\n" % rfstatCmd msg += "Exception: %s\n" % str(ex) msg += "Fatal error, abort stageout..." raise StageOutError(msg) # does not exist => create it if rfstatExitCode: if ( fileclass != None ): self.createDir(targetdir, '000') self.setFileClass(targetdir,fileclass) self.changeDirMode(targetdir, self.permissions) else: self.createDir(targetdir, self.permissions) else: # check if this is a directory regExpParser = re.compile('Protection.*: d') if ( regExpParser.match(rfstatOutput) == None): raise StageOutError("Output path is not a directory !") else: # check if directory is writable regExpParser = re.compile('Protection.*: d---------') if ( regExpParser.match(rfstatOutput) != None and fileclass != None ): self.setFileClass(targetdir,fileclass) self.makeDirWritable(targetdir) return
def createOutputDirectory(self, targetPFN): """ _createOutputDirectory_ create dir with group permission """ # check how the targetPFN looks like and parse out the target dir targetdir = None if targetdir == None: regExpParser = re.compile('/+castor/(.*)') match = regExpParser.match(targetPFN) if (match != None): targetdir = os.path.dirname(targetPFN) if targetdir == None: regExpParser = re.compile('rfio:/+castor/(.*)') match = regExpParser.match(targetPFN) if (match != None): targetdir = os.path.dirname('/castor/' + match.group(1)) if targetdir == None: regExpParser = re.compile('rfio:.*path=/+castor/(.*)') match = regExpParser.match(targetPFN) if (match != None): targetdir = os.path.dirname('/castor/' + match.group(1)) # raise exception if we have no rule that can parse the target dir if targetdir == None: raise StageOutError("Cannot parse directory out of targetPFN") # remove multi-slashes from path while (targetdir.find('//') > -1): targetdir = targetdir.replace('//', '/') # # determine file class from LFN # fileclass = None # temp or unmerged files use cms_no_tape if (fileclass == None): regExpParser = re.compile('.*/castor/cern.ch/cms/store/temp/') if (regExpParser.match(targetdir) != None): fileclass = 'cms_no_tape' if (fileclass == None): regExpParser = re.compile('.*/castor/cern.ch/cms/store/unmerged/') if (regExpParser.match(targetdir) != None): fileclass = 'cms_no_tape' # RAW data files use cms_raw if (fileclass == None): regExpParser = re.compile( '.*/castor/cern.ch/cms/store/data/[^/]+/[^/]+/RAW/') if (regExpParser.match(targetdir) != None): fileclass = 'cms_raw' # otherwise we assume another type of production file if (fileclass == None): fileclass = 'cms_production' print "Setting fileclass to : %s" % fileclass # check if directory exists rfstatCmd = "rfstat %s 2> /dev/null | grep Protection" % targetdir print "Check dir existence : %s" % rfstatCmd try: rfstatExitCode, rfstatOutput = runCommandWithOutput(rfstatCmd) except Exception as ex: msg = "Error: Exception while invoking command:\n" msg += "%s\n" % rfstatCmd msg += "Exception: %s\n" % str(ex) msg += "Fatal error, abort stageout..." raise StageOutError(msg) # does not exist => create it if rfstatExitCode: if (fileclass != None): self.createDir(targetdir, '000') self.setFileClass(targetdir, fileclass) self.changeDirMode(targetdir, self.permissions) else: self.createDir(targetdir, self.permissions) else: # check if this is a directory regExpParser = re.compile('Protection.*: d') if (regExpParser.match(rfstatOutput) == None): raise StageOutError("Output path is not a directory !") else: # check if directory is writable regExpParser = re.compile('Protection.*: d---------') if (regExpParser.match(rfstatOutput) != None and fileclass != None): self.setFileClass(targetdir, fileclass) self.makeDirWritable(targetdir) return