def doDelete(self, pfn, seName, command, options, protocol ): """ deletes a file, raises on error StageOutError (and inherited exceptions) are for expected errors such as temporary connection failures. Anything else will be handled as an unexpected error and skip retrying with this plugin """ runCommand(["hadoop", "fs", "-rm" ,pfn])
def doDelete(self, pfn, seName, command, options, protocol): """ deletes a file, raises on error StageOutError (and inherited exceptions) are for expected errors such as temporary connection failures. Anything else will be handled as an unexpected error and skip retrying with this plugin """ runCommand(["hadoop", "fs", "-rm", pfn])
def testRunCommand_results(self): self.assertEqual( 1, runCommand("%s %s -exit 0" % (self.python_runtime, self.base))) self.assertEqual( 1, runCommand("%s %s -exit a" % (self.python_runtime, self.base))) self.assertEqual( 2, runCommand("%s %s -text" % (self.python_runtime, self.base))) self.assertEqual( 0, runCommand("%s %s -text a" % (self.python_runtime, self.base)))
def runCommandFailOnNonZero(self, command): (exitCode, output) = runCommand(command) if not exitCode: logging.error("Error in file transfer:") logging.error(output) raise StageOutError, "Transfer failure" return (exitCode, output)
def doTransfer(self, fromPfn, toPfn, stageOut, seName, command, options, protocol): # Figures out the src and dst files if stageOut: srcPath = fromPfn dstPath = toPfn else: srcPath = toPfn dstPath = fromPfn # Creates the directory self.createOutputDirectory(os.path.dirname(dstPath)) # Does the copy command = "%s %s %s" % (self._cpScript, srcPath, dstPath) exitCode, output = runCommand(command) print(output) if exitCode != 0: logging.error("Error in file transfer:") logging.error(output) raise StageOutError, "Transfer failure" # Returns the path return dstPath
def runCommandWarnOnNonZero(self, command): logging.info("Executing %s", command) (exitCode, output) = runCommand(command) if exitCode: logging.error("Error in file transfer..ignoring:") logging.error(output) return (exitCode, output)
def runCommandWarnOnNonZero(self, command): logging.info("Executing %s" % command) (exitCode, output) = runCommand(command) if exitCode: logging.error("Error in file transfer..ignoring:") logging.error(output) return (exitCode, output)
def doWrapped(self, commandArgs): wrapperPath = os.path.join(getWMBASE(),'src','python','WMCore','Storage','Plugins','DCCPFNAL','wrapenv.sh') commandArgs.insert(0, wrapperPath) exitCode, output = runCommand(commandArgs) if exitCode != 0: logging.info("Non zero exit code: %s" % repr(exitCode)) return (exitCode, output)
def doDelete(self, pfnToRemove, seName, command, options, protocol): """ deletes a file, raises on error StageOutError (and inherited exceptions) are for expected errors such as temporary connection failures. Anything else will be handled as an unexpected error and skip retrying with this plugin """ if pfnToRemove.find('/store/unmerged/lustre/') == -1: pfnSplit = pfnToRemove.split("/store/", 1)[1] filePath = "/pnfs/cms/WAX/11/store/%s" % pfnSplit command = ["rm", "-fv", filePath] runCommand(command) else: pfnSplit = pfnToRemove.split("/store/unmerged/lustre/", 1)[1] pfnToRemove = "/lustre/unmerged/%s" % pfnSplit command = ["/bin/rm", pfnToRemove] runCommand(command)
def doDelete(self, pfnToRemove, pnn, command, options, protocol ): """ deletes a file, raises on error StageOutError (and inherited exceptions) are for expected errors such as temporary connection failures. Anything else will be handled as an unexpected error and skip retrying with this plugin """ if pfnToRemove.find('/store/unmerged/lustre/') == -1: pfnSplit = pfnToRemove.split("/store/", 1)[1] filePath = "/pnfs/cms/WAX/11/store/%s" % pfnSplit command = ["rm","-fv",filePath] runCommand(command) else: pfnSplit = pfnToRemove.split("/store/unmerged/lustre/", 1)[1] pfnToRemove = "/lustre/unmerged/%s" % pfnSplit command = ["/bin/rm", pfnToRemove] runCommand(command)
def runCommandFailOnNonZero(self, command): logging.info("Executing %s", command) (exitCode, output) = runCommand(command) if exitCode: logging.error("Error in file transfer:") logging.error(" Command executed was: %s", command) logging.error(" Output was: %s", output) logging.error(" Exit code was: %s", exitCode) raise StageOutError("Transfer failure") return (exitCode, output)
def runCommandFailOnNonZero(self, command): logging.info("Executing %s" % command) (exitCode, output) = runCommand(command) if exitCode: logging.error("Error in file transfer:") logging.error(" Command executed was: %s" % command ) logging.error(" Output was: %s" % output ) logging.error(" Exit code was: %s" % exitCode ) raise StageOutError, "Transfer failure" return (exitCode, output)
def doWrapped(self, commandArgs): wrapperPath = os.path.join(getWMBASE(), 'src', 'python', 'WMCore', 'Storage', 'Plugins', 'DCCPFNAL', 'wrapenv.sh') commandArgs.insert(0, wrapperPath) exitCode, output = runCommand(commandArgs) if exitCode != 0: logging.info("Non zero exit code: %s" % repr(exitCode)) return (exitCode, output)
def executeCommand(self, command): """ _execute_ Execute the command provided, throw a StageOutError if it exits non zero """ try: exitCode = runCommand(command) msg = "Command exited with status: %s" % (exitCode) print msg except Exception, ex: raise StageOutError(str(ex), Command = command, ExitCode = 60311)
def doDelete(self, pfn, seName, command, options, protocol): """ _removeFile_ Removes the pfn. """ command = "%s %s" % (self._rmScript, pfn) exitCode, output = runCommand(command) if exitCode != 0: logging.error("Error removing file from LStore") logging.error(output) raise StageOutError, "remove file failure command %s, error %s" % (command, output)
def doDelete(self, pfn, seName, command, options, protocol): """ _removeFile_ Removes the pfn. """ command = "%s %s" % (self._rmScript, pfn) exitCode, output = runCommand(command) if exitCode != 0: logging.error("Error removing file from LStore") logging.error(output) raise StageOutError("remove file failure command %s, error %s" % (command, output))
def createOutputDirectory(self, targetPFN): """ _createOutputDirectory_ Creates the directory for vanderbilt """ command = "%s %s" % (self._mkdirScript, os.path.dirname(targetPFN)) # Calls the parent execute command to invoke the script which should # throw a stage out error exitCode, output = runCommand(command) if exitCode != 0: logging.error("Error creating directory") logging.error(output)
def createOutputDirectory(self, targetPFN): """ _createOutputDirectory_ Creates the directory for vanderbilt """ command = "%s %s" % (self._mkdirScript, targetPFN) # Calls the parent execute command to invoke the script which should # throw a stage out error exitCode, output = runCommand(command) if exitCode != 0: logging.error("Error creating directory in LStore") logging.error("Command: %s" % command) logging.error(output)
def doDelete(self, pfn, seName, command, options, protocol ): """ deletes a file, raises on error StageOutError (and inherited exceptions) are for expected errors such as temporary connection failures. Anything else will be handled as an unexpected error and skip retrying with this plugin """ if pfn.startswith("srm://"): runCommand( "lcg-del -b -l -D srmv2 --vo cms %s" % pfn ) elif pfn.startswith("file:"): runCommand( "/bin/rm -f %s" % pfn.replace("file:", "", 1) ) else: runCommand( StageOutImpl.createRemoveFileCommand(self, pfn) )
def doDelete(self, pfn, pnn, command, options, protocol ): """ deletes a file, raises on error StageOutError (and inherited exceptions) are for expected errors such as temporary connection failures. Anything else will be handled as an unexpected error and skip retrying with this plugin """ if pfn.startswith("srm://"): runCommand( "lcg-del -b -l -D srmv2 --vo cms %s" % pfn ) elif pfn.startswith("file:"): runCommand( "/bin/rm -f %s" % pfn.replace("file:", "", 1) ) else: runCommand( StageOutImpl.createRemoveFileCommand(self, pfn) )
def doDelete(self, pfn, pnn, command, options, protocol): """ deletes a file, raises on error StageOutError (and inherited exceptions) are for expected errors such as temporary connection failures. Anything else will be handled as an unexpected error and skip retrying with this plugin """ if os.path.isfile(pfn): runCommand("/bin/rm -f %s" % pfn) elif pfn.startswith("file:"): runCommand("env -i X509_USER_PROXY=$X509_USER_PROXY gfal-rm -vvv %s" % pfn) else: runCommand(StageOutImpl.createRemoveFileCommand(self, pfn))
def doDelete(self, pfn, seName, command, options, protocol): """ deletes a file, raises on error StageOutError (and inherited exceptions) are for expected errors such as temporary connection failures. Anything else will be handled as an unexpected error and skip retrying with this plugin """ if os.path.isfile(pfn): runCommand("/bin/rm -f %s" % pfn) elif pfn.startswith("file:"): runCommand("env -i gfal-rm -vvv %s" % pfn) else: runCommand(StageOutImpl.createRemoveFileCommand(self, pfn))
def executeCommand(self, command): """ _execute_ Execute the command provided, throw a StageOutError if it exits non zero """ try: exitCode = runCommand(command) msg = "%s : Command exited with status: %s\n" % (time.strftime("%Y-%m-%dT%H:%M:%S"), exitCode) print msg except Exception as ex: raise StageOutError(str(ex), Command=command, ExitCode=60311) if exitCode: msg = "%s : Command exited non-zero" % time.strftime("%Y-%m-%dT%H:%M:%S") print "ERROR: Exception During Stage Out:\n" print msg raise StageOutError(msg, Command=command, ExitCode=exitCode) return
def doTransfer(self, fromPfn, toPfn, stageOut, seName, command, options, protocol, checksum): """ if stageOut is true: The fromPfn is the LOCAL FILE NAME on the node, without file:// the toPfn is the target PFN, mapped from the LFN using the TFC or overrrides if stageOut is false: The toPfn is the LOCAL FILE NAME on the node, without file:// the fromPfn is the source PFN, mapped from the LFN using the TFC or overrrides """ # Figures out the src and dst files if stageOut: dstPath = toPfn else: dstPath = fromPfn # Creates the directory if stageOut: self.createOutputDirectory(os.path.dirname(dstPath)) else: os.makedirs(os.path.dirname(dstPath)) # Does the copy if stageOut: command = "%s %s %s" % (self._cpScript, fromPfn, toPfn) else: command = "%s %s %s" % (self._downloadScript, fromPfn, toPfn) exitCode, output = runCommand(command) print(output) if exitCode != 0: logging.error("Error in file transfer:") logging.error(output) raise StageOutError("Transfer failure, command %s, error %s" % (command, output)) # Returns the path return dstPath
def doTransfer(self, fromPfn, toPfn, stageOut, seName, command, options, protocol, checksum): """ if stageOut is true: The fromPfn is the LOCAL FILE NAME on the node, without file:// the toPfn is the target PFN, mapped from the LFN using the TFC or overrrides if stageOut is false: The toPfn is the LOCAL FILE NAME on the node, without file:// the fromPfn is the source PFN, mapped from the LFN using the TFC or overrrides """ # Figures out the src and dst files if stageOut: dstPath = toPfn else: dstPath = fromPfn # Creates the directory if stageOut: self.createOutputDirectory(os.path.dirname(dstPath)) else: os.makedirs(os.path.dirname(dstPath)) # Does the copy if stageOut: command = "%s %s %s" % (self._cpScript, fromPfn, toPfn) else: command = "%s %s %s" % (self._downloadScript, fromPfn, toPfn) exitCode, output = runCommand(command) print(output) if exitCode != 0: logging.error("Error in file transfer:") logging.error(output) raise StageOutError, "Transfer failure, command %s, error %s" % (command, output) # Returns the path return dstPath
def executeCommand(self, command): """ _execute_ Execute the command provided, throw a StageOutError if it exits non zero """ try: exitCode = runCommand(command) msg = "%s : Command exited with status: %s\n" % ( time.strftime("%Y-%m-%dT%H:%M:%S"), exitCode) print(msg) except Exception as ex: raise StageOutError(str(ex), Command=command, ExitCode=60311) if exitCode: msg = "%s : Command exited non-zero" % time.strftime( "%Y-%m-%dT%H:%M:%S") print("ERROR: Exception During Stage Out:\n") print(msg) raise StageOutError(msg, Command=command, ExitCode=exitCode) return
def executeCommand(self, command): """ _execute_ Execute the command provided, throw a StageOutError if it exits non zero """ try: exitCode = runCommand(command) msg = "Command exited with status: %s" % (exitCode) print msg except Exception as ex: raise StageOutError(str(ex), Command=command, ExitCode=60311) if exitCode in self.directoryErrorCodes: raise StageOutInvalidPath() elif exitCode: msg = "Command exited non-zero" print "ERROR: Exception During Stage Out:\n" print msg raise StageOutError(msg, Command=command, ExitCode=exitCode) return
def doTransfer(self, fromPfn, toPfn, stageOut, seName, command, options, protocol): """ performs a transfer. stageOut tells you which way to go. returns the new pfn or raises on failure. StageOutError (and inherited exceptions) are for expected errors such as temporary connection failures. Anything else will be handled as an unexpected error and skip retrying with this plugin """ localFileName = fromPfn fromPfn = self.prependFileProtocol(fromPfn) transferCommand = ( "lcg-cp -b -D srmv2 --vo cms --srm-timeout 2400 --sendreceive-timeout 2400 --connect-timeout 300 --verbose %s %s %s " % (options, fromPfn, toPfn) ) logging.info("Staging out with lcg-cp") logging.info(" commandline: %s" % transferCommand) (exitCode, output) = runCommand(transferCommand) # riddle me this, the following line fails with: # not all arguments converted during string formatting # FIXME logging.info(" output from lcg-cp: %s" % output) logging.info(" complete. #") # exit code" is %s" % exitCode) logging.info("Verifying file sizes") localSize = os.path.getsize(localFileName) remoteSize = subprocess.Popen( ["lcg-ls", "-l", "-b", "-D", "srmv2", toPfn], stdout=subprocess.PIPE ).communicate()[0] logging.info("got the following from lcg-ls %s" % remoteSize) remoteSize = remoteSize.split()[4] logging.info("Localsize: %s Remotesize: %s" % (localSize, remoteSize)) if int(localSize) != int(remoteSize): try: self.doDelete(toPfn, None, None, None, None) except: pass raise StageOutFailure, "File sizes don't match" return toPfn
def executeCommand(self, command): """ _execute_ Execute the command provided, throw a StageOutError if it exits non zero """ try: exitCode = runCommand(command) msg = "Command exited with status: %s" % (exitCode) print msg except Exception as ex: raise StageOutError(str(ex), Command = command, ExitCode = 60311) if exitCode in self.directoryErrorCodes: raise StageOutInvalidPath() elif exitCode: msg = "Command exited non-zero" print "ERROR: Exception During Stage Out:\n" print msg raise StageOutError(msg, Command = command, ExitCode = exitCode) return
def testRunCommandWithOutput_error(self, mock_sys): runCommand("python %s -exit 0" % self.base) mock_sys.stdout.write.assert_called_with("") mock_sys.stderr.write.assert_any_call("") mock_sys.stderr.write.assert_any_call("0\n")
def testRunCommandWithOutput_noError(self, mock_sys): runCommand("python %s -text a" % self.base) mock_sys.stdout.write.assert_any_call("a\n") mock_sys.stderr.write.assert_any_call("")
def testRunCommandWithOutput_errorAndText(self, mock_sys): runCommand("python %s -exception test" % self.base) assert "Exception: test\n" in mock_sys.stderr.write.call_args_list[0][ 0][0] mock_sys.stdout.write.assert_any_call("test\n")
def testRunCommand_results(self): self.assertEqual(1, runCommand("python %s -exit 0" % self.base)) self.assertEqual(1, runCommand("python %s -exit a" % self.base)) self.assertEqual(2, runCommand("python %s -text" % self.base)) self.assertEqual(0, runCommand("python %s -text a" % self.base))
def testRunCommandWithOutput_errorAndText(self, mock_sys): runCommand("python %s -exception test" % self.base) assert "Exception: test\n" in mock_sys.stderr.write.call_args_list[0][0][0] mock_sys.stdout.write.assert_any_call("test\n")
def runCommandWarnOnNonZero(self, command): (exitCode, output) = runCommand(command) if not exitCode: logging.error("Error in file transfer..ignoring:") logging.error(output) return (exitCode, output)
def doWrapped(self, commandArgs): wrapperPath = os.path.join(getWMBASE(), 'src', 'python', 'WMCore', 'Storage', 'Plugins', 'XRDCP', 'wrapenv.sh') commandArgs.insert(0, wrapperPath) return runCommand(commandArgs)
def doWrapped(self, commandArgs): wrapperPath = os.path.join(getWMBASE(),'src','python','WMCore','Storage','Plugins','XRDCP','wrapenv.sh') commandArgs.insert(0,wrapperPath) return runCommand(commandArgs)