def run(self): cmds = [] cmds.extend(self.envSetupCmds) cmds.append('cd %s' % self.runPath) cmds.append('diffPoolFiles.py -f %s -r %s' % (self.theFile, self.theRefFile)) # launch the commands sc = ShellCommand(cmds, self.logger) reply = sc.getRawReply() [self.logger.debug(str(l)) for l in reply] handle = None outFile = '%s__%s.diffPool' % (os.path.basename( self.theFile), os.path.basename(self.theRefFile)) outPath = os.path.join(self.runPath, outFile) try: handle = open(outPath, 'w') handle.writelines(reply) handle.close() self.logger.debug('Successfully wrote out: %s' % outPath) return self.outcome(reply) except Exception, e: m = 'Unable to output results of diffPoolFiles to file: %s\n' % outPath m += str(e) self.logger.error(m) if handle: handle.close() return 1
def run(self): cmds = [] cmds.extend(self.envSetupCmds) cmds.append('cd %s;perfmon.py %s %s' % (self.runPath, self.options, self.fileName)) # launch the commands sc = ShellCommand(cmds, self.logger) reply = sc.getRawReply() [self.logger.debug(str(l)) for l in reply] handle = None outFile = 'out.perfmon.stdout.txt' outPath = os.path.join(self.runPath, outFile) try: handle = open(outPath, 'w') handle.writelines(reply) handle.close() self.logger.debug('Successfully wrote out: %s' % outPath) except Exception, e: m = 'Unable to output results of checkFile to file: %s\n' % outPath m += str(e) self.logger.error(m) if handle: handle.close() return 1
class Test_ShellCommand(unittest.TestCase): def setUp(self): self.executor = ShellCommand() logging.basicConfig(level=logging.INFO) def tearDown(self): pass def testExecShellCmd(self): cmdstring = "ls -l" res = self.executor.execShellCmd(cmdstring) self.assertEqual(0, res) def testExecShellCmdWithOutput(self): cmdstring = "date +%Y-%m-%d" (retcode, output) = self.executor.execShellCmdWithOutput(cmdstring) datestring = datetime.date.today().strftime("%Y-%m-%d") self.assertEqual(0, retcode, 'executed command failed: %s!' % (cmdstring)) self.assertEqual(datestring, output.strip(), 'output of command was wrong: expected "%s", but was "%s"' % (datestring, output.strip()))
def query(self): try: sc = ShellCommand([self.queryCmd], self.logger, 100, None, True) except: self.status = -1 self.output = 'query %s failed' % self.queryCmd return self.status = 0 self.output = string.join(sc.getReply(), '\n')
def stopJob(self): if self.done: return try: cmd = ['bkill ' + str(self.pid)] sc = ShellCommand(cmd, self.logger, 100, None, True) self.logger.debug(sc.getReply()) except Exception, e: m = 'Job: ' + str(self.identifiedName) m += ' threw an exception while issuing cmd: %s' % cmd[0] self.logger.warning(m)
def findJobOptionsSearchPath(cmds, paths, logger): logger.debug('Calculating JOBOPTIONSSEARCHPATH') # cmds.extend(['source setup.sh']) cmds.extend(['echo $JOBOPTSEARCHPATH', 'echo $PATH']) nlines = -1 # do not limit the number of lines shellCom = ShellCommand(cmds, logger, nlines) reply = shellCom.getReply() #m = 'Reply on setting up CMT to find JO searchpath: %s' % str(reply) #logger.debug(m) searchPath = '' # the env variable path will be used e.g. to locate pre athena scripts PATH = '' if len(reply) > 1: searchPath = reply[-2] PATH = reply[-1] # These paths are critical to all Athena jobs. As these are most # of the RTT jobs, give up if the search path is not set. else: m = """Could not obtain env vars JOBOPTSEARCHPATH and PATH, cmds sent to seek env variables: %s Reply: %s abandoning run """ % (str(cmds), str(reply)) logger.critical(m) raise RTTInputError(m) # logger.debug('Found JOBOPTSEARCHPATH: %s' % searchPath) # logger.debug('Found PATH: %s' % PATH) def jobOpSearchPathSepChar(): if paths.isNightly: return ':' if int(paths.release.split('.')[0]) < 13: return ',' return ':' searchPaths = { 'jobOptionsSearchPath': { 'path': searchPath, 'separater': jobOpSearchPathSepChar() }, 'PATH': { 'path': PATH, 'separater': ':' } } return searchPaths
def fire(self, what): """ Submits the job by spawning a subprocess. The Popen objects captures std error, and std out 'what' is a a key in to tthe self.scripts dictionary """ cmd = self.scripts[what]['cmd'] m = 'fire commandL\n: %s' % cmd self.logger.debug(m) sc = ShellCommand([cmd], self.logger) self.subLog = sc.getReply()
def configureCMTDirs(localRTTRun, cmtConfigurer, logger): # Not necessary if this is a local run if localRTTRun: logger.info('Local run, not configuring CMT dir') return cmtConfigurer.makeHomeCMTDir() nlines = -1 # do not limit the number of lines cmds = [cmtConfigurer.configure()] logger.info('cmds to configure cmt dir:\n%s' % str(cmds)) shellCom = ShellCommand(cmds, logger, nlines) reply = shellCom.getReply() m = 'Reply on configuring CMT: %s' % str(reply) logger.info(m)
def batchReplyProvesJobIsAbsentFromQueue(self, reply): """if the original bjobs command returned a 1 line reply it might be the case that the job is part of the history. check the history to see if the job is known. if so, assume the job ran.""" if not reply: return False if len(reply) == 1: sc = ShellCommand(['bhist %s' % str(self.pid)], self.logger) if len(sc.getReply())>1: return True else: return False return not self.batchReplyProvesJobIsInQueue(reply)
def findProjectPaths(self): self.projectCMTDirs = {self.topProject: {'Release': '', 'RunTime': ''}} for thing in ['Release', 'RunTime']: if self.isNightly: topProjectPath = os.path.join( '/afs/cern.ch/atlas/software/builds/nightlies/', self.originalBranch, self.topProject, self.release, self.topProject + thing, 'cmt') else: topProjectPath = os.path.join( '/afs/cern.ch/atlas/software/builds/', self.topProject, self.release, self.topProject + thing, 'cmt') self.projectCMTDirs[self.topProject][thing] = topProjectPath cmtInfo = decodeCMTCONFIG(self.cmtConfig) cmd = [ 'cd %s' % self.cmtHomeDir, 'source setup.sh -tag=%s,%s,%s,%s,%s,%s,%s' % (self.topProject, self.otherProject, cmtInfo['build'], self.originalBranch, cmtInfo['compiler'], cmtInfo['nbits'], self.release), 'cd %s;source setup.sh;cmt show projects' % self.projectCMTDirs[self.topProject]['Release'] ] try: sc = ShellCommand(cmd, self.logger, 100, None, True) except Exception, e: m = 'Unable to obtain project dirs. cmd:\n %s\nException:%s\nTraceback:%s' % ( cmd, str(e), exc2string2()) self.logger.critical(m) return
def run(self): # check the macro exists. NOTE: this cannot be done in __init__ because # class instantiation occurs before copying of auxFilePatterns. if not self.macroExists(): self.logger.warning("Root macro does not exit") return 1 # note the current directory to reset it later currCwd = os.getcwd() # cmds = [] # following cd must be part of the shell command has so other threads # cant mess it up # cmds.append('cd ' + self.runPath) # cmds.extend(self.rootCmds) self.logger.info("Launching macro " + str(self.rootCmds)) # to import, need the RTT src dir. tried to do the import in __init__. # Could import there, but ShellCommand was not visible here. from ShellCommand import ShellCommand sc = ShellCommand(self.rootCmds, self.logger, self.maxPrintLines, self.timeLimit) reply = sc.getReply() [self.logger.debug(str(l)) for l in reply] # reset the current directory as it was before os.chdir(currCwd) # reply is a list of strings, let's turn it into a single string replyString = reduce(lambda x, y: x + y + '\n', reply, '') # output reply to the logger name = str(self.identifiedName) msg = 'root command output for job %s\n %s' % (name, replyString) self.logger.info(msg) # Also output reply to file 'ROOToutput.log' in the run path # This file is a keep file pattern in the TopLevelJobGroup.xml file. h = open(os.path.join(self.runPath, 'ROOToutput.log'), 'a+') h.write(replyString) h.close() return 0
def __init__(self, options, test = False): ''' Constructor ''' self.__cmdArgs = self.parseCmdLine(options) self.__toolsDir = os.path.abspath("../../tools") + "/" self.__dataDir = os.path.abspath("../../data") + "/" self.__osmosisCmd = OsmosisCommand() self.__executor = ShellCommand(test) logging.basicConfig(level=logging.INFO) logging.debug('MapCreator created with testmode = %s' % test)
def run(self): poolFiles = [f for f in os.listdir(self.runPath) if f.endswith('.pool.root')] statusCode = 0 for poolFile in poolFiles: pathToPoolFile = os.path.join(self.runPath, poolFile) cmds = [] cmds.extend(self.envSetupCmds) cmds.append('checkFile.py -f %s' % pathToPoolFile) cmds.append('echo "RTT determined that checkFile exit status was: " $?') # launch the commands sc = ShellCommand(cmds, self.logger) reply = sc.getRawReply() [self.logger.debug(str(l)) for l in reply] handle = None outFile = poolFile + '.checkFile' outPath = os.path.join(self.runPath, outFile) try: handle = open(outPath, 'w') handle.writelines(reply) handle.close() self.logger.debug('Successfully wrote out: %s' % outPath) status = self.checkFileStatus(reply) if status!=0: self.logger.warning('CheckFile returned error, non-zero status (%d) for %s' % (status, poolFile)) statusCode += status except Exception, e: m = 'Unable to output results of checkFile to file: %s\n' % outPath m += str(e) self.logger.error(m) if handle: handle.close() return 1
def run( self ): if ( self.error == None ): try: from ShellCommand import ShellCommand # FIXME - self.logger could be just python False... sc = ShellCommand( self.command, self.logger ) replay = sc.getReply() reLine = re.compile("OVERALL STATISTICS STATUS") status = 1 for line in replay: for apiClass in self.__reAPIClasses: if ( apiClass.search( line ) ): self.debug( line ) if ( reLine.search( line ) ): if ( "OK" in line ): status = 0 return status except ImportError: self.info("No ShellCommand in PYTHONPATH! 'No Cin-Cin, no run...'") return 1 else: self.info(self.error) return 1
def run(self): if (self.error == None): try: from ShellCommand import ShellCommand # FIXME - self.logger could be just python False... sc = ShellCommand(self.command, self.logger) replay = sc.getReply() reLine = re.compile("OVERALL STATISTICS STATUS") status = 1 for line in replay: for apiClass in self.__reAPIClasses: if (apiClass.search(line)): self.debug(line) if (reLine.search(line)): if ("OK" in line): status = 0 return status except ImportError: self.info( "No ShellCommand in PYTHONPATH! 'No Cin-Cin, no run...'") return 1 else: self.info(self.error) return 1
def make_qsub_niwg_weights_mc(run_name, production, data_type): """make niwg_weights command for mc files""" bincorr_dir = Directory('%s/%s/%s' % (BINCORRBASE, production, run_name.low())) if not bincorr_dir.exists(): print 'ERROR: there is no directory' print '\"%s\"' % (bincorr_dir.get()) return None bincorr_dir_list = '-L %s' % (bincorr_dir.get()) queue = '-q %s' % (QUEUE) hosts = '-N %s' % (HOSTS) minutes = '-M %s' % (MCMIN) memory = '-m %s' % (MEM) output_path_name = Directory('%s/%s/%s' % (NIWGWEIGHTSBASE, production, run_name.low())) run_type = '-r %s' % (data_type) if not output_path_name.exists(): print 'WARNING: directory' print '\"%s\"' % (output_path_name.get()) print 'does not exist. Will create it now' # output_path_name.mkdir() output_path = '-p %s' % (output_path_name.get()) output_name = '-o %s_niwg_weights' % (run_name.upper_case()) this_run = ShellCommand('nohup %s' % (QSUBNIWGWEIGHTS)) this_run.add(bincorr_dir_list) this_run.add(queue) this_run.add(hosts) this_run.add(memory) this_run.add(minutes) this_run.add(run_type) this_run.add(output_path) this_run.add(output_name) return this_run
def make_qsub_spline_data(run_name, production): """make spline command for data files""" flattree_dir = Directory('%s/%s/%s' % (FLATTREEBASE, production, run_name.low())) if not flattree_dir.exists(): print 'ERROR: there is no directory' print '\"%s\"' % (flattree_dir.get()) return None flatttree_dir_list = '-L %s' % (flattree_dir.get()) queue = '-q %s' % (QUEUE) hosts = '-N %s' % (HOSTS) minutes = '-M %s' % (DATAMIN) memory = '-m %s' % (MEM) output_path_name = Directory('%s/%s/%s' % (SPLINEBASE, production, run_name.low())) run_type = '-r data' if not output_path_name.exists(): print 'WARNING: directory' print '\"%s\"' % (output_path_name.get()) print 'does not exist. Will create it now' # output_path_name.mkdir() output_path = '-p %s' % (output_path_name.get()) output_name = '-o %s_spline' % (run_name.upper_case()) this_run = ShellCommand('nohup %s' % (QSUBSPLINE)) this_run.add(flatttree_dir_list) this_run.add(queue) this_run.add(hosts) this_run.add(memory) this_run.add(minutes) this_run.add(run_type) this_run.add(output_path) this_run.add(output_name) return this_run
def execute(self): for c in self.commands.keys(): sc = ShellCommand([c], self.logger) reply = sc.getReply() # [self.logger.debug(r) for r in reply] self.commands[c].append(reply)
def setUp(self): self.executor = ShellCommand() logging.basicConfig(level=logging.INFO)
class MapCreator(object): ''' classdocs ''' def __init__(self, options, test = False): ''' Constructor ''' self.__cmdArgs = self.parseCmdLine(options) self.__toolsDir = os.path.abspath("../../tools") + "/" self.__dataDir = os.path.abspath("../../data") + "/" self.__osmosisCmd = OsmosisCommand() self.__executor = ShellCommand(test) logging.basicConfig(level=logging.INFO) logging.debug('MapCreator created with testmode = %s' % test) def parseCmdLine(self, options): parser = ArgumentParser(description='Create a Garmin map from OpenStreetMap data') mandatory = parser.add_argument_group('mandatory arguments') # coordinate options helpMsg = 'bottom coordinate of the area to cut from input data' parser.add_argument('-b', '--bottom', action='store', type=float, dest='bottom', required=False, default=47.2, help=helpMsg) helpMsg = 'top coordinate of the area to cut from input data' parser.add_argument('-t', '--top', action='store', type=float, dest='top', required=False, default=55.2, help=helpMsg) helpMsg = 'western coordinate of the area to cut from input data' parser.add_argument('-l', '--left', action='store', type=float, dest='left', required=False, default=5.7, help=helpMsg) helpMsg = 'eastern coordinate of the area to cut from input data' parser.add_argument('-r', '--right', action='store', type=float, dest='right', required=False, default=15.1, help=helpMsg) helpMsg = 'path of polygon file' parser.add_argument('-p', '--poly', action='store', type=str, dest='poly', required=False, help=helpMsg) # contourline options helpMsg = 'distance between minor contour lines' parser.add_argument('--ci', '--contour-step-min', action='store', type=int, dest='cstepmin', required=False, default=10, help=helpMsg) helpMsg = 'distance between medium contourlines' parser.add_argument('--ce', '--contour-step-med', action='store', type=int, dest='cstepmed', required=False, default=50, help=helpMsg) helpMsg = 'distance between major contourlines' parser.add_argument('--ca', '--contour-step-max', action='store', type=int, dest='cstepmax', required=False, default=100, help=helpMsg) # mapid and mapname options helpMsg = 'map id (a unique 4 digit integer)' parser.add_argument('--mi', '--map-id', action='store', type=int, dest='mapid', required=False, default=6400, help=helpMsg) helpMsg = 'mapname (a string giving a name to the map)' parser.add_argument('--mn', '--map-name', action='store', type=str, dest='mapname', required=False, default='GER', help=helpMsg) # family ids for main map and contour maps helpMsg = 'family id (a unique 4 digit integer)' parser.add_argument('--fi', '--family-id', action='store', type=int, dest='famid', required=False, default=1441, help=helpMsg) helpMsg = 'minor contourmap family id (a unique 4 digit integer)' parser.add_argument('--cid', '--contour-min-family-id', action='store', type=int, dest='cminid', required=False, default=2441, help=helpMsg) helpMsg = 'major contourmap family id (a unique 4 digit integer)' parser.add_argument('--cad', '--contour-max-family-id', action='store', type=int, dest='cmaxid', required=False, default=2443, help=helpMsg) # nocontours flag helpMsg = 'scan directories recursive' parser.add_argument('--nc', '--no-contours', action='store_true', dest='nocontours', required=False, default=False, help=helpMsg) # inputfile option helpMsg = 'input file containing OpenStreetMap data' mandatory.add_argument('-i', '--input', action='store', type=str, dest='inputfile', required=True, help=helpMsg) args = parser.parse_args(options) return args def getArgs(self): return self.__cmdArgs def cutMapDataWithPolygon(self): self.isDataFileOk(self.getArgs().inputfile) self.isPolyFileOk(self.getArgs().poly) cmdstr = self.__osmosisCmd.cutMapWithPolygon(infile=self.getArgs().inputfile, outfile=self.__dataDir + "temp.osm", poly=self.getArgs().poly) logging.debug('cutMapDataWithPolygon: cmdstr = %s' % cmdstr) res = self.__executor.execShellCmd(cmdstr) return res def cutMapDataWithBoundingBox(self): self.isDataFileOk(self.getArgs().inputfile) cmdstr = self.__osmosisCmd.cutMapWithBoundingBox(infile = self.getArgs().inputfile, outfile = self.__dataDir + "temp.osm", top = self.getArgs().top, left = self.getArgs().left, bottom = self.getArgs().bottom, right = self.getArgs().right) logging.debug('cutMapDataWithBoundingBox: cmdstr = %s' % cmdstr) res = self.__executor.execShellCmd(cmdstr) return res def isKnownDataFileExtension(self, inputfile): dummy, file_extension = os.path.splitext(inputfile) return file_extension in ('.bz2', '.osm', '.pbf') def isKnownPolyFileExtension(self, inputfile): dummy, file_extension = os.path.splitext(inputfile) return file_extension in ('.poly', '.txt') def checkFileExists(self, infile): # the following statement will throw an error if the input file can't be read with open(infile, 'r') as dummy: pass return True def isDataFileOk(self, datafile): self.checkFileExists(datafile) return self.isKnownDataFileExtension(datafile) def isPolyFileOk(self, polyfile): self.checkFileExists(polyfile) return self.isKnownPolyFileExtension(polyfile)
def make_qsub(run_name, production, is_mc): """makes a MC qsubmitter.py command class for a particular RN class input""" flattree_dir = Directory(join(FLATTREEBASE, production, run_name.low())) if not flattree_dir.exists(): print 'ERROR: there is no directory' print '\"%s\"' % (flattree_dir.get()) return None flattree_dir_list = '-L %s' % (flattree_dir.get()) queue = '-q %s' % (QUEUE) hosts = '-N %s' % (HOSTS) minutes = '-M %s' % (MCMIN) memory = '-m %s' % (MEM) output_path_name = Directory('%s/%s/%s' % (RUNSYSTNEWBASE, production, run_name.low())) if not output_path_name.exists(): print 'WARNING: directory' print '\"%s\"' % (output_path_name.get()) print 'does not exist. Will create it now' output_path = '-p %s' % (output_path_name.get()) output_name = '-o %s' % (run_name.upper_case()) run_type = '-r "MC"' if is_mc == DATATYPES.MC else '-r "Data"' this_run = ShellCommand('nohup %s' % (QSUBRUNSYSTNEW)) this_run.add(flattree_dir_list) this_run.add(memory) this_run.add(queue) this_run.add(hosts) this_run.add(minutes) this_run.add(output_path) this_run.add(output_name) this_run.add(run_type) return this_run
def make_qsub_flattree_data(run_name, production): """makes a data qsubmitter.py command class for a particular RN class input""" run_list_file = TextFile('%s/%s/%s.list' % (RUNLISTS, production, run_name.upper_case())) if not run_list_file.exists(): print 'ERROR: run list file' print '\"%s\"' % (run_list_file.get_file_name()) print 'does not exist!' return None run_list = '-L %s' % (run_list_file.get_file_name()) queue = '-q %s' % (QUEUE) hosts = '-N %s' % (HOSTS) minutes = '-M %s' % (DATAMIN) memory = '-m %s' % (MEM) output_path_name = Directory('%s/%s/%s' % (FLATTREEBASE, production, run_name.low())) if not output_path_name.exists(): print 'WARNING: directory' print '\"%s\"' % (output_path_name.get()) print 'does not exist. Will create it now' # output_path_name.mkdir() output_path = '-p %s' % (output_path_name.get()) output_name = '-o %s' % (run_name.upper_case()) n_jobs = '-n %s' % (DATANJOBS) this_run = ShellCommand('nohup %s' % (QSUBFLAT)) this_run.add(run_list) this_run.add(queue) this_run.add(hosts) this_run.add(memory) this_run.add(minutes) this_run.add(output_path) this_run.add(n_jobs) this_run.add(output_name) return this_run
def run(self): self.createRunnerScript() # dump user runner script into runpath sc = ShellCommand(self.cmds, self.logger) self.logger.debug(sc.getRawReply())
class FileComparator: def __init__(self, paramDict): self.logger = Logger() self.logger.debug('instantiating file comparator') jDescriptor = paramDict.get('JobDescriptor') #location = paramDict.get('refFileLocation','rttDB') location = 'install' #self.diffFile = "differ.out" self.testFile = paramDict['testFile'] self.refFile = paramDict['refFile'] self.diffFile = paramDict['diffFile'] self.shareArea = jDescriptor.paths.shareArea #self.success = 0 #self.error = -1 self.error = 'error' self.success = 'success' # ................ srcDir = jDescriptor.paths.RTTSrcDir if srcDir not in sys.path: sys.path.append(srcDir) #location=self.getLocation(jDescriptor) if location == 'install': self.refFile = os.path.join(self.shareArea, self.refFile) self.diffFile = os.path.join(jDescriptor.runPath, self.diffFile) self.testFile = os.path.join(jDescriptor.runPath, self.testFile) elif location == 'rttDB': pass #self.refFile = dbFileRetriever.writeRefFile(self.refFile) #dbFileRetriever.storeFileInDB(self.testFile) else: self.logger.error("Ref File location unknown " + str(location)) def getLocation(self, descriptor, whereFrom='refFileSystem'): from formatCollection import formatCollection path, file = os.path.split(descriptor.runPath) from FileLocationKeyGenerator import KeyGeneratorFactory lo = KeyGeneratorFactory().create(whereFrom, path) return lo def run(self): self.logger.debug('Running simple file comparator') if not os.path.exists(self.refFile): self.logger.error( 'reference file %s does not exist or not found ' % str(self.refFile)) return 'error' cwd = os.getcwd() testF = "" refF = "" testFile = self.testFile #reference files will be installed or will be in RTT DB if not os.path.isabs(self.testFile): testFile = os.path.join(cwd, self.testFile) try: testF = open(testFile, "rb").read() refF = open(self.refFile, "rb").read() except Exception, e: self.logger.error("Could not read files to compare ") self.logger.error(str(e)) msg = 'File Comparator failed : ' + str(e) self.logger.error(msg) return self.error if testF != refF: print 'files are different' self.logger.debug("The following two files are different") self.logger.debug("File 1 :" + str(testFile)) self.logger.debug("File 2 :" + str(self.refFile)) diffCommand = [ "diff " + testFile + " " + self.refFile + " >" + self.diffFile ] from ShellCommand import ShellCommand runDiff = ShellCommand(diffCommand, self.logger) #runDiff.wait() msg = 'File Comparator failed. \n' msg += 'The following files are different : \n%s\n%s' % ( str(testFile), str(self.refFile)) self.logger.error(msg) return self.error else: self.logger.info("The following two files are the same ") self.logger.info("File 1: " + str(testFile)) self.logger.info("File 2: " + str(self.refFile)) #logger.info("File 2: "+str(self.refFile)) return self.success