Пример #1
0
    def run(self):
        cmds = []
        cmds.extend(self.envSetupCmds)
        cmds.append('cd %s' % self.runPath)
        cmds.append('diffPoolFiles.py -f %s -r %s' %
                    (self.theFile, self.theRefFile))

        # launch the commands
        sc = ShellCommand(cmds, self.logger)
        reply = sc.getRawReply()
        [self.logger.debug(str(l)) for l in reply]

        handle = None
        outFile = '%s__%s.diffPool' % (os.path.basename(
            self.theFile), os.path.basename(self.theRefFile))
        outPath = os.path.join(self.runPath, outFile)

        try:
            handle = open(outPath, 'w')
            handle.writelines(reply)
            handle.close()
            self.logger.debug('Successfully wrote out: %s' % outPath)
            return self.outcome(reply)
        except Exception, e:
            m = 'Unable to output results of diffPoolFiles to file: %s\n' % outPath
            m += str(e)
            self.logger.error(m)
            if handle: handle.close()
            return 1
Пример #2
0
    def run(self):
        cmds = []
        cmds.extend(self.envSetupCmds)
        cmds.append('cd %s;perfmon.py %s %s' %
                    (self.runPath, self.options, self.fileName))

        # launch the commands
        sc = ShellCommand(cmds, self.logger)
        reply = sc.getRawReply()
        [self.logger.debug(str(l)) for l in reply]

        handle = None
        outFile = 'out.perfmon.stdout.txt'
        outPath = os.path.join(self.runPath, outFile)

        try:
            handle = open(outPath, 'w')
            handle.writelines(reply)
            handle.close()
            self.logger.debug('Successfully wrote out: %s' % outPath)
        except Exception, e:
            m = 'Unable to output results of checkFile to file: %s\n' % outPath
            m += str(e)
            self.logger.error(m)
            if handle: handle.close()
            return 1
Пример #3
0
class Test_ShellCommand(unittest.TestCase):


    def setUp(self):
        self.executor = ShellCommand()
        logging.basicConfig(level=logging.INFO)


    def tearDown(self):
        pass
    
    
    def testExecShellCmd(self):
        cmdstring = "ls -l"
        res = self.executor.execShellCmd(cmdstring)
        self.assertEqual(0, res)
        

    def testExecShellCmdWithOutput(self):
        cmdstring = "date +%Y-%m-%d"
        (retcode, output) = self.executor.execShellCmdWithOutput(cmdstring)
        datestring = datetime.date.today().strftime("%Y-%m-%d")
        self.assertEqual(0, retcode, 'executed command failed: %s!' % (cmdstring))
        self.assertEqual(datestring, output.strip(),
             'output of command was wrong: expected "%s", but was "%s"' % (datestring, output.strip()))
Пример #4
0
    def query(self):

        try:
            sc = ShellCommand([self.queryCmd], self.logger, 100, None, True)
        except:
            self.status = -1
            self.output = 'query %s failed' % self.queryCmd
            return

        self.status = 0
        self.output = string.join(sc.getReply(), '\n')
Пример #5
0
 def stopJob(self):
     if self.done: return
     
     try:
         cmd = ['bkill ' + str(self.pid)]
         sc       = ShellCommand(cmd, self.logger, 100, None, True)
         self.logger.debug(sc.getReply())
     except Exception, e:
         m  = 'Job: ' + str(self.identifiedName)
         m += ' threw an exception while issuing cmd: %s' % cmd[0] 
         self.logger.warning(m)
Пример #6
0
def findJobOptionsSearchPath(cmds, paths, logger):

    logger.debug('Calculating JOBOPTIONSSEARCHPATH')

    # cmds.extend(['source setup.sh'])
    cmds.extend(['echo $JOBOPTSEARCHPATH', 'echo $PATH'])

    nlines = -1  # do not limit the number of lines
    shellCom = ShellCommand(cmds, logger, nlines)
    reply = shellCom.getReply()
    #m = 'Reply on setting up CMT to find JO searchpath: %s' % str(reply)
    #logger.debug(m)
    searchPath = ''
    # the env variable path will be used e.g. to locate pre athena scripts
    PATH = ''

    if len(reply) > 1:
        searchPath = reply[-2]
        PATH = reply[-1]
        # These paths are critical to all Athena jobs. As these are most
        # of the RTT jobs, give up if the search path is not set.
    else:
        m = """Could not obtain env vars JOBOPTSEARCHPATH and PATH,
        cmds sent to seek env variables:
        %s
        Reply:  %s
        abandoning run """ % (str(cmds), str(reply))
        logger.critical(m)
        raise RTTInputError(m)

    # logger.debug('Found JOBOPTSEARCHPATH: %s' % searchPath)
    # logger.debug('Found PATH: %s' % PATH)

    def jobOpSearchPathSepChar():
        if paths.isNightly: return ':'
        if int(paths.release.split('.')[0]) < 13: return ','
        return ':'

    searchPaths = {
        'jobOptionsSearchPath': {
            'path': searchPath,
            'separater': jobOpSearchPathSepChar()
        },
        'PATH': {
            'path': PATH,
            'separater': ':'
        }
    }

    return searchPaths
Пример #7
0
    def fire(self, what):
        """
        Submits the job by spawning a subprocess.
        The Popen objects captures std error, and std out
        'what' is a a key in to tthe self.scripts dictionary
        """

        cmd = self.scripts[what]['cmd']

        m = 'fire commandL\n: %s' % cmd
        self.logger.debug(m)

        sc = ShellCommand([cmd], self.logger)
        self.subLog = sc.getReply()
Пример #8
0
def configureCMTDirs(localRTTRun, cmtConfigurer, logger):
    # Not necessary if this is a local run
    if localRTTRun:
        logger.info('Local run, not configuring CMT dir')
        return

    cmtConfigurer.makeHomeCMTDir()

    nlines = -1  # do not limit the number of lines
    cmds = [cmtConfigurer.configure()]
    logger.info('cmds to configure cmt dir:\n%s' % str(cmds))
    shellCom = ShellCommand(cmds, logger, nlines)
    reply = shellCom.getReply()
    m = 'Reply on configuring CMT: %s' % str(reply)
    logger.info(m)
Пример #9
0
 def batchReplyProvesJobIsAbsentFromQueue(self, reply):
     """if the original bjobs command returned a 1 line reply
     it might be the case that the job is part of the history.
     check the history to see if the job is known.
     if so, assume the job ran."""
     
     if not reply: return False
     if len(reply) == 1:
         sc = ShellCommand(['bhist %s' % str(self.pid)], self.logger)
         if len(sc.getReply())>1:
             return True
         else:
             return False
         
     return not self.batchReplyProvesJobIsInQueue(reply)
Пример #10
0
    def findProjectPaths(self):
        self.projectCMTDirs = {self.topProject: {'Release': '', 'RunTime': ''}}

        for thing in ['Release', 'RunTime']:
            if self.isNightly:
                topProjectPath = os.path.join(
                    '/afs/cern.ch/atlas/software/builds/nightlies/',
                    self.originalBranch, self.topProject, self.release,
                    self.topProject + thing, 'cmt')
            else:
                topProjectPath = os.path.join(
                    '/afs/cern.ch/atlas/software/builds/', self.topProject,
                    self.release, self.topProject + thing, 'cmt')

            self.projectCMTDirs[self.topProject][thing] = topProjectPath

        cmtInfo = decodeCMTCONFIG(self.cmtConfig)

        cmd = [
            'cd %s' % self.cmtHomeDir,
            'source setup.sh -tag=%s,%s,%s,%s,%s,%s,%s' %
            (self.topProject, self.otherProject, cmtInfo['build'],
             self.originalBranch, cmtInfo['compiler'], cmtInfo['nbits'],
             self.release),
            'cd %s;source setup.sh;cmt show projects' %
            self.projectCMTDirs[self.topProject]['Release']
        ]

        try:
            sc = ShellCommand(cmd, self.logger, 100, None, True)
        except Exception, e:
            m = 'Unable to obtain project dirs. cmd:\n %s\nException:%s\nTraceback:%s' % (
                cmd, str(e), exc2string2())
            self.logger.critical(m)
            return
Пример #11
0
    def run(self):
        # check the macro exists. NOTE: this cannot be done in __init__ because
        # class instantiation occurs before copying of auxFilePatterns.
        if not self.macroExists():
            self.logger.warning("Root macro does not exit")
            return 1

        # note the current directory to reset it later
        currCwd = os.getcwd()

        # cmds = []
        # following cd must be part of the shell command has so other threads
        # cant mess it up
        # cmds.append('cd ' + self.runPath)
        # cmds.extend(self.rootCmds)
        self.logger.info("Launching macro " + str(self.rootCmds))

        # to import, need the RTT src dir. tried to do the import in __init__.
        # Could import there, but ShellCommand was not visible here.
        from ShellCommand import ShellCommand
        sc = ShellCommand(self.rootCmds, self.logger, self.maxPrintLines,
                          self.timeLimit)

        reply = sc.getReply()
        [self.logger.debug(str(l)) for l in reply]
        # reset the current directory as it was before
        os.chdir(currCwd)

        # reply is a list of strings, let's turn it into a single string
        replyString = reduce(lambda x, y: x + y + '\n', reply, '')

        # output reply to the logger
        name = str(self.identifiedName)
        msg = 'root command output for job %s\n %s' % (name, replyString)
        self.logger.info(msg)

        # Also output reply to file 'ROOToutput.log' in the run path
        # This file is a keep file pattern in the TopLevelJobGroup.xml file.
        h = open(os.path.join(self.runPath, 'ROOToutput.log'), 'a+')
        h.write(replyString)
        h.close()

        return 0
Пример #12
0
 def __init__(self, options, test = False):
     '''
     Constructor
     '''
     self.__cmdArgs = self.parseCmdLine(options)
     self.__toolsDir = os.path.abspath("../../tools") + "/"
     self.__dataDir = os.path.abspath("../../data") + "/"
     self.__osmosisCmd = OsmosisCommand()
     self.__executor = ShellCommand(test)
     logging.basicConfig(level=logging.INFO)
     logging.debug('MapCreator created with testmode = %s' % test)
Пример #13
0
    def run(self):
        poolFiles = [f for f in os.listdir(self.runPath) if f.endswith('.pool.root')]

        statusCode = 0

        for poolFile in poolFiles:
            pathToPoolFile = os.path.join(self.runPath, poolFile)

            cmds = []
            cmds.extend(self.envSetupCmds)
            cmds.append('checkFile.py -f %s' % pathToPoolFile)
            cmds.append('echo "RTT determined that checkFile exit status was: " $?')

            # launch the commands
            sc    = ShellCommand(cmds, self.logger)            
            reply = sc.getRawReply()
            [self.logger.debug(str(l)) for l in reply]

            handle = None
            outFile = poolFile + '.checkFile'
            outPath = os.path.join(self.runPath, outFile)
            
            try:
                handle = open(outPath, 'w')
                handle.writelines(reply)
                handle.close()
                self.logger.debug('Successfully wrote out: %s' % outPath)
                status = self.checkFileStatus(reply)
                if status!=0:
                    self.logger.warning('CheckFile returned error, non-zero status (%d) for %s' % (status, poolFile))
                statusCode += status
            except Exception, e:
                m  = 'Unable to output results of checkFile to file: %s\n' % outPath
                m += str(e)
                self.logger.error(m)
                if handle: handle.close()
                return 1
Пример #14
0
 def run( self ):
       if ( self.error == None ):
             try:
                   from ShellCommand import ShellCommand
                   # FIXME - self.logger could be just python False...
                   sc = ShellCommand( self.command,
                                      self.logger ) 
                   replay = sc.getReply()
                   reLine = re.compile("OVERALL STATISTICS STATUS")
                   status = 1
                   for line in replay:
                         for apiClass in self.__reAPIClasses:
                               if ( apiClass.search( line ) ):
                                     self.debug( line )
                                     if ( reLine.search( line ) ):
                                           if ( "OK" in line ): status = 0
                               
                   return status
             except ImportError: 
                   self.info("No ShellCommand in PYTHONPATH! 'No Cin-Cin, no run...'")
                   return 1
       else:
             self.info(self.error)
             return 1
Пример #15
0
    def run(self):
        if (self.error == None):
            try:
                from ShellCommand import ShellCommand
                # FIXME - self.logger could be just python False...
                sc = ShellCommand(self.command, self.logger)
                replay = sc.getReply()
                reLine = re.compile("OVERALL STATISTICS STATUS")
                status = 1
                for line in replay:
                    for apiClass in self.__reAPIClasses:
                        if (apiClass.search(line)):
                            self.debug(line)
                            if (reLine.search(line)):
                                if ("OK" in line): status = 0

                return status
            except ImportError:
                self.info(
                    "No ShellCommand in PYTHONPATH! 'No Cin-Cin, no run...'")
                return 1
        else:
            self.info(self.error)
            return 1
def make_qsub_niwg_weights_mc(run_name, production, data_type):
    """make niwg_weights command for mc files"""
    bincorr_dir = Directory('%s/%s/%s' %
                            (BINCORRBASE, production, run_name.low()))
    if not bincorr_dir.exists():
        print 'ERROR: there is no directory'
        print '\"%s\"' % (bincorr_dir.get())
        return None
    bincorr_dir_list = '-L %s' % (bincorr_dir.get())
    queue = '-q %s' % (QUEUE)
    hosts = '-N %s' % (HOSTS)
    minutes = '-M %s' % (MCMIN)
    memory = '-m %s' % (MEM)
    output_path_name = Directory('%s/%s/%s' %
                                 (NIWGWEIGHTSBASE, production, run_name.low()))
    run_type = '-r %s' % (data_type)
    if not output_path_name.exists():
        print 'WARNING: directory'
        print '\"%s\"' % (output_path_name.get())
        print 'does not exist. Will create it now'
        # output_path_name.mkdir()
    output_path = '-p %s' % (output_path_name.get())
    output_name = '-o %s_niwg_weights' % (run_name.upper_case())

    this_run = ShellCommand('nohup %s' % (QSUBNIWGWEIGHTS))
    this_run.add(bincorr_dir_list)
    this_run.add(queue)
    this_run.add(hosts)
    this_run.add(memory)
    this_run.add(minutes)
    this_run.add(run_type)
    this_run.add(output_path)
    this_run.add(output_name)
    return this_run
Пример #17
0
def make_qsub_spline_data(run_name, production):
    """make spline command for data files"""
    flattree_dir = Directory('%s/%s/%s' % (FLATTREEBASE,
                                           production,
                                           run_name.low()))
    if not flattree_dir.exists():
        print 'ERROR: there is no directory'
        print '\"%s\"' % (flattree_dir.get())
        return None
    flatttree_dir_list = '-L %s' % (flattree_dir.get())
    queue = '-q %s' % (QUEUE)
    hosts = '-N %s' % (HOSTS)
    minutes = '-M %s' % (DATAMIN)
    memory = '-m %s' % (MEM)
    output_path_name = Directory('%s/%s/%s' % (SPLINEBASE,
                                               production,
                                               run_name.low()))
    run_type = '-r data'
    if not output_path_name.exists():
        print 'WARNING: directory'
        print '\"%s\"' % (output_path_name.get())
        print 'does not exist. Will create it now'
        # output_path_name.mkdir()
    output_path = '-p %s' % (output_path_name.get())
    output_name = '-o %s_spline' % (run_name.upper_case())

    this_run = ShellCommand('nohup %s' % (QSUBSPLINE))
    this_run.add(flatttree_dir_list)
    this_run.add(queue)
    this_run.add(hosts)
    this_run.add(memory)
    this_run.add(minutes)
    this_run.add(run_type)
    this_run.add(output_path)
    this_run.add(output_name)
    return this_run
Пример #18
0
 def execute(self):
     for c in self.commands.keys():
         sc = ShellCommand([c], self.logger)
         reply = sc.getReply()
         # [self.logger.debug(r) for r in reply]
         self.commands[c].append(reply)
Пример #19
0
 def setUp(self):
     self.executor = ShellCommand()
     logging.basicConfig(level=logging.INFO)
Пример #20
0
class MapCreator(object):
    '''
    classdocs
    '''


    def __init__(self, options, test = False):
        '''
        Constructor
        '''
        self.__cmdArgs = self.parseCmdLine(options)
        self.__toolsDir = os.path.abspath("../../tools") + "/"
        self.__dataDir = os.path.abspath("../../data") + "/"
        self.__osmosisCmd = OsmosisCommand()
        self.__executor = ShellCommand(test)
        logging.basicConfig(level=logging.INFO)
        logging.debug('MapCreator created with testmode = %s' % test)
    
    def parseCmdLine(self, options):
        parser = ArgumentParser(description='Create a Garmin map from OpenStreetMap data')

        mandatory = parser.add_argument_group('mandatory arguments')

        # coordinate options
        helpMsg = 'bottom coordinate of the area to cut from input data'
        parser.add_argument('-b', '--bottom', action='store', type=float, dest='bottom',
                            required=False, default=47.2, help=helpMsg)
        
        helpMsg = 'top coordinate of the area to cut from input data'
        parser.add_argument('-t', '--top', action='store', type=float, dest='top',
                            required=False, default=55.2, help=helpMsg)
        
        helpMsg = 'western coordinate of the area to cut from input data'
        parser.add_argument('-l', '--left', action='store', type=float, dest='left',
                            required=False, default=5.7, help=helpMsg)
        
        helpMsg = 'eastern coordinate of the area to cut from input data'
        parser.add_argument('-r', '--right', action='store', type=float, dest='right',
                            required=False, default=15.1, help=helpMsg)
        
        helpMsg = 'path of polygon file'
        parser.add_argument('-p', '--poly', action='store', type=str, dest='poly',
                            required=False, help=helpMsg)
        
        # contourline options
        helpMsg = 'distance between minor contour lines'
        parser.add_argument('--ci', '--contour-step-min', action='store', type=int, dest='cstepmin',
                            required=False, default=10, help=helpMsg)
        
        helpMsg = 'distance between medium contourlines'
        parser.add_argument('--ce', '--contour-step-med', action='store', type=int, dest='cstepmed',
                            required=False, default=50, help=helpMsg)
        
        helpMsg = 'distance between major contourlines'
        parser.add_argument('--ca', '--contour-step-max', action='store', type=int, dest='cstepmax',
                            required=False, default=100, help=helpMsg)
        
        # mapid and mapname options
        helpMsg = 'map id (a unique 4 digit integer)'
        parser.add_argument('--mi', '--map-id', action='store', type=int, dest='mapid',
                            required=False, default=6400, help=helpMsg)
        
        helpMsg = 'mapname (a string giving a name to the map)'
        parser.add_argument('--mn', '--map-name', action='store', type=str, dest='mapname',
                            required=False, default='GER', help=helpMsg)
        
        # family ids for main map and contour maps
        helpMsg = 'family id (a unique 4 digit integer)'
        parser.add_argument('--fi', '--family-id', action='store', type=int, dest='famid',
                            required=False, default=1441, help=helpMsg)
        
        helpMsg = 'minor contourmap family id (a unique 4 digit integer)'
        parser.add_argument('--cid', '--contour-min-family-id', action='store', type=int, dest='cminid',
                            required=False, default=2441, help=helpMsg)
        
        helpMsg = 'major contourmap family id (a unique 4 digit integer)'
        parser.add_argument('--cad', '--contour-max-family-id', action='store', type=int, dest='cmaxid',
                            required=False, default=2443, help=helpMsg)
        
        # nocontours flag
        helpMsg = 'scan directories recursive'
        parser.add_argument('--nc', '--no-contours', action='store_true', dest='nocontours',
                            required=False, default=False, help=helpMsg)
        
        # inputfile option
        helpMsg = 'input file containing OpenStreetMap data'
        mandatory.add_argument('-i', '--input', action='store', type=str, dest='inputfile',
                               required=True, help=helpMsg)
        
        args = parser.parse_args(options)
        return args
    
    def getArgs(self):
        return self.__cmdArgs
    
    def cutMapDataWithPolygon(self):
        self.isDataFileOk(self.getArgs().inputfile)
        self.isPolyFileOk(self.getArgs().poly)
        cmdstr = self.__osmosisCmd.cutMapWithPolygon(infile=self.getArgs().inputfile, 
                                                     outfile=self.__dataDir + "temp.osm",
                                                     poly=self.getArgs().poly)
        logging.debug('cutMapDataWithPolygon: cmdstr = %s' % cmdstr)
        res = self.__executor.execShellCmd(cmdstr)
        return res
    
    def cutMapDataWithBoundingBox(self):
        self.isDataFileOk(self.getArgs().inputfile)
        cmdstr = self.__osmosisCmd.cutMapWithBoundingBox(infile  = self.getArgs().inputfile, 
                                                         outfile = self.__dataDir + "temp.osm",
                                                         top     = self.getArgs().top,
                                                         left    = self.getArgs().left,
                                                         bottom  = self.getArgs().bottom,
                                                         right   = self.getArgs().right)
        logging.debug('cutMapDataWithBoundingBox: cmdstr = %s' % cmdstr)
        res = self.__executor.execShellCmd(cmdstr)
        return res
    
    def isKnownDataFileExtension(self, inputfile):
        dummy, file_extension = os.path.splitext(inputfile)
        return file_extension in ('.bz2', '.osm', '.pbf')
      
    def isKnownPolyFileExtension(self, inputfile):
        dummy, file_extension = os.path.splitext(inputfile)
        return file_extension in ('.poly', '.txt')
    
    def checkFileExists(self, infile):
        # the following statement will throw an error if the input file can't be read
        with open(infile, 'r') as dummy:
            pass
        return True
    
    def isDataFileOk(self, datafile):
        self.checkFileExists(datafile)
        return self.isKnownDataFileExtension(datafile)
    
    def isPolyFileOk(self, polyfile):
        self.checkFileExists(polyfile)
        return self.isKnownPolyFileExtension(polyfile)
Пример #21
0
def make_qsub(run_name, production, is_mc):
    """makes a MC qsubmitter.py command class
    for a particular RN class input"""
    flattree_dir = Directory(join(FLATTREEBASE,
                             production, run_name.low()))
    if not flattree_dir.exists():
        print 'ERROR: there is no directory'
        print '\"%s\"' % (flattree_dir.get())
        return None
    flattree_dir_list = '-L %s' % (flattree_dir.get())
    queue = '-q %s' % (QUEUE)
    hosts = '-N %s' % (HOSTS)
    minutes = '-M %s' % (MCMIN)
    memory = '-m %s' % (MEM)
    output_path_name = Directory('%s/%s/%s' % (RUNSYSTNEWBASE,
                                               production, run_name.low()))
    if not output_path_name.exists():
        print 'WARNING: directory'
        print '\"%s\"' % (output_path_name.get())
        print 'does not exist. Will create it now'

    output_path = '-p %s' % (output_path_name.get())
    output_name = '-o %s' % (run_name.upper_case())

    run_type = '-r "MC"' if is_mc == DATATYPES.MC else '-r "Data"'

    this_run = ShellCommand('nohup %s' % (QSUBRUNSYSTNEW))
    this_run.add(flattree_dir_list)
    this_run.add(memory)
    this_run.add(queue)
    this_run.add(hosts)
    this_run.add(minutes)
    this_run.add(output_path)
    this_run.add(output_name)
    this_run.add(run_type)
    return this_run
Пример #22
0
def make_qsub_flattree_data(run_name, production):
    """makes a data qsubmitter.py command class for a
    particular RN class input"""
    run_list_file = TextFile('%s/%s/%s.list' % (RUNLISTS,
                                                production,
                                                run_name.upper_case()))
    if not run_list_file.exists():
        print 'ERROR: run list file'
        print '\"%s\"' % (run_list_file.get_file_name())
        print 'does not exist!'
        return None

    run_list = '-L %s' % (run_list_file.get_file_name())
    queue = '-q %s' % (QUEUE)
    hosts = '-N %s' % (HOSTS)
    minutes = '-M %s' % (DATAMIN)
    memory = '-m %s' % (MEM)
    output_path_name = Directory('%s/%s/%s' % (FLATTREEBASE,
                                               production,
                                               run_name.low()))
    if not output_path_name.exists():
        print 'WARNING: directory'
        print '\"%s\"' % (output_path_name.get())
        print 'does not exist. Will create it now'
        # output_path_name.mkdir()
    output_path = '-p %s' % (output_path_name.get())
    output_name = '-o %s' % (run_name.upper_case())
    n_jobs = '-n %s' % (DATANJOBS)

    this_run = ShellCommand('nohup %s' % (QSUBFLAT))
    this_run.add(run_list)
    this_run.add(queue)
    this_run.add(hosts)
    this_run.add(memory)
    this_run.add(minutes)
    this_run.add(output_path)
    this_run.add(n_jobs)
    this_run.add(output_name)
    return this_run
Пример #23
0
 def run(self):
     self.createRunnerScript()  # dump user runner script into runpath
     sc = ShellCommand(self.cmds, self.logger)
     self.logger.debug(sc.getRawReply())
Пример #24
0
class FileComparator:
    def __init__(self, paramDict):
        self.logger = Logger()
        self.logger.debug('instantiating file comparator')
        jDescriptor = paramDict.get('JobDescriptor')
        #location             = paramDict.get('refFileLocation','rttDB')
        location = 'install'
        #self.diffFile        = "differ.out"
        self.testFile = paramDict['testFile']
        self.refFile = paramDict['refFile']
        self.diffFile = paramDict['diffFile']
        self.shareArea = jDescriptor.paths.shareArea

        #self.success  =  0
        #self.error    = -1
        self.error = 'error'
        self.success = 'success'
        # ................

        srcDir = jDescriptor.paths.RTTSrcDir
        if srcDir not in sys.path: sys.path.append(srcDir)

        #location=self.getLocation(jDescriptor)
        if location == 'install':
            self.refFile = os.path.join(self.shareArea, self.refFile)
            self.diffFile = os.path.join(jDescriptor.runPath, self.diffFile)
            self.testFile = os.path.join(jDescriptor.runPath, self.testFile)
        elif location == 'rttDB':
            pass
            #self.refFile = dbFileRetriever.writeRefFile(self.refFile)
            #dbFileRetriever.storeFileInDB(self.testFile)
        else:
            self.logger.error("Ref File location unknown " + str(location))

    def getLocation(self, descriptor, whereFrom='refFileSystem'):
        from formatCollection import formatCollection
        path, file = os.path.split(descriptor.runPath)
        from FileLocationKeyGenerator import KeyGeneratorFactory

        lo = KeyGeneratorFactory().create(whereFrom, path)
        return lo

    def run(self):
        self.logger.debug('Running simple file comparator')
        if not os.path.exists(self.refFile):
            self.logger.error(
                'reference file %s does not exist or not found ' %
                str(self.refFile))
            return 'error'

        cwd = os.getcwd()
        testF = ""
        refF = ""
        testFile = self.testFile
        #reference files will be installed or will be in RTT DB
        if not os.path.isabs(self.testFile):
            testFile = os.path.join(cwd, self.testFile)

        try:
            testF = open(testFile, "rb").read()
            refF = open(self.refFile, "rb").read()
        except Exception, e:
            self.logger.error("Could not read files to compare ")
            self.logger.error(str(e))
            msg = 'File Comparator failed : ' + str(e)
            self.logger.error(msg)
            return self.error

        if testF != refF:
            print 'files are different'
            self.logger.debug("The following two files are different")
            self.logger.debug("File 1 :" + str(testFile))
            self.logger.debug("File 2 :" + str(self.refFile))
            diffCommand = [
                "diff " + testFile + " " + self.refFile + " >" + self.diffFile
            ]

            from ShellCommand import ShellCommand
            runDiff = ShellCommand(diffCommand, self.logger)
            #runDiff.wait()

            msg = 'File Comparator failed. \n'
            msg += 'The following files are different : \n%s\n%s' % (
                str(testFile), str(self.refFile))

            self.logger.error(msg)
            return self.error
        else:
            self.logger.info("The following two files are the same ")
            self.logger.info("File 1: " + str(testFile))
            self.logger.info("File 2: " + str(self.refFile))
            #logger.info("File 2: "+str(self.refFile))
            return self.success