Ejemplo n.º 1
0
 def CrabCrusher(self, dir):
     if os.path.isdir(dir) == False:
         return None
     fileString = ''
     rootFiles = 0
     for file in os.listdir(dir):
         if '.root' in file:
             fileString = fileString + 'FileList->Add( TFile::Open("' + file + '") );\n'
             rootFiles = rootFiles + 1
     if rootFiles == 0:
         return None
     tfile = open(Configuration.variables['HomeDirectory'] + '/Crab/Hadd.C',
                  'r')
     template = Template(tfile.read())
     tfile.close()
     cfile = open(dir + '/Hadd.C', 'w')
     cfile.write(
         template.safe_substitute(filelist=fileString, output="Crab.root"))
     cfile.close()
     os.chdir(dir)
     os.system('root -l -b Hadd.C')
     if os.path.isfile(dir + '/Crab.root'):
         return dir + '/Crab.root'
     else:
         return None
Ejemplo n.º 2
0
 def create( self, release, cfgfile):
   # Get the release name
   self.__release = release
   # Get the cfg file name
   self.__package = cfgfile.split('/')[-1].split('.')[0]
   # Generate a random string
   rvector = [random.choice(string.letters) for x in xrange(10)]
   rstring = "".join(rvector)
   # Name of the scrtach area
   self.__scratch = Configuration.variables['DropBox'] + '/Scratch/' + rstring
   # Creates a directory
   os.mkdir( self.__scratch )
   # Copy cfg file to scratch area
   shutil.copy(cfgfile, self.__scratch)
   # Create a cmsrun script
   tfile = open (Configuration.variables['VTRoot'] + '/data/cmsrun.template' ,'r')
   template = Template(tfile.read())
   tfile.close()
   cfile = open ( self.__scratch + '/cmsrun', 'w' )
   cmssws = Configuration.variables['WorkAreas'] + '/' + release + '/src' 
   cfile.write( template.substitute(scratch=self.__scratch, cfg=cfgfile, cmssw=cmssws) )
   cfile.close()
   os.chmod( self.__scratch + '/cmsrun', 0755 )
   # Create a condor script
   tfile = open (Configuration.variables['VTRoot'] + '/data/condor/condor.template' ,'r')
   template = Template(tfile.read())
   tfile.close()
   cfile = open ( self.__scratch + '/condor.script', 'w' ) 
   cfile.write( template.substitute(scratch=self.__scratch) )
   cfile.close()
Ejemplo n.º 3
0
    def createcfgs(self, lock, postprocess):
        # Get the subscription list 
        subscriptions = Subscriber()
        subscriptions.read(Configuration.variables['VTRoot']+'/data/subscriptions.xml')

        # Message list
        messages = []
               
        # For a given dataset get all the subscripted templates
        if subscriptions.map.has_key(self.__dataset):
        
          for template, interface in subscriptions.map[self.__dataset].iteritems():
    
            # open the associated template file
            tfile = open(template, 'r')
            tobject = Template(tfile.read())
            tfile.close()
    
            # create the proper source string
            string = 'source = PoolSource{\n\nuntracked vstring fileNames = {\n'
            for i in range(len(self.__lfns)):
              if i == len(self.__lfns) - 1:
                string = string + '\'' + self.__lfns[i] + '\'\n'
              else:
                string = string + '\'' + self.__lfns[i] + '\',\n'
            string = string + '}\n}\n'

            # creates a config file
            lock.acquire()
            if os.path.isdir(Configuration.variables['DropBox'] + '/CfgFiles/' + self.__release + '/') == False:
              os.mkdir(Configuration.variables['DropBox'] + '/CfgFiles/' + self.__release + '/')
            lock.release()
            cfgfile = Configuration.variables['DropBox'] + '/CfgFiles/' + self.__release + '/' + self.__dataset + '__' + template.split('/')[-1].split('.')[0] + '.cfg'
            cfile = open(cfgfile, 'w')
            cfile.write( tobject.safe_substitute(lfns=string, dataset=self.__dataset) )
            cfile.close()

            # Add a message for submitting the job
            message = {}
            message['startjob'] = { 'groupid' : 'None', 'interface' : str(interface), 'release' : self.__release, 'cfgfile' : str(cfgfile), 'postprocess' : postprocess }
            messages.append(message)
        
        return messages        
Ejemplo n.º 4
0
 def CrabCrusher(self, dir):
     if os.path.isdir(dir) == False:
         return None
     fileString = ''
     rootFiles = 0
     for file in os.listdir(dir):
         if '.root' in file:
             fileString = fileString + 'FileList->Add( TFile::Open("'+file+'") );\n'
             rootFiles = rootFiles + 1
     if rootFiles == 0:
         return None
     tfile = open (Configuration.variables['HomeDirectory'] + '/Crab/Hadd.C' ,'r')
     template = Template(tfile.read())
     tfile.close()
     cfile = open(dir+'/Hadd.C', 'w')
     cfile.write(template.safe_substitute(filelist=fileString, output="Crab.root" ))
     cfile.close()
     os.chdir(dir)
     os.system('root -l -b Hadd.C')
     if os.path.isfile(dir+'/Crab.root'):
         return dir+'/Crab.root'
     else:
         return None
Ejemplo n.º 5
0
    def batch_run(self, run, process, compare, batch):
        generator_List = []
        self.__release_List = {}
        self.__jobNumber = {}
        process_List = []
        os.chdir(Configuration.variables["HomeDirectory"] + 'bin/')
        if run != None or process != None:
            # Find release for each generator used
            dir = os.listdir(Configuration.variables["HomeDirectory"] +
                             'test/')
            if run.upper() != "ALL":
                temp_status, temp_output = commands.getstatusoutput(
                    'scramv1 tool info ' + run)
                if "Version" not in temp_output:
                    print "Error obtaining release, please make sure scramv1 is working"
                    self.__release_List[run] = run + '__external'
                    generator_List.append(run)
                for line in temp_output.split('\n'):
                    if "Version" in line:
                        self.__release_List[run] = run + "__" + line.split(
                            ':')[-1].strip(' ')
                        generator_List.append(run)
            if compare.upper() != "ALL" and compare.upper() != run.upper():
                temp_status, temp_output = commands.getstatusoutput(
                    'scramv1 tool info ' + compare)
                if "Version" not in temp_output:
                    print "Error obtaining release, please make sure scramv1 is working"
                    self.__release_List[compare] = compare + '__external'
                    generator_List.append(compare)
                for line in temp_output.split('\n'):
                    if "Version" in line:
                        self.__release_List[
                            compare] = compare + "__" + line.split(
                                ':')[-1].strip(' ')
                        generator_List.append(compare)
            if run.upper() == "ALL" or compare.upper() == "ALL":
                for probe in dir:
                    if 'CVS' in probe:
                        continue
                    if len(probe.split('__')) == 1:
                        continue
                    if '~' in probe:
                        continue
                    if '.cfg' not in probe and '.py' not in probe:
                        continue
                    if probe.split('.')[0].split(
                            '__')[1] not in generator_List:
                        print 'scramv1 tool info ' + probe.split('.')[0].split(
                            '__')[1]
                        temp_status, temp_output = commands.getstatusoutput(
                            'scramv1 tool info ' +
                            probe.split('.')[0].split('__')[1])
                        if "Version" not in temp_output:
                            print "Error obtaining release, please make sure scramv1 is working"
                            self.__release_List[compare] = probe.split(
                                '.')[0].split('__')[1] + '__external'
                            generator_List.append(
                                probe.split('.')[0].split('__')[1])
                        for line in temp_output.split('\n'):
                            if "Version" in line:
                                self.__release_List[probe.split('.')[0].split(
                                    '__')[1]] = probe.split('.')[0].split(
                                        '__')[1] + "__" + line.split(
                                            ':')[-1].strip(' ')
                                generator_List.append(
                                    probe.split('.')[0].split('__')[1])
            for file in dir:
                if 'CVS' in file:
                    continue
                if len(file.split('__')) == 1:
                    continue
                if '~' in file:
                    continue
                if '.cfg' not in file and '.py' not in file:
                    continue
                if process.upper() != "ALL" and process.upper(
                ) not in file.upper():
                    continue
                if file.split('.')[0].split('__')[0] not in process_List:
                    self.__jobNumber[file.split('.')[0].split('__')[0]] = []
                    process_List.append(file.split('.')[0].split('__')[0])
                splitter = file.split('.')[0].split('__')
                if len(splitter) == 3:
                    scratch = Configuration.variables[
                        "HomeDirectory"] + 'DropBox/scratch/' + file.split(
                            '.')[0].split('__')[0] + "/" + file.split(
                                '.')[0].split('__')[1] + '/' + file.split(
                                    '.')[0].split('__')[2] + '-EDM'
                else:
                    scratch = Configuration.variables[
                        "HomeDirectory"] + 'DropBox/scratch/' + file.split(
                            '.')[0].split('__')[0] + "/" + file.split('.')[
                                0].split('__')[1] + '/' + self.__release_List[
                                    file.split('.')[0].split('__')[1]]
                if os.path.exists(scratch + '/') == False:
                    os.makedirs(scratch + '/')
                CMSstatus, CMSoutput = commands.getstatusoutput('$CMSSW_BASE')
                CMS_dir = CMSoutput.split(':')[1].strip(' ')
                if 'CERN' not in batch.upper():
                    tfile = open(
                        Configuration.variables['HomeDirectory'] +
                        '/data/condor.template', 'r')
                    template = Template(tfile.read())
                    tfile.close()
                    cfile = open(scratch + "/condor.submit", 'w')
                    cfile.write(
                        template.safe_substitute(
                            file=scratch + '/cmsrun' +
                            file.split('.')[0].split('__')[0],
                            scratch=scratch))
                    cfile.close()
                    tfile = open(
                        Configuration.variables['HomeDirectory'] +
                        '/data/cmsrun.template', 'r')
                    template = Template(tfile.read())
                    tfile.close()

                    cfile = open(
                        scratch + "/cmsrun" +
                        file.split('.')[0].split('__')[0], 'w')
                    cfile.write(
                        template.safe_substitute(
                            directory=scratch,
                            cfg=Configuration.variables["HomeDirectory"] +
                            'test/' + file,
                            CMSSW=CMS_dir))
                    cfile.close()
                    os.chmod(
                        scratch + "/cmsrun" +
                        file.split('.')[0].split('__')[0], 0755)
                    status, output = commands.getstatusoutput(
                        "condor_submit " + scratch + "/condor.submit")
                    for line in output.split('\n'):
                        if "submitted" in line:
                            self.__jobNumber[file.split('.')[0].split('__')
                                             [0]].append(line.split(' ')[5])
                            jobID = line.split(' ')[5]
                    if status != 0:
                        print file + " wasn't submitted properly"
                        jobID = None
                else:
                    tfile = open(
                        Configuration.variables['HomeDirectory'] +
                        '/interface/cmsrunbsub.template', 'r')
                    template = Template(tfile.read())
                    tfile.close()
                    cfile = open(
                        scratch + "/cmsrunbsub" +
                        file.split('.')[0].split('__')[0], 'w')
                    cfile.write(
                        template.safe_substitute(
                            directory=scratch,
                            cfg=Configuration.variables["HomeDirectory"] +
                            'test/' + file,
                            CMSSW=CMS_dir))
                    cfile.close()
                    os.chmod(
                        scratch + "/cmsrunbsub" +
                        file.split('.')[0].split('__')[0], 0755)
                    os.chdir(scratch)
                    status, output = commands.getstatusoutput(
                        "bsub cmsrunbsub" + file.split('.')[0].split('__')[0])
                    for line in output.split('\n'):
                        if "submitted" in line:
                            self.__jobNumber[file.split('.')[0].split(
                                '__')[0]].append(
                                    line.split(' ')[2].lstrip('<').rstrip('>'))
                            jobID = line.split(' ')[2].lstrip('<').rstrip('>')
                    if status != 0:
                        print file + " wasn't submitted properly"
                        jobID = None
                    os.chdir(Configuration.variables["HomeDirectory"] +
                             '/bin/')
            return jobID
Ejemplo n.º 6
0
 def new_config_run(self, run, process, compare):
     generator_List = []
     self.__release_List = {}
     os.chdir(Configuration.variables["HomeDirectory"]+'bin/')
     if run != None or process != None:
         ######### Find release for each generator used ##########
         dir = os.listdir(Configuration.variables["HomeDirectory"]+'data/')
         temp_dir = os.listdir(Configuration.variables["HomeDirectory"]+'templates/')
         if run.upper() != "ALL":
             temp_status, temp_output =  commands.getstatusoutput('scramv1 tool info '+run )
             if "Version" not in temp_output:
                 #print "Error obtaining "+run+" release, please make sure scramv1 is working"
                 self.__release_List[run] = run+'__external'
                 generator_List.append(run)
             for line in temp_output.split('\n'):
                 if "Version" in line:
                     self.__release_List[run] = run+"__"+line.split(':')[-1].strip(' ')
                     generator_List.append(run)
         if compare.upper() != "ALL" and compare.upper() != run.upper():
             temp_status, temp_output =  commands.getstatusoutput('scramv1 tool info '+compare )
             if "Version" not in temp_output:
                 #print "Error obtaining "+compare+" release, please make sure scramv1 is working"
                 self.__release_List[compare] = compare+'__external'
                 generator_List.append(compare)
             for line in temp_output.split('\n'):
                 if "Version" in line:
                     self.__release_List[compare] = compare+"__"+line.split(':')[-1].strip(' ')
                     generator_List.append(compare)
         if run.upper() == "ALL" or compare.upper() == "ALL":
             for probe in dir:
                 if len(probe.split('__')) == 1:
                     continue 
                 if probe.split('.')[0].split('__')[1] not in generator_List:
                     temp_status, temp_output =  commands.getstatusoutput('scramv1 tool info '+probe.split('.')[0].split('__')[1] )
                     if "Version" not in temp_output:
                         self.__release_List[compare] = probe.split('.')[0].split('__')[1]+'__external'
                         generator_List.append(probe.split('.')[0].split('__')[1])
                     for line in temp_output.split('\n'):
                         if "Version" in line:
                             self.__release_List[probe.split('.')[0].split('__')[1]] = probe.split('.')[0].split('__')[1]+"__"+line.split(':')[-1].strip(' ')
                             generator_List.append(probe.split('.')[0].split('__')[1])
         
         #########  Make cfg files from templates and run #############
         for file in dir:
             if '~' in file or '.cfi' not in file:
                 continue
             if run.upper() != "ALL" and compare.upper() != "ALL" and run.upper() not in file.upper() and compare.upper() not in file.upper():
                 continue
             if process.upper() in file.upper():
                     print "About to run cmsRun", file
                     splitter = file.split('.')[0].split('__')
                     if len(splitter) > 2:
                         directory = file.split('.')[0].split('__')[1]+'/'+file.split('.')[0].split('__')[1]+'__'+file.split('.')[0].split('__')[2]+'EDM'
                     else:
                         directory = file.split('.')[0].split('__')[1]+'/'+self.__release_List[file.split('.')[0].split('__')[1]]
                     if os.path.exists(Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+process+'/'+directory+'/') == False:
                         os.makedirs(Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+process+'/'+directory+'/')
                     scratch = Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+process+'/'+directory+'/'
                     for template in temp_dir:
                         if template.split('.')[0] == file.split('__')[0]:
                             tfile = open (Configuration.variables['HomeDirectory'] + '/templates/'+template ,'r')
                             template = Template(tfile.read())
                             tfile.close()
                             file_reader = open(Configuration.variables['HomeDirectory'] + '/data/'+file, 'r')
                             source_string = file_reader.read()
                             file_reader.close()
                             cfg_file = file.split('.')[0]+'.cfg'
                             cfile = open(scratch+cfg_file, 'w')
                             cfile.write(template.safe_substitute(source = source_string))
                             cfile.close()
                             os.chdir(Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+process+'/'+directory+'/')
                             os.system('cmsRun '+cfg_file+' >& '+file.split('.')[0]+'.log')
                             root_files = 0
                             for data_file in os.listdir(Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+process+'/'+directory+'/'):
                                 if '.root' in data_file or '.log' in data_file:
                                     if len(splitter) > 2:
                                         directory1 = file.split('.')[0].split('__')[1]+'/'+file.split('.')[0].split('__')[1]+'__'+file.split('.')[0].split('__')[2]+'EDM'
                                     else:
                                         directory1 = file.split('.')[0].split('__')[1]+'/'+self.__release_List[file.split('.')[0].split('__')[1]]
                                     if os.path.exists(Configuration.variables['ReleaseDirectory']+directory1+'/'+file.split('.')[0].split('__')[0]+'/') != True:
                                         os.makedirs(Configuration.variables['ReleaseDirectory']+directory1+'/'+file.split('.')[0].split('__')[0]+'/')
                                     shutil.copyfile(Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+process+'/'+directory+'/'+data_file, Configuration.variables['ReleaseDirectory']+directory1+'/'+file.split('.')[0].split('__')[0]+'/'+data_file)
                                     if '.root' in data_file:
                                         root_files = 1
                             os.system('rm '+Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+process+'/'+directory+'/*')
                             if root_files == 0:
                                 print file+" didn't run properly, please check log file"
Ejemplo n.º 7
0
    def Vista_run(self, Site, job):
        #
        self.__jobNumber = [] 
        List = os.listdir(Configuration.variables['HomeDirectory']+'data/')
        Vista_List = []
        process_List = []
        CMSstatus, CMSoutput = commands.getstatusoutput('$CMSSW_BASE')
        CMS_dir = CMSoutput.split(':')[1].strip(' ')
        for file in List:
            if "VISTA" in file.upper() and file not in Vista_List:
                Vista_List.append(file)
        print Vista_List, "Vista"
        Remaining = Vista_List
        head = """source = PoolSource \n
        { \n
        untracked vstring fileNames = \n 
        { \n
        """
        foot ="""  } \n
        } \n """
        while len(Remaining) != 0:
            for run in Remaining:
                if run.split('.')[0].split('__')[0] not in process_List:
                    self.__jobNumber[run.split('.')[0].split('__')[0]] = []
                    process_List.append(run.split('.')[0].split('__')[0])
                if "DBS" in run.upper():
                    stat, out = commands.getstatusoutput('./rssparser --dataset='+run.split('__')[0].strip("-Vista")+' --release='+run.split('__')[2])
                    body_string = ''
                    if '.root' in out:
                        scratch = Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+run.split('__')[0]+'/'+"Vista"+'/'+run.split('__')[2]
                        if os.path.exists(scratch) == False:
                            os.makedirs(scratch)
                        for line in out.split('\n'):
                            if '.root' in line:
                                body_string = body_string + line + '\n'
                        tfile = open (Configuration.variables['HomeDirectory'] + '/templates/runVista.template' ,'r')
                        template = Template(tfile.read())
                        tfile.close()
                        source_string = head + body_string + foot 
                        cfile = open(scratch+"/runVista.cfg", 'w')
                        cfile.write(template.safe_substitute(source = source_string ))
                        cfile.close()
                        if Site.upper() == "FNAL":
                            tfile = open (Configuration.variables['HomeDirectory'] + '/interface/condor.template' ,'r')
                            template = Template(tfile.read())
                            tfile.close()
                            cfile = open(scratch+"/condor.submit", 'w')
                            cfile.write(template.safe_substitute(file = scratch+'/cmsruncondor'+run.split('.')[0].split('__')[0], scratch = scratch))
                            cfile.close()
                            tfile = open (Configuration.variables['HomeDirectory'] + '/interface/cmsruncondor.template' ,'r')
                            template = Template(tfile.read())
                            tfile.close()
                            cfile = open(scratch+"/cmsruncondor"+run.split('.')[0].split('__')[0], 'w')
                            cfile.write(template.safe_substitute(directory=scratch, cfg = scratch+'/runVista.cfg', CMSSW=CMS_dir ))
                            cfile.close()
                            os.chmod(scratch+"/cmsruncondor"+run.split('.')[0].split('__')[0], 0755)
                            status, output = commands.getstatusoutput("condor_submit " + scratch+"/condor.submit")
                            Remaining.remove(run)
                        elif Site.upper() == "CERN":
                            tfile = open (Configuration.variables['HomeDirectory'] + '/interface/cmsrunbsub.template' ,'r')
                            template = Template(tfile.read())
                            tfile.close()
                            cfile = open(scratch+"/cmsrunbsub"+run.split('.')[0].split('__')[0], 'w')
                            cfile.write(template.safe_substitute(directory=scratch, cfg = scratch+'/runVista.cfg', CMSSW=CMS_dir ))
                            cfile.close()
                            os.chmod(scratch+"/cmsrunbsub"+run.split('.')[0].split('__')[0], 0755)
                            status, output = commands.getstatusoutput("bsub " + scratch+"/cmsrunbsub"+run.split('.')[0].split('__')[0])
                            for line in output.split('\n'):
                                if "submitted" in line:
                                    self.__jobNumber[run.split('.')[0].split('__')[0]].append(line.split(' ')[2].lstrip('<').rstrip('>'))
                            if status != 0:
                                print file + " wasn't submitted properly"
                            Remaining.remove(run)
                        else:
                            print run, job, "DBS"
                            os.chdir(scratch)
                            status, output = commands.getstatusoutput("cmsRun runVista.cfg >& "+run.split('.')[0].split('__')[0]+'.log')
                            os.chdir(Configuration.variables['HomeDirectory'])
                    elif 'ERROR' in out.upper():
                        print "Error reading DBS for "+run
                        Remaining.remove(run)
                    else:
                        time.sleep(40)
                else:
                    file = open(Configuration.variables['HomeDirectory']+'data/'+run, 'r')
                    source_string = file.read()
                    file.close()
                    scratch = Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+run.split('__')[0]+'/'+"Vista"+'/'+run.split('__')[2]
                    tfile = open (Configuration.variables['HomeDirectory'] + '/tempaltes/runVista.template' ,'r')
                    template = Template(tfile.read())
                    tfile.close()
                    cfile = open(scratch+"/runVista.cfg", 'w')
                    cfile.write(template.safe_substitute(source = source_string ))
                    cfile.close()
                    if job.upper() != "BATCH":
                        print run, job
                        os.chdir(scratch)
                        status, output = commands.getstatusoutput("cmsRun runVista.cfg >& "+run.split('.')[0].split('__')[0]+'.log')
                        os.chdir(Configuration.variables['HomeDirectory'])
                    else:
                        if Site.upper() == "FNAL":
                            tfile = open (Configuration.variables['HomeDirectory'] + '/interface/condor.template' ,'r')
                            template = Template(tfile.read())
                            tfile.close()
                            cfile = open(scratch+"/condor.submit", 'w')
                            cfile.write(template.safe_substitute(file = scratch+'/cmsrun'+run.split('.')[0].split('__')[0], scratch = scratch))
                            cfile.close()
                            tfile = open (Configuration.variables['HomeDirectory'] + '/interface/cmsrun.template' ,'r')
                            template = Template(tfile.read())
                            tfile.close()
                            cfile = open(scratch+"/cmsrun"+run.split('.')[0].split('__')[0], 'w')
                            cfile.write(template.safe_substitute(directory=scratch, cfg = scratch+'/runVista.cfg', CMSSW=CMS_dir ))
                            cfile.close()
                            os.chmod(scratch+"/cmsrun"+run.split('.')[0].split('__')[0], 0755)
                            status, output = commands.getstatusoutput("condor_submit " + scratch+"/condor.submit")
                            print run, "BATCH"
                            for line in output.split('\n'):
                                if "submitted" in line:
                                    self.__jobNumber[run.split('.')[0].split('__')[0]].append(line.split(' ')[5])
                        if Site.upper() == "CERN":
                            tfile = open (Configuration.variables['HomeDirectory'] + '/interface/cmsrunbsub.template' ,'r')
                            template = Template(tfile.read())
                            tfile.close()
                            cfile = open(scratch+"/cmsrunbsub"+run.split('.')[0].split('__')[0], 'w')
                            cfile.write(template.safe_substitute(directory=scratch, cfg = scratch+'/runVista.cfg', CMSSW=CMS_dir ))
                            cfile.close()
                            os.chmod(scratch+"/cmsrunbsub"+run.split('.')[0].split('__')[0], 0755)
                            status, output = commands.getstatusoutput("bsub " + scratch+"/cmsrunbsub"+run.split('.')[0].split('__')[0])
                            for line in output.split('\n'):
                                if "submitted" in line:
                                    self.__jobNumber[run.split('.')[0].split('__')[0]].append(line.split(' ')[2].lstrip('<').rstrip('>'))
                            if status != 0:
                                print file + " wasn't submitted properly"

                    Remaining.remove(run)   
Ejemplo n.º 8
0
    def new_batch_run(self, run, process, compare, batch):
        generator_List = []
        self.__release_List = {}
        self.__jobNumber = {}
        process_List = []
        os.chdir(Configuration.variables["HomeDirectory"]+'bin/')
        if run != None or process != None:
            ####### Find release for each generator used ##########
            dir = os.listdir(Configuration.variables["HomeDirectory"]+'data/')
            temp_dir = os.listdir(Configuration.variables["HomeDirectory"]+'templates/')
            if run.upper() != "ALL":
                temp_status, temp_output =  commands.getstatusoutput('scramv1 tool info '+run )
                if "Version" not in temp_output:
                    #print "Error obtaining release, please make sure scramv1 is working"
                    self.__release_List[run] = run+'__external'
                    generator_List.append(run)
                for line in temp_output.split('\n'):
                    if "Version" in line:
                        self.__release_List[run] = run+"__"+line.split(':')[-1].strip(' ')
                        generator_List.append(run)
            if compare.upper() != "ALL" and compare.upper() != run.upper():
                temp_status, temp_output =  commands.getstatusoutput('scramv1 tool info '+compare )
                if "Version" not in temp_output:
                    #print "Error obtaining release, please make sure scramv1 is working"
                    self.__release_List[compare] = compare+'__external'
                    generator_List.append(compare)
                for line in temp_output.split('\n'):
                    if "Version" in line:
                        self.__release_List[compare] = compare+"__"+line.split(':')[-1].strip(' ')
                        generator_List.append(compare)
            if run.upper() == "ALL" or compare.upper() == "ALL":
                for probe in dir:
                    if probe.split('.')[0].split('__')[1] not in generator_List:
                        #print 'scramv1 tool info '+probe.split('.')[0].split('__')[1] 
                        temp_status, temp_output =  commands.getstatusoutput('scramv1 tool info '+probe.split('.')[0].split('__')[1] )
                        if "Version" not in temp_output:
                            #print "Error obtaining release, please make sure scramv1 is working"
                            self.__release_List[compare] = probe.split('.')[0].split('__')[1]+'__external'
                            generator_List.append(probe.split('.')[0].split('__')[1])
                        for line in temp_output.split('\n'):
                            if "Version" in line:
                                self.__release_List[probe.split('.')[0].split('__')[1]] = probe.split('.')[0].split('__')[1]+"__"+line.split(':')[-1].strip(' ')
                                generator_List.append(probe.split('.')[0].split('__')[1])


            ########  Make cfgs from templates and run ##############
            for file in dir:
                if '~' in file or '.cfi' not in file:
                    continue
                if 'cmsrun'  in file:
                    continue
                if 'condor' in file:
                    continue
                #print file, 'file'
                if process.upper() != "ALL" and process.upper() not in file.upper():
                    continue
                if file.split('.')[0].split('__')[0] not in process_List:
                    self.__jobNumber[file.split('.')[0].split('__')[0]] = []
                    process_List.append(file.split('.')[0].split('__')[0])
                splitter = file.split('.')[0].split('__')
                if len(splitter) > 2:
                    scratch = Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+splitter[0]+"/"+splitter[1]+'/'+splitter[2]+'EDM/'
                else:
                    scratch = Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+file.split('.')[0].split('__')[0]+"/"+file.split('.')[0].split('__')[1]+'/'+self.__release_List[file.split('.')[0].split('__')[1]]+'/'
                if os.path.exists(scratch+'/') == False:
                    os.makedirs(scratch+'/')
               
                CMSstatus, CMSoutput = commands.getstatusoutput('$CMSSW_BASE')
                CMS_dir = CMSoutput.split(':')[1].strip(' ')
                for temp in temp_dir:
                    print temp, 'template'
                    if temp.split('.')[0] == file.split('__')[0]:
                        tfile = open (Configuration.variables['HomeDirectory'] + '/templates/'+temp ,'r')
                        template = Template(tfile.read())
                        tfile.close()
                        file_reader = open(Configuration.variables['HomeDirectory'] + '/data/'+file, 'r')
                        source_string = file_reader.read()
                        file_reader.close()
                        cfg_filename = file.split('.')[0]+'.cfg'
                        cfile = open(scratch+cfg_filename, 'w')
                        cfile.write(template.safe_substitute(source = source_string))
                        cfile.close()
                        if batch.upper() == "FNAL":
                            tfile = open (Configuration.variables['HomeDirectory'] + '/interface/condor.template' ,'r')
                            template = Template(tfile.read())
                            tfile.close()
                            cfile = open(scratch+"/condor.submit", 'w')
                            cfile.write(template.safe_substitute(file = scratch+'/cmsruncondor'+file.split('.')[0].split('__')[0], scratch = scratch))
                            cfile.close()
                            tfile = open (Configuration.variables['HomeDirectory'] + '/interface/cmsruncondor.template' ,'r')
                            template = Template(tfile.read())
                            tfile.close()
                            cfile = open(scratch+"/cmsruncondor"+file.split('.')[0].split('__')[0], 'w')
                            cfile.write(template.safe_substitute(directory=scratch, cfg = scratch+'/'+cfg_filename, CMSSW=CMS_dir ))
                            cfile.close()
                            os.chmod(scratch+"/cmsruncondor"+file.split('.')[0].split('__')[0], 0755)
                            status, output = commands.getstatusoutput("condor_submit " + scratch+"/condor.submit")
                            for line in output.split('\n'):
                                if "submitted" in line:
                                    self.__jobNumber[file.split('.')[0].split('__')[0]].append(line.split(' ')[5])
                            if status != 0:
                                print file + " wasn't submitted properly"
                        if batch.upper() == "CERN":
                            tfile = open (Configuration.variables['HomeDirectory'] + '/interface/cmsrunbsub.template' ,'r')
                            template = Template(tfile.read())
                            tfile.close()
                            cfile = open(scratch+"/cmsrunbsub"+file.split('.')[0].split('__')[0], 'w')
                            cfile.write(template.safe_substitute(directory=scratch, cfg = scratch+'/'+cfg_filename, CMSSW=CMS_dir ))
                            cfile.close()
                            os.chmod(scratch+"/cmsrunbsub"+file.split('.')[0].split('__')[0], 0755)
                            status, output = commands.getstatusoutput("bsub " + scratch+"/cmsrunbsub"+file.split('.')[0].split('__')[0])
                            for line in output.split('\n'):
                                if "submitted" in line:
                                    self.__jobNumber[file.split('.')[0].split('__')[0]].append(line.split(' ')[2].lstrip('<').rstrip('>'))
                            if status != 0:
                                print file + " wasn't submitted properly"
Ejemplo n.º 9
0
 def batch_run(self, run, process, compare, batch):
     generator_List = []
     self.__release_List = {}
     self.__jobNumber = {}
     process_List = []
     os.chdir(Configuration.variables["HomeDirectory"]+'bin/')
     if run != None or process != None:
         # Find release for each generator used
         dir = os.listdir(Configuration.variables["HomeDirectory"]+'test/')
         if run.upper() != "ALL":
             temp_status, temp_output =  commands.getstatusoutput('scramv1 tool info '+run )
             if "Version" not in temp_output:
                 print "Error obtaining release, please make sure scramv1 is working"
                 self.__release_List[run] = run+'__external'
                 generator_List.append(run)
             for line in temp_output.split('\n'):
                 if "Version" in line:
                     self.__release_List[run] = run+"__"+line.split(':')[-1].strip(' ')
                     generator_List.append(run)
         if compare.upper() != "ALL" and compare.upper() != run.upper():
             temp_status, temp_output =  commands.getstatusoutput('scramv1 tool info '+compare )
             if "Version" not in temp_output:
                 print "Error obtaining release, please make sure scramv1 is working"
                 self.__release_List[compare] = compare+'__external'
                 generator_List.append(compare)
             for line in temp_output.split('\n'):
                 if "Version" in line:
                     self.__release_List[compare] = compare+"__"+line.split(':')[-1].strip(' ')
                     generator_List.append(compare)
         if run.upper() == "ALL" or compare.upper() == "ALL":
             for probe in dir:
                 if 'CVS' in probe:
                     continue
                 if len(probe.split('__')) == 1:
                     continue
                 if '~' in probe:
                     continue
                 if '.cfg' not in probe and '.py' not in probe:
                     continue
                 if probe.split('.')[0].split('__')[1] not in generator_List:
                     print 'scramv1 tool info '+probe.split('.')[0].split('__')[1] 
                     temp_status, temp_output =  commands.getstatusoutput('scramv1 tool info '+probe.split('.')[0].split('__')[1] )
                     if "Version" not in temp_output:
                         print "Error obtaining release, please make sure scramv1 is working"
                         self.__release_List[compare] = probe.split('.')[0].split('__')[1]+'__external'
                         generator_List.append(probe.split('.')[0].split('__')[1])
                     for line in temp_output.split('\n'):
                         if "Version" in line:
                             self.__release_List[probe.split('.')[0].split('__')[1]] = probe.split('.')[0].split('__')[1]+"__"+line.split(':')[-1].strip(' ')
                             generator_List.append(probe.split('.')[0].split('__')[1])
         for file in dir:
             if 'CVS' in file:
                 continue
             if len(file.split('__')) == 1:
                 continue
             if '~' in file:
                 continue
             if '.cfg' not in file and '.py' not in file:
                 continue
             if process.upper() != "ALL" and process.upper() not in file.upper():
                 continue
             if file.split('.')[0].split('__')[0] not in process_List:
                 self.__jobNumber[file.split('.')[0].split('__')[0]] = []
                 process_List.append(file.split('.')[0].split('__')[0])
             splitter = file.split('.')[0].split('__')
             if len(splitter) == 3:
                 scratch = Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+file.split('.')[0].split('__')[0]+"/"+file.split('.')[0].split('__')[1]+'/'+file.split('.')[0].split('__')[2]+'-EDM'
             else:
                 scratch = Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+file.split('.')[0].split('__')[0]+"/"+file.split('.')[0].split('__')[1]+'/'+self.__release_List[file.split('.')[0].split('__')[1]]
             if os.path.exists(scratch+'/') == False:
                 os.makedirs(scratch+'/')
             CMSstatus, CMSoutput = commands.getstatusoutput('$CMSSW_BASE')
             CMS_dir = CMSoutput.split(':')[1].strip(' ')
             if 'CERN' not in batch.upper():
                 tfile = open (Configuration.variables['HomeDirectory'] + '/data/condor.template' ,'r')
                 template = Template(tfile.read())
                 tfile.close()
                 cfile = open(scratch+"/condor.submit", 'w')
                 cfile.write(template.safe_substitute(file = scratch+'/cmsrun'+file.split('.')[0].split('__')[0], scratch = scratch))
                 cfile.close()
                 tfile = open (Configuration.variables['HomeDirectory'] + '/data/cmsrun.template' ,'r')
                 template = Template(tfile.read())
                 tfile.close()
                 
                 cfile = open(scratch+"/cmsrun"+file.split('.')[0].split('__')[0], 'w')
                 cfile.write(template.safe_substitute(directory=scratch, cfg = Configuration.variables["HomeDirectory"]+'test/'+file, CMSSW=CMS_dir ))
                 cfile.close()
                 os.chmod(scratch+"/cmsrun"+file.split('.')[0].split('__')[0], 0755)
                 status, output = commands.getstatusoutput("condor_submit " + scratch+"/condor.submit")
                 for line in output.split('\n'):
                     if "submitted" in line:
                         self.__jobNumber[file.split('.')[0].split('__')[0]].append(line.split(' ')[5])
                         jobID = line.split(' ')[5]
                 if status != 0:
                     print file + " wasn't submitted properly"
                     jobID = None
             else:
                 tfile = open (Configuration.variables['HomeDirectory'] + '/interface/cmsrunbsub.template' ,'r')
                 template = Template(tfile.read())
                 tfile.close()
                 cfile = open(scratch+"/cmsrunbsub"+file.split('.')[0].split('__')[0], 'w')
                 cfile.write(template.safe_substitute(directory=scratch, cfg = Configuration.variables["HomeDirectory"]+'test/'+file, CMSSW=CMS_dir ))
                 cfile.close()
                 os.chmod(scratch+"/cmsrunbsub"+file.split('.')[0].split('__')[0], 0755)
                 os.chdir(scratch)
                 status, output = commands.getstatusoutput("bsub cmsrunbsub"+file.split('.')[0].split('__')[0])
                 for line in output.split('\n'):
                     if "submitted" in line:
                         self.__jobNumber[file.split('.')[0].split('__')[0]].append(line.split(' ')[2].lstrip('<').rstrip('>'))
                         jobID = line.split(' ')[2].lstrip('<').rstrip('>')
                 if status != 0:
                     print file + " wasn't submitted properly"
                     jobID = None
                 os.chdir(Configuration.variables["HomeDirectory"]+'/bin/')
         return jobID