예제 #1
0
    def NewPublisher(self, run, process, compare, all, job, location, grid,
                     jobid):
        os.chdir(Configuration.variables["HomeDirectory"] + '/bin/')
        if job.upper() == "BATCH":
            if location.upper() != 'CERN':
                condor_status = 0
                while condor_status == 0:
                    Unfinished_Job = 0
                    #for jobid in self.__jobNumber[process]:
                    status, output = commands.getstatusoutput('condor_q ' +
                                                              jobid)
                    if jobid in output:
                        Unfinished_Job = Unfinished_Job + 1
                    if Unfinished_Job == 0:
                        condor_status = 1
                        print "All jobs for " + process + " are done"
                    if Unfinished_Job > 0:
                        time.sleep(15)
            else:
                bsub_status = 0
                while bsub_status == 0:
                    Unfinished_Job = 0
                    status, output = commands.getstatusoutput('bjobs ')
                    if 'No unfinished job' in output:
                        bsub_status = 1
                        print "All jobs for " + process + " are done"
                    else:
                        time.sleep(15)

            for generator in os.listdir(
                    Configuration.variables["HomeDirectory"] +
                    'DropBox/scratch/' + process):
                for sub_gen in os.listdir(
                        Configuration.variables["HomeDirectory"] +
                        'DropBox/scratch/' + process + '/' + generator + '/'):
                    for subdir in os.listdir(
                            Configuration.variables["HomeDirectory"] +
                            'DropBox/scratch/' + process + '/' + generator +
                            '/' + sub_gen + '/'):
                        if '.root' in subdir or '.log' in subdir:
                            if os.path.isdir(Configuration.
                                             variables["ReleaseDirectory"] +
                                             generator + '/' + sub_gen + '/' +
                                             process + '/') == False:
                                os.makedirs(Configuration.
                                            variables["ReleaseDirectory"] +
                                            generator + '/' + sub_gen + '/' +
                                            process + '/')
                            shutil.copyfile(
                                Configuration.variables["HomeDirectory"] +
                                'DropBox/scratch/' + process + '/' +
                                generator + '/' + sub_gen + '/' + subdir,
                                Configuration.variables["ReleaseDirectory"] +
                                generator + '/' + sub_gen + '/' + process +
                                '/' + subdir)
                    #os.system('rm '+Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+process+'/'+generator+'/'+sub_gen+'/*')

        if grid.upper() == 'TRUE':
            cooked_crab = []
            sub_jobs = 0
            total_jobs = 0
            jobs_done = 0
            while len(cooked_crab) < 1:
                if sub_jobs > 3:
                    print "More than 3 jobs in queue, sleeping for 3 minutes"
                    print "Jobs left: ", int(
                        total_jobs
                    ) - jobs_done - sub_jobs, "Out of: ", total_jobs
                    time.sleep(180)
                else:
                    print "Sleeping for one minute, waiting for CRAB"
                    if total_jobs != 0:
                        print "Jobs left: ", int(
                            total_jobs
                        ) - jobs_done - sub_jobs, "Out of: ", total_jobs
                    time.sleep(60)

                #for jobid in self.__crab_List:
                if jobid in cooked_crab:
                    continue
                crab_status, crab_output = commands.getstatusoutput(
                    'crab -status -c ' + jobid)
                if crab_status != 0:
                    print "Something went wrong here."
                else:
                    total_jobs = 0
                    jobs_done = 0
                    sub_jobs = 0
                    for line in crab_output.split('\n'):
                        if '>>>' in line:
                            if 'TOTAL JOBS' in line.upper():
                                total_jobs = line.split(' ')[1]
                                #print total_jobs, "Total Jobs"
                        if 'CLEARED' in line.upper(
                        ) or 'DONE' in line.upper() and '>>>' not in line:
                            jobs_done = jobs_done + 1
                        if 'SUBMITTED' in line.upper():
                            sub_jobs = sub_jobs + 1

                    if int(total_jobs) == 0:
                        print "Crab must not have been submitted properly"
                        cooked_crab.append(jobid)
                    elif int(total_jobs) == jobs_done:
                        print "Crab job: " + jobid + " done!"
                        out_stat, out_out = commands.getstatusoutput(
                            'crab -getoutput -c ' + jobid)
                        out_file = self.CrabCrusher(jobid + '/res/')
                        cooked_crab.append(jobid)
                        if out_file == None:
                            print "No root files in scracth"
                        else:
                            proc = out_file.split('/')[-5]
                            gen = out_file.split('/')[-4]
                            rel = out_file.split('/')[-3]
                            if os.path.isdir(Configuration.
                                             variables['ReleaseDirectory'] +
                                             gen + '/' + rel + '/' + proc +
                                             '/') == False:
                                os.makedirs(Configuration.
                                            variables['ReleaseDirectory'] +
                                            gen + '/' + rel + '-preGen/' +
                                            proc + '/')
                            shutil.copyfile(
                                out_file,
                                Configuration.variables['ReleaseDirectory'] +
                                gen + '/' + rel + '-preGen/' + proc + '/')

        print "Begin Publishing", process
        ### Publishes and compares, may be able to replace 'data' too ###
        gen_List = os.listdir(Configuration.variables["ReleaseDirectory"])
        if compare.upper() == "SAME" or compare.upper() == "ALL":
            for List in gen_List:
                if compare.upper(
                ) == "SAME" and List not in run and run.upper() != "ALL":
                    continue
                if 'CVS' in List:
                    continue
                sub_dir = os.listdir(
                    Configuration.variables["ReleaseDirectory"] + List)
                for i in range(len(sub_dir) - 1):
                    for j in range(i + 1, len(sub_dir)):
                        if sub_dir[i] == sub_dir[j]:
                            continue
                        if os.path.isdir(
                                Configuration.variables["ReleaseDirectory"] +
                                List + '/' + sub_dir[i] + '/' +
                                process) == False:
                            continue
                        if os.path.isdir(
                                Configuration.variables["ReleaseDirectory"] +
                                List + '/' + sub_dir[j] + '/' +
                                process) == False:
                            continue
                        web_dir = Configuration.variables[
                            "WebDirectory"] + sub_dir[i] + '/' + sub_dir[
                                j] + '/' + process + '/'
                        if os.path.isdir(web_dir) == True:
                            continue
                        if os.path.isdir(
                                Configuration.variables["WebDirectory"] +
                                sub_dir[j] + '/' + sub_dir[i] + '/' + process +
                                '/') and os.listdir(
                                    Configuration.variables["WebDirectory"] +
                                    sub_dir[j] + '/' + sub_dir[i] + '/' +
                                    process + '/') < 2:
                            continue
                        for rel_file in os.listdir(
                                Configuration.variables["ReleaseDirectory"] +
                                List + '/' + sub_dir[i] + '/' + process):
                            if '.root' in rel_file:
                                for ref_file in os.listdir(
                                        Configuration.
                                        variables["ReleaseDirectory"] + List +
                                        '/' + sub_dir[j] + '/' + process):
                                    if '.root' in ref_file:
                                        if os.path.isdir(web_dir) == False:
                                            os.makedirs(web_dir)

                                        Publisher.StaticWeb().plot(
                                            Configuration.
                                            variables["ReleaseDirectory"] +
                                            List + '/' + sub_dir[i] + '/' +
                                            process + '/' + rel_file,
                                            Configuration.
                                            variables["ReleaseDirectory"] +
                                            List + '/' + sub_dir[j] + '/' +
                                            process + '/' + ref_file, web_dir,
                                            process + '--' + sub_dir[i] +
                                            '--' + sub_dir[j], '', sub_dir[i],
                                            sub_dir[j])
                                        Publisher.StaticWeb().index(
                                            sub_dir[i], sub_dir[j], process,
                                            web_dir)

        # Compares new release to old ones.... #
        if compare.upper() != "SAME":  # or compare.upper() == "ALL":
            for i in range(len(gen_List) - 1):
                for j in range(i + 1, len(gen_List)):
                    if compare.upper() != "ALL":
                        if compare.upper(
                        ) not in gen_List[i] and compare.upper() != "OPPOSITE":
                            continue
                        if compare.upper(
                        ) not in gen_List[j] and compare.upper() != "OPPOSITE":
                            continue
                        if run.upper() != "ALL":
                            if run.upper() not in gen_List[i] or run.upper(
                            ) not in gen_List[j]:
                                continue
                        if gen_List[i] == 'CVS' or gen_List[j] == 'CVS':
                            continue
                    rel_List = os.listdir(
                        Configuration.variables["ReleaseDirectory"] +
                        gen_List[i])
                    ref_List = os.listdir(
                        Configuration.variables["ReleaseDirectory"] +
                        gen_List[j])
                    for rel_dir in rel_List:
                        if os.path.isdir(
                                Configuration.variables["ReleaseDirectory"] +
                                gen_List[i] + '/' + rel_dir + '/' +
                                process) == False:
                            continue
                        for ref_dir in ref_List:
                            if os.path.isdir(Configuration.
                                             variables["ReleaseDirectory"] +
                                             gen_List[j] + '/' + ref_dir +
                                             '/' + process) == False:
                                continue
                            if ref_dir == rel_dir:
                                continue
                            if all.upper() == "FALSE":
                                if ref_dir != self.__release_List[gen_List[
                                        j]] and ref_dir != self.__release_List[gen_List[
                                            i]] and rel_dir != self.__release_List[
                                                gen_List[
                                                    j]] and rel_dir != self.__release_List[
                                                        gen_List[i]]:
                                    continue
                            web_dir = Configuration.variables[
                                "WebDirectory"] + rel_dir + '/' + ref_dir + '/' + process + '/'
                            if os.path.exists(
                                    web_dir) and os.listdir(web_dir) < 2:
                                continue
                            if os.path.exists(
                                    Configuration.variables["WebDirectory"] +
                                    ref_dir + '/' + rel_dir + '/' + process +
                                    '/'):
                                continue
                            rel_loc = Configuration.variables[
                                "ReleaseDirectory"] + gen_List[
                                    i] + '/' + rel_dir + '/' + process + '/'
                            ref_loc = Configuration.variables[
                                "ReleaseDirectory"] + gen_List[
                                    j] + '/' + ref_dir + '/' + process + '/'
                            print gen_List[i], gen_List[j]
                            for rel_file in os.listdir(rel_loc):
                                if '.root' in rel_file:
                                    for ref_file in os.listdir(ref_loc):
                                        if '.root' in ref_file:
                                            if os.path.isdir(web_dir) == False:
                                                os.makedirs(web_dir)

                                            Publisher.StaticWeb().plot(
                                                rel_loc + '/' + rel_file,
                                                ref_loc + '/' + ref_file,
                                                web_dir, process + '--' +
                                                rel_dir + '--' + ref_dir, '',
                                                rel_dir, ref_dir)
                                            Publisher.StaticWeb().index(
                                                rel_dir, ref_dir, process,
                                                web_dir)
예제 #2
0
    def NewPublisher(self, run, process, compare, all, job):
        os.chdir(Configuration.variables["HomeDirectory"]+'/bin/')
        if job.upper() != "LOCAL":
            condor_status = 0
            while condor_status == 0:
                Unfinished_Job = 0
                for jobid in self.__jobNumber[process]:
                    status, output = commands.getstatusoutput('condor_q '+jobid)
                    if jobid in output:
                        Unfinished_Job = Unfinished_Job + 1 
                if Unfinished_Job == 0:
                    condor_status = 1
                    print "All jobs for " +process+ " are done" 
                if Unfinished_Job > 0:
                    time.sleep(15)
            for generator in os.listdir(Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+process):
                for sub_gen in os.listdir(Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+process+'/'+generator+'/'):
                    for subdir in os.listdir(Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+process+'/'+generator+'/'+sub_gen+'/'):
                        if '.root' in subdir or '.log' in subdir:
                            if os.path.isdir(Configuration.variables['ReleaseDirectory']+generator+'/'+sub_gen+'/'+process+'/') == False:
                                os.makedirs(Configuration.variables['ReleaseDirectory']+generator+'/'+sub_gen+'/'+process+'/')
                            shutil.copyfile(Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+process+'/'+generator+'/'+sub_gen+'/'+subdir, Configuration.variables['ReleaseDirectory']+generator+'/'+sub_gen+'/'+process+'/'+subdir)
                    #os.system('rm '+Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+process+'/'+generator+'/'+sub_gen+'/*')
                
        #print "Begin Publishing" , process
        ### Publishes and compares, may be able to replace 'data' too ###
        gen_List = os.listdir(Configuration.variables['ReleaseDirectory'])
        if compare.upper() == "SAME" or compare.upper() == "ALL":
            for List in gen_List:
                if compare.upper() == "SAME" and List not in run and run.upper() != "ALL":
                    continue
                sub_dir = os.listdir(Configuration.variables['ReleaseDirectory']+List)
                for i in range(len(sub_dir) - 1):
                    for j in range(i+1, len(sub_dir)):
                        if sub_dir[i] == sub_dir[j]:
                            continue
                        if os.path.isdir(Configuration.variables['ReleaseDirectory']+List+'/'+sub_dir[i]+'/'+process) == False:
                            continue
                        if os.path.isdir(Configuration.variables['ReleaseDirectory']+List+'/'+sub_dir[j]+'/'+process) == False:
                            continue
                        web_dir = Configuration.variables["WebDirectory"]+sub_dir[i]+'/'+sub_dir[j]+'/'+process+'/'
                        if os.path.isdir(web_dir) == True:
                            continue
                        if os.path.isdir(Configuration.variables["WebDirectory"]+sub_dir[j]+'/'+sub_dir[i]+'/'+process+'/') and os.listdir(Configuration.variables["WebDirectory"]+sub_dir[j]+'/'+sub_dir[i]+'/'+process+'/') < 2:
                            continue 
                        for rel_file in os.listdir(Configuration.variables['ReleaseDirectory']+List+'/'+sub_dir[i]+'/'+process):
                            if '.root' in rel_file:
                                for ref_file in os.listdir(Configuration.variables['ReleaseDirectory']+List+'/'+sub_dir[j]+'/'+process):
                                    if '.root' in ref_file:
                                        if os.path.isdir(web_dir) == False:
                                            os.makedirs(web_dir)
                                        
                                        Publisher.StaticWeb().plot(Configuration.variables['ReleaseDirectory']+List+'/'+sub_dir[i]+'/'+process+'/'+rel_file, Configuration.variables['ReleaseDirectory']+List+'/'+sub_dir[j]+'/'+process+'/'+ref_file, web_dir, process+'--'+sub_dir[i]+'--'+sub_dir[j], '', sub_dir[i], sub_dir[j])
                                        Publisher.StaticWeb().index(sub_dir[i], sub_dir[j], process, web_dir)
                                              

        # Compares new release to old ones.... #
        if compare.upper() != "SAME":# or compare.upper() == "ALL":
            for i in range(len(gen_List) - 1):
                for j in range(i+1,len(gen_List)):
                    if compare.upper() != "ALL":
                        if compare.upper() not in gen_List[i] and compare.upper() != "OPPOSITE":
                            continue
                        if compare.upper() not in gen_List[j] and compare.upper() != "OPPOSITE":
                            continue
                        if run.upper() != "ALL":
                            if run.upper() not in gen_List[i] or run.upper() not in gen_List[j]:
                                continue
                    rel_List = os.listdir(Configuration.variables['ReleaseDirectory']+gen_List[i])
                    ref_List = os.listdir(Configuration.variables['ReleaseDirectory']+gen_List[j])
                    for rel_dir in rel_List:
                        if os.path.isdir(Configuration.variables['ReleaseDirectory']+gen_List[i]+'/'+rel_dir+'/'+process) == False:
                            continue
                        for ref_dir in ref_List:
                            if os.path.isdir(Configuration.variables['ReleaseDirectory']+gen_List[j]+'/'+ref_dir+'/'+process) == False:
                                continue
                            if ref_dir == rel_dir:
                                continue 
                            if all.upper() == "FALSE":
                                if ref_dir != self.__release_List[gen_List[j]] and ref_dir != self.__release_List[gen_List[i]] and rel_dir != self.__release_List[gen_List[j]] and rel_dir != self.__release_List[gen_List[i]]:
                                    continue
                            web_dir = Configuration.variables["WebDirectory"]+rel_dir+'/'+ref_dir+'/'+process+'/'
                            if os.path.exists(web_dir) and os.listdir(web_dir) < 2:
                                continue
                            if os.path.exists(Configuration.variables["WebDirectory"]+ref_dir+'/'+rel_dir+'/'+process+'/'):
                                continue
                            rel_loc = Configuration.variables['ReleaseDirectory']+gen_List[i]+'/'+rel_dir+ '/' + process+'/'
                            ref_loc = Configuration.variables['ReleaseDirectory']+gen_List[j]+'/'+ref_dir+ '/' + process+'/'
                            #print gen_List[i], gen_List[j]
                            for rel_file in os.listdir(rel_loc):
                                if '.root' in rel_file:
                                    for ref_file in os.listdir(ref_loc):
                                        if '.root' in ref_file:
                                            if os.path.isdir(web_dir) == False:
                                                os.makedirs(web_dir)
                                            
                                            Publisher.StaticWeb().plot(rel_loc+'/'+rel_file, ref_loc+'/'+ref_file, web_dir, process+'--'+rel_dir+'--'+ref_dir, '', rel_dir, ref_dir)
                                            Publisher.StaticWeb().index(rel_dir, ref_dir, process, web_dir)