def NewPublisher(self, run, process, compare, all, job, location, grid,
jobid):
os.chdir(Configuration.variables["HomeDirectory"] + '/bin/')
if job.upper() == "BATCH":
if location.upper() != 'CERN':
condor_status = 0
while condor_status == 0:
Unfinished_Job = 0
#for jobid in self.__jobNumber[process]:
status, output = commands.getstatusoutput('condor_q ' +
jobid)
if jobid in output:
Unfinished_Job = Unfinished_Job + 1
if Unfinished_Job == 0:
condor_status = 1
print "All jobs for " + process + " are done"
if Unfinished_Job > 0:
time.sleep(15)
else:
bsub_status = 0
while bsub_status == 0:
Unfinished_Job = 0
status, output = commands.getstatusoutput('bjobs ')
if 'No unfinished job' in output:
bsub_status = 1
print "All jobs for " + process + " are done"
else:
time.sleep(15)
for generator in os.listdir(
Configuration.variables["HomeDirectory"] +
'DropBox/scratch/' + process):
for sub_gen in os.listdir(
Configuration.variables["HomeDirectory"] +
'DropBox/scratch/' + process + '/' + generator + '/'):
for subdir in os.listdir(
Configuration.variables["HomeDirectory"] +
'DropBox/scratch/' + process + '/' + generator +
'/' + sub_gen + '/'):
if '.root' in subdir or '.log' in subdir:
if os.path.isdir(Configuration.
variables["ReleaseDirectory"] +
generator + '/' + sub_gen + '/' +
process + '/') == False:
os.makedirs(Configuration.
variables["ReleaseDirectory"] +
generator + '/' + sub_gen + '/' +
process + '/')
shutil.copyfile(
Configuration.variables["HomeDirectory"] +
'DropBox/scratch/' + process + '/' +
generator + '/' + sub_gen + '/' + subdir,
Configuration.variables["ReleaseDirectory"] +
generator + '/' + sub_gen + '/' + process +
'/' + subdir)
#os.system('rm '+Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+process+'/'+generator+'/'+sub_gen+'/*')
if grid.upper() == 'TRUE':
cooked_crab = []
sub_jobs = 0
total_jobs = 0
jobs_done = 0
while len(cooked_crab) < 1:
if sub_jobs > 3:
print "More than 3 jobs in queue, sleeping for 3 minutes"
print "Jobs left: ", int(
total_jobs
) - jobs_done - sub_jobs, "Out of: ", total_jobs
time.sleep(180)
else:
print "Sleeping for one minute, waiting for CRAB"
if total_jobs != 0:
print "Jobs left: ", int(
total_jobs
) - jobs_done - sub_jobs, "Out of: ", total_jobs
time.sleep(60)
#for jobid in self.__crab_List:
if jobid in cooked_crab:
continue
crab_status, crab_output = commands.getstatusoutput(
'crab -status -c ' + jobid)
if crab_status != 0:
print "Something went wrong here."
else:
total_jobs = 0
jobs_done = 0
sub_jobs = 0
for line in crab_output.split('\n'):
if '>>>' in line:
if 'TOTAL JOBS' in line.upper():
total_jobs = line.split(' ')[1]
#print total_jobs, "Total Jobs"
if 'CLEARED' in line.upper(
) or 'DONE' in line.upper() and '>>>' not in line:
jobs_done = jobs_done + 1
if 'SUBMITTED' in line.upper():
sub_jobs = sub_jobs + 1
if int(total_jobs) == 0:
print "Crab must not have been submitted properly"
cooked_crab.append(jobid)
elif int(total_jobs) == jobs_done:
print "Crab job: " + jobid + " done!"
out_stat, out_out = commands.getstatusoutput(
'crab -getoutput -c ' + jobid)
out_file = self.CrabCrusher(jobid + '/res/')
cooked_crab.append(jobid)
if out_file == None:
print "No root files in scracth"
else:
proc = out_file.split('/')[-5]
gen = out_file.split('/')[-4]
rel = out_file.split('/')[-3]
if os.path.isdir(Configuration.
variables['ReleaseDirectory'] +
gen + '/' + rel + '/' + proc +
'/') == False:
os.makedirs(Configuration.
variables['ReleaseDirectory'] +
gen + '/' + rel + '-preGen/' +
proc + '/')
shutil.copyfile(
out_file,
Configuration.variables['ReleaseDirectory'] +
gen + '/' + rel + '-preGen/' + proc + '/')
print "Begin Publishing", process
### Publishes and compares, may be able to replace 'data' too ###
gen_List = os.listdir(Configuration.variables["ReleaseDirectory"])
if compare.upper() == "SAME" or compare.upper() == "ALL":
for List in gen_List:
if compare.upper(
) == "SAME" and List not in run and run.upper() != "ALL":
continue
if 'CVS' in List:
continue
sub_dir = os.listdir(
Configuration.variables["ReleaseDirectory"] + List)
for i in range(len(sub_dir) - 1):
for j in range(i + 1, len(sub_dir)):
if sub_dir[i] == sub_dir[j]:
continue
if os.path.isdir(
Configuration.variables["ReleaseDirectory"] +
List + '/' + sub_dir[i] + '/' +
process) == False:
continue
if os.path.isdir(
Configuration.variables["ReleaseDirectory"] +
List + '/' + sub_dir[j] + '/' +
process) == False:
continue
web_dir = Configuration.variables[
"WebDirectory"] + sub_dir[i] + '/' + sub_dir[
j] + '/' + process + '/'
if os.path.isdir(web_dir) == True:
continue
if os.path.isdir(
Configuration.variables["WebDirectory"] +
sub_dir[j] + '/' + sub_dir[i] + '/' + process +
'/') and os.listdir(
Configuration.variables["WebDirectory"] +
sub_dir[j] + '/' + sub_dir[i] + '/' +
process + '/') < 2:
continue
for rel_file in os.listdir(
Configuration.variables["ReleaseDirectory"] +
List + '/' + sub_dir[i] + '/' + process):
if '.root' in rel_file:
for ref_file in os.listdir(
Configuration.
variables["ReleaseDirectory"] + List +
'/' + sub_dir[j] + '/' + process):
if '.root' in ref_file:
if os.path.isdir(web_dir) == False:
os.makedirs(web_dir)
Publisher.StaticWeb().plot(
Configuration.
variables["ReleaseDirectory"] +
List + '/' + sub_dir[i] + '/' +
process + '/' + rel_file,
Configuration.
variables["ReleaseDirectory"] +
List + '/' + sub_dir[j] + '/' +
process + '/' + ref_file, web_dir,
process + '--' + sub_dir[i] +
'--' + sub_dir[j], '', sub_dir[i],
sub_dir[j])
Publisher.StaticWeb().index(
sub_dir[i], sub_dir[j], process,
web_dir)
# Compares new release to old ones.... #
if compare.upper() != "SAME": # or compare.upper() == "ALL":
for i in range(len(gen_List) - 1):
for j in range(i + 1, len(gen_List)):
if compare.upper() != "ALL":
if compare.upper(
) not in gen_List[i] and compare.upper() != "OPPOSITE":
continue
if compare.upper(
) not in gen_List[j] and compare.upper() != "OPPOSITE":
continue
if run.upper() != "ALL":
if run.upper() not in gen_List[i] or run.upper(
) not in gen_List[j]:
continue
if gen_List[i] == 'CVS' or gen_List[j] == 'CVS':
continue
rel_List = os.listdir(
Configuration.variables["ReleaseDirectory"] +
gen_List[i])
ref_List = os.listdir(
Configuration.variables["ReleaseDirectory"] +
gen_List[j])
for rel_dir in rel_List:
if os.path.isdir(
Configuration.variables["ReleaseDirectory"] +
gen_List[i] + '/' + rel_dir + '/' +
process) == False:
continue
for ref_dir in ref_List:
if os.path.isdir(Configuration.
variables["ReleaseDirectory"] +
gen_List[j] + '/' + ref_dir +
'/' + process) == False:
continue
if ref_dir == rel_dir:
continue
if all.upper() == "FALSE":
if ref_dir != self.__release_List[gen_List[
j]] and ref_dir != self.__release_List[gen_List[
i]] and rel_dir != self.__release_List[
gen_List[
j]] and rel_dir != self.__release_List[
gen_List[i]]:
continue
web_dir = Configuration.variables[
"WebDirectory"] + rel_dir + '/' + ref_dir + '/' + process + '/'
if os.path.exists(
web_dir) and os.listdir(web_dir) < 2:
continue
if os.path.exists(
Configuration.variables["WebDirectory"] +
ref_dir + '/' + rel_dir + '/' + process +
'/'):
continue
rel_loc = Configuration.variables[
"ReleaseDirectory"] + gen_List[
i] + '/' + rel_dir + '/' + process + '/'
ref_loc = Configuration.variables[
"ReleaseDirectory"] + gen_List[
j] + '/' + ref_dir + '/' + process + '/'
print gen_List[i], gen_List[j]
for rel_file in os.listdir(rel_loc):
if '.root' in rel_file:
for ref_file in os.listdir(ref_loc):
if '.root' in ref_file:
if os.path.isdir(web_dir) == False:
os.makedirs(web_dir)
Publisher.StaticWeb().plot(
rel_loc + '/' + rel_file,
ref_loc + '/' + ref_file,
web_dir, process + '--' +
rel_dir + '--' + ref_dir, '',
rel_dir, ref_dir)
Publisher.StaticWeb().index(
rel_dir, ref_dir, process,
web_dir)
def NewPublisher(self, run, process, compare, all, job):
os.chdir(Configuration.variables["HomeDirectory"]+'/bin/')
if job.upper() != "LOCAL":
condor_status = 0
while condor_status == 0:
Unfinished_Job = 0
for jobid in self.__jobNumber[process]:
status, output = commands.getstatusoutput('condor_q '+jobid)
if jobid in output:
Unfinished_Job = Unfinished_Job + 1
if Unfinished_Job == 0:
condor_status = 1
print "All jobs for " +process+ " are done"
if Unfinished_Job > 0:
time.sleep(15)
for generator in os.listdir(Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+process):
for sub_gen in os.listdir(Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+process+'/'+generator+'/'):
for subdir in os.listdir(Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+process+'/'+generator+'/'+sub_gen+'/'):
if '.root' in subdir or '.log' in subdir:
if os.path.isdir(Configuration.variables['ReleaseDirectory']+generator+'/'+sub_gen+'/'+process+'/') == False:
os.makedirs(Configuration.variables['ReleaseDirectory']+generator+'/'+sub_gen+'/'+process+'/')
shutil.copyfile(Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+process+'/'+generator+'/'+sub_gen+'/'+subdir, Configuration.variables['ReleaseDirectory']+generator+'/'+sub_gen+'/'+process+'/'+subdir)
#os.system('rm '+Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+process+'/'+generator+'/'+sub_gen+'/*')
#print "Begin Publishing" , process
### Publishes and compares, may be able to replace 'data' too ###
gen_List = os.listdir(Configuration.variables['ReleaseDirectory'])
if compare.upper() == "SAME" or compare.upper() == "ALL":
for List in gen_List:
if compare.upper() == "SAME" and List not in run and run.upper() != "ALL":
continue
sub_dir = os.listdir(Configuration.variables['ReleaseDirectory']+List)
for i in range(len(sub_dir) - 1):
for j in range(i+1, len(sub_dir)):
if sub_dir[i] == sub_dir[j]:
continue
if os.path.isdir(Configuration.variables['ReleaseDirectory']+List+'/'+sub_dir[i]+'/'+process) == False:
continue
if os.path.isdir(Configuration.variables['ReleaseDirectory']+List+'/'+sub_dir[j]+'/'+process) == False:
continue
web_dir = Configuration.variables["WebDirectory"]+sub_dir[i]+'/'+sub_dir[j]+'/'+process+'/'
if os.path.isdir(web_dir) == True:
continue
if os.path.isdir(Configuration.variables["WebDirectory"]+sub_dir[j]+'/'+sub_dir[i]+'/'+process+'/') and os.listdir(Configuration.variables["WebDirectory"]+sub_dir[j]+'/'+sub_dir[i]+'/'+process+'/') < 2:
continue
for rel_file in os.listdir(Configuration.variables['ReleaseDirectory']+List+'/'+sub_dir[i]+'/'+process):
if '.root' in rel_file:
for ref_file in os.listdir(Configuration.variables['ReleaseDirectory']+List+'/'+sub_dir[j]+'/'+process):
if '.root' in ref_file:
if os.path.isdir(web_dir) == False:
os.makedirs(web_dir)
Publisher.StaticWeb().plot(Configuration.variables['ReleaseDirectory']+List+'/'+sub_dir[i]+'/'+process+'/'+rel_file, Configuration.variables['ReleaseDirectory']+List+'/'+sub_dir[j]+'/'+process+'/'+ref_file, web_dir, process+'--'+sub_dir[i]+'--'+sub_dir[j], '', sub_dir[i], sub_dir[j])
Publisher.StaticWeb().index(sub_dir[i], sub_dir[j], process, web_dir)
# Compares new release to old ones.... #
if compare.upper() != "SAME":# or compare.upper() == "ALL":
for i in range(len(gen_List) - 1):
for j in range(i+1,len(gen_List)):
if compare.upper() != "ALL":
if compare.upper() not in gen_List[i] and compare.upper() != "OPPOSITE":
continue
if compare.upper() not in gen_List[j] and compare.upper() != "OPPOSITE":
continue
if run.upper() != "ALL":
if run.upper() not in gen_List[i] or run.upper() not in gen_List[j]:
continue
rel_List = os.listdir(Configuration.variables['ReleaseDirectory']+gen_List[i])
ref_List = os.listdir(Configuration.variables['ReleaseDirectory']+gen_List[j])
for rel_dir in rel_List:
if os.path.isdir(Configuration.variables['ReleaseDirectory']+gen_List[i]+'/'+rel_dir+'/'+process) == False:
continue
for ref_dir in ref_List:
if os.path.isdir(Configuration.variables['ReleaseDirectory']+gen_List[j]+'/'+ref_dir+'/'+process) == False:
continue
if ref_dir == rel_dir:
continue
if all.upper() == "FALSE":
if ref_dir != self.__release_List[gen_List[j]] and ref_dir != self.__release_List[gen_List[i]] and rel_dir != self.__release_List[gen_List[j]] and rel_dir != self.__release_List[gen_List[i]]:
continue
web_dir = Configuration.variables["WebDirectory"]+rel_dir+'/'+ref_dir+'/'+process+'/'
if os.path.exists(web_dir) and os.listdir(web_dir) < 2:
continue
if os.path.exists(Configuration.variables["WebDirectory"]+ref_dir+'/'+rel_dir+'/'+process+'/'):
continue
rel_loc = Configuration.variables['ReleaseDirectory']+gen_List[i]+'/'+rel_dir+ '/' + process+'/'
ref_loc = Configuration.variables['ReleaseDirectory']+gen_List[j]+'/'+ref_dir+ '/' + process+'/'
#print gen_List[i], gen_List[j]
for rel_file in os.listdir(rel_loc):
if '.root' in rel_file:
for ref_file in os.listdir(ref_loc):
if '.root' in ref_file:
if os.path.isdir(web_dir) == False:
os.makedirs(web_dir)
Publisher.StaticWeb().plot(rel_loc+'/'+rel_file, ref_loc+'/'+ref_file, web_dir, process+'--'+rel_dir+'--'+ref_dir, '', rel_dir, ref_dir)
Publisher.StaticWeb().index(rel_dir, ref_dir, process, web_dir)