def CrabCrusher(self, dir):
if os.path.isdir(dir) == False:
return None
fileString = ''
rootFiles = 0
for file in os.listdir(dir):
if '.root' in file:
fileString = fileString + 'FileList->Add( TFile::Open("' + file + '") );\n'
rootFiles = rootFiles + 1
if rootFiles == 0:
return None
tfile = open(Configuration.variables['HomeDirectory'] + '/Crab/Hadd.C',
'r')
template = Template(tfile.read())
tfile.close()
cfile = open(dir + '/Hadd.C', 'w')
cfile.write(
template.safe_substitute(filelist=fileString, output="Crab.root"))
cfile.close()
os.chdir(dir)
os.system('root -l -b Hadd.C')
if os.path.isfile(dir + '/Crab.root'):
return dir + '/Crab.root'
else:
return None
def create( self, release, cfgfile):
# Get the release name
self.__release = release
# Get the cfg file name
self.__package = cfgfile.split('/')[-1].split('.')[0]
# Generate a random string
rvector = [random.choice(string.letters) for x in xrange(10)]
rstring = "".join(rvector)
# Name of the scrtach area
self.__scratch = Configuration.variables['DropBox'] + '/Scratch/' + rstring
# Creates a directory
os.mkdir( self.__scratch )
# Copy cfg file to scratch area
shutil.copy(cfgfile, self.__scratch)
# Create a cmsrun script
tfile = open (Configuration.variables['VTRoot'] + '/data/cmsrun.template' ,'r')
template = Template(tfile.read())
tfile.close()
cfile = open ( self.__scratch + '/cmsrun', 'w' )
cmssws = Configuration.variables['WorkAreas'] + '/' + release + '/src'
cfile.write( template.substitute(scratch=self.__scratch, cfg=cfgfile, cmssw=cmssws) )
cfile.close()
os.chmod( self.__scratch + '/cmsrun', 0755 )
# Create a condor script
tfile = open (Configuration.variables['VTRoot'] + '/data/condor/condor.template' ,'r')
template = Template(tfile.read())
tfile.close()
cfile = open ( self.__scratch + '/condor.script', 'w' )
cfile.write( template.substitute(scratch=self.__scratch) )
cfile.close()
def createcfgs(self, lock, postprocess):
# Get the subscription list
subscriptions = Subscriber()
subscriptions.read(Configuration.variables['VTRoot']+'/data/subscriptions.xml')
# Message list
messages = []
# For a given dataset get all the subscripted templates
if subscriptions.map.has_key(self.__dataset):
for template, interface in subscriptions.map[self.__dataset].iteritems():
# open the associated template file
tfile = open(template, 'r')
tobject = Template(tfile.read())
tfile.close()
# create the proper source string
string = 'source = PoolSource{\n\nuntracked vstring fileNames = {\n'
for i in range(len(self.__lfns)):
if i == len(self.__lfns) - 1:
string = string + '\'' + self.__lfns[i] + '\'\n'
else:
string = string + '\'' + self.__lfns[i] + '\',\n'
string = string + '}\n}\n'
# creates a config file
lock.acquire()
if os.path.isdir(Configuration.variables['DropBox'] + '/CfgFiles/' + self.__release + '/') == False:
os.mkdir(Configuration.variables['DropBox'] + '/CfgFiles/' + self.__release + '/')
lock.release()
cfgfile = Configuration.variables['DropBox'] + '/CfgFiles/' + self.__release + '/' + self.__dataset + '__' + template.split('/')[-1].split('.')[0] + '.cfg'
cfile = open(cfgfile, 'w')
cfile.write( tobject.safe_substitute(lfns=string, dataset=self.__dataset) )
cfile.close()
# Add a message for submitting the job
message = {}
message['startjob'] = { 'groupid' : 'None', 'interface' : str(interface), 'release' : self.__release, 'cfgfile' : str(cfgfile), 'postprocess' : postprocess }
messages.append(message)
return messages
def CrabCrusher(self, dir):
if os.path.isdir(dir) == False:
return None
fileString = ''
rootFiles = 0
for file in os.listdir(dir):
if '.root' in file:
fileString = fileString + 'FileList->Add( TFile::Open("'+file+'") );\n'
rootFiles = rootFiles + 1
if rootFiles == 0:
return None
tfile = open (Configuration.variables['HomeDirectory'] + '/Crab/Hadd.C' ,'r')
template = Template(tfile.read())
tfile.close()
cfile = open(dir+'/Hadd.C', 'w')
cfile.write(template.safe_substitute(filelist=fileString, output="Crab.root" ))
cfile.close()
os.chdir(dir)
os.system('root -l -b Hadd.C')
if os.path.isfile(dir+'/Crab.root'):
return dir+'/Crab.root'
else:
return None
def batch_run(self, run, process, compare, batch):
generator_List = []
self.__release_List = {}
self.__jobNumber = {}
process_List = []
os.chdir(Configuration.variables["HomeDirectory"] + 'bin/')
if run != None or process != None:
# Find release for each generator used
dir = os.listdir(Configuration.variables["HomeDirectory"] +
'test/')
if run.upper() != "ALL":
temp_status, temp_output = commands.getstatusoutput(
'scramv1 tool info ' + run)
if "Version" not in temp_output:
print "Error obtaining release, please make sure scramv1 is working"
self.__release_List[run] = run + '__external'
generator_List.append(run)
for line in temp_output.split('\n'):
if "Version" in line:
self.__release_List[run] = run + "__" + line.split(
':')[-1].strip(' ')
generator_List.append(run)
if compare.upper() != "ALL" and compare.upper() != run.upper():
temp_status, temp_output = commands.getstatusoutput(
'scramv1 tool info ' + compare)
if "Version" not in temp_output:
print "Error obtaining release, please make sure scramv1 is working"
self.__release_List[compare] = compare + '__external'
generator_List.append(compare)
for line in temp_output.split('\n'):
if "Version" in line:
self.__release_List[
compare] = compare + "__" + line.split(
':')[-1].strip(' ')
generator_List.append(compare)
if run.upper() == "ALL" or compare.upper() == "ALL":
for probe in dir:
if 'CVS' in probe:
continue
if len(probe.split('__')) == 1:
continue
if '~' in probe:
continue
if '.cfg' not in probe and '.py' not in probe:
continue
if probe.split('.')[0].split(
'__')[1] not in generator_List:
print 'scramv1 tool info ' + probe.split('.')[0].split(
'__')[1]
temp_status, temp_output = commands.getstatusoutput(
'scramv1 tool info ' +
probe.split('.')[0].split('__')[1])
if "Version" not in temp_output:
print "Error obtaining release, please make sure scramv1 is working"
self.__release_List[compare] = probe.split(
'.')[0].split('__')[1] + '__external'
generator_List.append(
probe.split('.')[0].split('__')[1])
for line in temp_output.split('\n'):
if "Version" in line:
self.__release_List[probe.split('.')[0].split(
'__')[1]] = probe.split('.')[0].split(
'__')[1] + "__" + line.split(
':')[-1].strip(' ')
generator_List.append(
probe.split('.')[0].split('__')[1])
for file in dir:
if 'CVS' in file:
continue
if len(file.split('__')) == 1:
continue
if '~' in file:
continue
if '.cfg' not in file and '.py' not in file:
continue
if process.upper() != "ALL" and process.upper(
) not in file.upper():
continue
if file.split('.')[0].split('__')[0] not in process_List:
self.__jobNumber[file.split('.')[0].split('__')[0]] = []
process_List.append(file.split('.')[0].split('__')[0])
splitter = file.split('.')[0].split('__')
if len(splitter) == 3:
scratch = Configuration.variables[
"HomeDirectory"] + 'DropBox/scratch/' + file.split(
'.')[0].split('__')[0] + "/" + file.split(
'.')[0].split('__')[1] + '/' + file.split(
'.')[0].split('__')[2] + '-EDM'
else:
scratch = Configuration.variables[
"HomeDirectory"] + 'DropBox/scratch/' + file.split(
'.')[0].split('__')[0] + "/" + file.split('.')[
0].split('__')[1] + '/' + self.__release_List[
file.split('.')[0].split('__')[1]]
if os.path.exists(scratch + '/') == False:
os.makedirs(scratch + '/')
CMSstatus, CMSoutput = commands.getstatusoutput('$CMSSW_BASE')
CMS_dir = CMSoutput.split(':')[1].strip(' ')
if 'CERN' not in batch.upper():
tfile = open(
Configuration.variables['HomeDirectory'] +
'/data/condor.template', 'r')
template = Template(tfile.read())
tfile.close()
cfile = open(scratch + "/condor.submit", 'w')
cfile.write(
template.safe_substitute(
file=scratch + '/cmsrun' +
file.split('.')[0].split('__')[0],
scratch=scratch))
cfile.close()
tfile = open(
Configuration.variables['HomeDirectory'] +
'/data/cmsrun.template', 'r')
template = Template(tfile.read())
tfile.close()
cfile = open(
scratch + "/cmsrun" +
file.split('.')[0].split('__')[0], 'w')
cfile.write(
template.safe_substitute(
directory=scratch,
cfg=Configuration.variables["HomeDirectory"] +
'test/' + file,
CMSSW=CMS_dir))
cfile.close()
os.chmod(
scratch + "/cmsrun" +
file.split('.')[0].split('__')[0], 0755)
status, output = commands.getstatusoutput(
"condor_submit " + scratch + "/condor.submit")
for line in output.split('\n'):
if "submitted" in line:
self.__jobNumber[file.split('.')[0].split('__')
[0]].append(line.split(' ')[5])
jobID = line.split(' ')[5]
if status != 0:
print file + " wasn't submitted properly"
jobID = None
else:
tfile = open(
Configuration.variables['HomeDirectory'] +
'/interface/cmsrunbsub.template', 'r')
template = Template(tfile.read())
tfile.close()
cfile = open(
scratch + "/cmsrunbsub" +
file.split('.')[0].split('__')[0], 'w')
cfile.write(
template.safe_substitute(
directory=scratch,
cfg=Configuration.variables["HomeDirectory"] +
'test/' + file,
CMSSW=CMS_dir))
cfile.close()
os.chmod(
scratch + "/cmsrunbsub" +
file.split('.')[0].split('__')[0], 0755)
os.chdir(scratch)
status, output = commands.getstatusoutput(
"bsub cmsrunbsub" + file.split('.')[0].split('__')[0])
for line in output.split('\n'):
if "submitted" in line:
self.__jobNumber[file.split('.')[0].split(
'__')[0]].append(
line.split(' ')[2].lstrip('<').rstrip('>'))
jobID = line.split(' ')[2].lstrip('<').rstrip('>')
if status != 0:
print file + " wasn't submitted properly"
jobID = None
os.chdir(Configuration.variables["HomeDirectory"] +
'/bin/')
return jobID
def new_config_run(self, run, process, compare):
generator_List = []
self.__release_List = {}
os.chdir(Configuration.variables["HomeDirectory"]+'bin/')
if run != None or process != None:
######### Find release for each generator used ##########
dir = os.listdir(Configuration.variables["HomeDirectory"]+'data/')
temp_dir = os.listdir(Configuration.variables["HomeDirectory"]+'templates/')
if run.upper() != "ALL":
temp_status, temp_output = commands.getstatusoutput('scramv1 tool info '+run )
if "Version" not in temp_output:
#print "Error obtaining "+run+" release, please make sure scramv1 is working"
self.__release_List[run] = run+'__external'
generator_List.append(run)
for line in temp_output.split('\n'):
if "Version" in line:
self.__release_List[run] = run+"__"+line.split(':')[-1].strip(' ')
generator_List.append(run)
if compare.upper() != "ALL" and compare.upper() != run.upper():
temp_status, temp_output = commands.getstatusoutput('scramv1 tool info '+compare )
if "Version" not in temp_output:
#print "Error obtaining "+compare+" release, please make sure scramv1 is working"
self.__release_List[compare] = compare+'__external'
generator_List.append(compare)
for line in temp_output.split('\n'):
if "Version" in line:
self.__release_List[compare] = compare+"__"+line.split(':')[-1].strip(' ')
generator_List.append(compare)
if run.upper() == "ALL" or compare.upper() == "ALL":
for probe in dir:
if len(probe.split('__')) == 1:
continue
if probe.split('.')[0].split('__')[1] not in generator_List:
temp_status, temp_output = commands.getstatusoutput('scramv1 tool info '+probe.split('.')[0].split('__')[1] )
if "Version" not in temp_output:
self.__release_List[compare] = probe.split('.')[0].split('__')[1]+'__external'
generator_List.append(probe.split('.')[0].split('__')[1])
for line in temp_output.split('\n'):
if "Version" in line:
self.__release_List[probe.split('.')[0].split('__')[1]] = probe.split('.')[0].split('__')[1]+"__"+line.split(':')[-1].strip(' ')
generator_List.append(probe.split('.')[0].split('__')[1])
######### Make cfg files from templates and run #############
for file in dir:
if '~' in file or '.cfi' not in file:
continue
if run.upper() != "ALL" and compare.upper() != "ALL" and run.upper() not in file.upper() and compare.upper() not in file.upper():
continue
if process.upper() in file.upper():
print "About to run cmsRun", file
splitter = file.split('.')[0].split('__')
if len(splitter) > 2:
directory = file.split('.')[0].split('__')[1]+'/'+file.split('.')[0].split('__')[1]+'__'+file.split('.')[0].split('__')[2]+'EDM'
else:
directory = file.split('.')[0].split('__')[1]+'/'+self.__release_List[file.split('.')[0].split('__')[1]]
if os.path.exists(Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+process+'/'+directory+'/') == False:
os.makedirs(Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+process+'/'+directory+'/')
scratch = Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+process+'/'+directory+'/'
for template in temp_dir:
if template.split('.')[0] == file.split('__')[0]:
tfile = open (Configuration.variables['HomeDirectory'] + '/templates/'+template ,'r')
template = Template(tfile.read())
tfile.close()
file_reader = open(Configuration.variables['HomeDirectory'] + '/data/'+file, 'r')
source_string = file_reader.read()
file_reader.close()
cfg_file = file.split('.')[0]+'.cfg'
cfile = open(scratch+cfg_file, 'w')
cfile.write(template.safe_substitute(source = source_string))
cfile.close()
os.chdir(Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+process+'/'+directory+'/')
os.system('cmsRun '+cfg_file+' >& '+file.split('.')[0]+'.log')
root_files = 0
for data_file in os.listdir(Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+process+'/'+directory+'/'):
if '.root' in data_file or '.log' in data_file:
if len(splitter) > 2:
directory1 = file.split('.')[0].split('__')[1]+'/'+file.split('.')[0].split('__')[1]+'__'+file.split('.')[0].split('__')[2]+'EDM'
else:
directory1 = file.split('.')[0].split('__')[1]+'/'+self.__release_List[file.split('.')[0].split('__')[1]]
if os.path.exists(Configuration.variables['ReleaseDirectory']+directory1+'/'+file.split('.')[0].split('__')[0]+'/') != True:
os.makedirs(Configuration.variables['ReleaseDirectory']+directory1+'/'+file.split('.')[0].split('__')[0]+'/')
shutil.copyfile(Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+process+'/'+directory+'/'+data_file, Configuration.variables['ReleaseDirectory']+directory1+'/'+file.split('.')[0].split('__')[0]+'/'+data_file)
if '.root' in data_file:
root_files = 1
os.system('rm '+Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+process+'/'+directory+'/*')
if root_files == 0:
print file+" didn't run properly, please check log file"
def Vista_run(self, Site, job):
#
self.__jobNumber = []
List = os.listdir(Configuration.variables['HomeDirectory']+'data/')
Vista_List = []
process_List = []
CMSstatus, CMSoutput = commands.getstatusoutput('$CMSSW_BASE')
CMS_dir = CMSoutput.split(':')[1].strip(' ')
for file in List:
if "VISTA" in file.upper() and file not in Vista_List:
Vista_List.append(file)
print Vista_List, "Vista"
Remaining = Vista_List
head = """source = PoolSource \n
{ \n
untracked vstring fileNames = \n
{ \n
"""
foot =""" } \n
} \n """
while len(Remaining) != 0:
for run in Remaining:
if run.split('.')[0].split('__')[0] not in process_List:
self.__jobNumber[run.split('.')[0].split('__')[0]] = []
process_List.append(run.split('.')[0].split('__')[0])
if "DBS" in run.upper():
stat, out = commands.getstatusoutput('./rssparser --dataset='+run.split('__')[0].strip("-Vista")+' --release='+run.split('__')[2])
body_string = ''
if '.root' in out:
scratch = Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+run.split('__')[0]+'/'+"Vista"+'/'+run.split('__')[2]
if os.path.exists(scratch) == False:
os.makedirs(scratch)
for line in out.split('\n'):
if '.root' in line:
body_string = body_string + line + '\n'
tfile = open (Configuration.variables['HomeDirectory'] + '/templates/runVista.template' ,'r')
template = Template(tfile.read())
tfile.close()
source_string = head + body_string + foot
cfile = open(scratch+"/runVista.cfg", 'w')
cfile.write(template.safe_substitute(source = source_string ))
cfile.close()
if Site.upper() == "FNAL":
tfile = open (Configuration.variables['HomeDirectory'] + '/interface/condor.template' ,'r')
template = Template(tfile.read())
tfile.close()
cfile = open(scratch+"/condor.submit", 'w')
cfile.write(template.safe_substitute(file = scratch+'/cmsruncondor'+run.split('.')[0].split('__')[0], scratch = scratch))
cfile.close()
tfile = open (Configuration.variables['HomeDirectory'] + '/interface/cmsruncondor.template' ,'r')
template = Template(tfile.read())
tfile.close()
cfile = open(scratch+"/cmsruncondor"+run.split('.')[0].split('__')[0], 'w')
cfile.write(template.safe_substitute(directory=scratch, cfg = scratch+'/runVista.cfg', CMSSW=CMS_dir ))
cfile.close()
os.chmod(scratch+"/cmsruncondor"+run.split('.')[0].split('__')[0], 0755)
status, output = commands.getstatusoutput("condor_submit " + scratch+"/condor.submit")
Remaining.remove(run)
elif Site.upper() == "CERN":
tfile = open (Configuration.variables['HomeDirectory'] + '/interface/cmsrunbsub.template' ,'r')
template = Template(tfile.read())
tfile.close()
cfile = open(scratch+"/cmsrunbsub"+run.split('.')[0].split('__')[0], 'w')
cfile.write(template.safe_substitute(directory=scratch, cfg = scratch+'/runVista.cfg', CMSSW=CMS_dir ))
cfile.close()
os.chmod(scratch+"/cmsrunbsub"+run.split('.')[0].split('__')[0], 0755)
status, output = commands.getstatusoutput("bsub " + scratch+"/cmsrunbsub"+run.split('.')[0].split('__')[0])
for line in output.split('\n'):
if "submitted" in line:
self.__jobNumber[run.split('.')[0].split('__')[0]].append(line.split(' ')[2].lstrip('<').rstrip('>'))
if status != 0:
print file + " wasn't submitted properly"
Remaining.remove(run)
else:
print run, job, "DBS"
os.chdir(scratch)
status, output = commands.getstatusoutput("cmsRun runVista.cfg >& "+run.split('.')[0].split('__')[0]+'.log')
os.chdir(Configuration.variables['HomeDirectory'])
elif 'ERROR' in out.upper():
print "Error reading DBS for "+run
Remaining.remove(run)
else:
time.sleep(40)
else:
file = open(Configuration.variables['HomeDirectory']+'data/'+run, 'r')
source_string = file.read()
file.close()
scratch = Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+run.split('__')[0]+'/'+"Vista"+'/'+run.split('__')[2]
tfile = open (Configuration.variables['HomeDirectory'] + '/tempaltes/runVista.template' ,'r')
template = Template(tfile.read())
tfile.close()
cfile = open(scratch+"/runVista.cfg", 'w')
cfile.write(template.safe_substitute(source = source_string ))
cfile.close()
if job.upper() != "BATCH":
print run, job
os.chdir(scratch)
status, output = commands.getstatusoutput("cmsRun runVista.cfg >& "+run.split('.')[0].split('__')[0]+'.log')
os.chdir(Configuration.variables['HomeDirectory'])
else:
if Site.upper() == "FNAL":
tfile = open (Configuration.variables['HomeDirectory'] + '/interface/condor.template' ,'r')
template = Template(tfile.read())
tfile.close()
cfile = open(scratch+"/condor.submit", 'w')
cfile.write(template.safe_substitute(file = scratch+'/cmsrun'+run.split('.')[0].split('__')[0], scratch = scratch))
cfile.close()
tfile = open (Configuration.variables['HomeDirectory'] + '/interface/cmsrun.template' ,'r')
template = Template(tfile.read())
tfile.close()
cfile = open(scratch+"/cmsrun"+run.split('.')[0].split('__')[0], 'w')
cfile.write(template.safe_substitute(directory=scratch, cfg = scratch+'/runVista.cfg', CMSSW=CMS_dir ))
cfile.close()
os.chmod(scratch+"/cmsrun"+run.split('.')[0].split('__')[0], 0755)
status, output = commands.getstatusoutput("condor_submit " + scratch+"/condor.submit")
print run, "BATCH"
for line in output.split('\n'):
if "submitted" in line:
self.__jobNumber[run.split('.')[0].split('__')[0]].append(line.split(' ')[5])
if Site.upper() == "CERN":
tfile = open (Configuration.variables['HomeDirectory'] + '/interface/cmsrunbsub.template' ,'r')
template = Template(tfile.read())
tfile.close()
cfile = open(scratch+"/cmsrunbsub"+run.split('.')[0].split('__')[0], 'w')
cfile.write(template.safe_substitute(directory=scratch, cfg = scratch+'/runVista.cfg', CMSSW=CMS_dir ))
cfile.close()
os.chmod(scratch+"/cmsrunbsub"+run.split('.')[0].split('__')[0], 0755)
status, output = commands.getstatusoutput("bsub " + scratch+"/cmsrunbsub"+run.split('.')[0].split('__')[0])
for line in output.split('\n'):
if "submitted" in line:
self.__jobNumber[run.split('.')[0].split('__')[0]].append(line.split(' ')[2].lstrip('<').rstrip('>'))
if status != 0:
print file + " wasn't submitted properly"
Remaining.remove(run)
def new_batch_run(self, run, process, compare, batch):
generator_List = []
self.__release_List = {}
self.__jobNumber = {}
process_List = []
os.chdir(Configuration.variables["HomeDirectory"]+'bin/')
if run != None or process != None:
####### Find release for each generator used ##########
dir = os.listdir(Configuration.variables["HomeDirectory"]+'data/')
temp_dir = os.listdir(Configuration.variables["HomeDirectory"]+'templates/')
if run.upper() != "ALL":
temp_status, temp_output = commands.getstatusoutput('scramv1 tool info '+run )
if "Version" not in temp_output:
#print "Error obtaining release, please make sure scramv1 is working"
self.__release_List[run] = run+'__external'
generator_List.append(run)
for line in temp_output.split('\n'):
if "Version" in line:
self.__release_List[run] = run+"__"+line.split(':')[-1].strip(' ')
generator_List.append(run)
if compare.upper() != "ALL" and compare.upper() != run.upper():
temp_status, temp_output = commands.getstatusoutput('scramv1 tool info '+compare )
if "Version" not in temp_output:
#print "Error obtaining release, please make sure scramv1 is working"
self.__release_List[compare] = compare+'__external'
generator_List.append(compare)
for line in temp_output.split('\n'):
if "Version" in line:
self.__release_List[compare] = compare+"__"+line.split(':')[-1].strip(' ')
generator_List.append(compare)
if run.upper() == "ALL" or compare.upper() == "ALL":
for probe in dir:
if probe.split('.')[0].split('__')[1] not in generator_List:
#print 'scramv1 tool info '+probe.split('.')[0].split('__')[1]
temp_status, temp_output = commands.getstatusoutput('scramv1 tool info '+probe.split('.')[0].split('__')[1] )
if "Version" not in temp_output:
#print "Error obtaining release, please make sure scramv1 is working"
self.__release_List[compare] = probe.split('.')[0].split('__')[1]+'__external'
generator_List.append(probe.split('.')[0].split('__')[1])
for line in temp_output.split('\n'):
if "Version" in line:
self.__release_List[probe.split('.')[0].split('__')[1]] = probe.split('.')[0].split('__')[1]+"__"+line.split(':')[-1].strip(' ')
generator_List.append(probe.split('.')[0].split('__')[1])
######## Make cfgs from templates and run ##############
for file in dir:
if '~' in file or '.cfi' not in file:
continue
if 'cmsrun' in file:
continue
if 'condor' in file:
continue
#print file, 'file'
if process.upper() != "ALL" and process.upper() not in file.upper():
continue
if file.split('.')[0].split('__')[0] not in process_List:
self.__jobNumber[file.split('.')[0].split('__')[0]] = []
process_List.append(file.split('.')[0].split('__')[0])
splitter = file.split('.')[0].split('__')
if len(splitter) > 2:
scratch = Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+splitter[0]+"/"+splitter[1]+'/'+splitter[2]+'EDM/'
else:
scratch = Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+file.split('.')[0].split('__')[0]+"/"+file.split('.')[0].split('__')[1]+'/'+self.__release_List[file.split('.')[0].split('__')[1]]+'/'
if os.path.exists(scratch+'/') == False:
os.makedirs(scratch+'/')
CMSstatus, CMSoutput = commands.getstatusoutput('$CMSSW_BASE')
CMS_dir = CMSoutput.split(':')[1].strip(' ')
for temp in temp_dir:
print temp, 'template'
if temp.split('.')[0] == file.split('__')[0]:
tfile = open (Configuration.variables['HomeDirectory'] + '/templates/'+temp ,'r')
template = Template(tfile.read())
tfile.close()
file_reader = open(Configuration.variables['HomeDirectory'] + '/data/'+file, 'r')
source_string = file_reader.read()
file_reader.close()
cfg_filename = file.split('.')[0]+'.cfg'
cfile = open(scratch+cfg_filename, 'w')
cfile.write(template.safe_substitute(source = source_string))
cfile.close()
if batch.upper() == "FNAL":
tfile = open (Configuration.variables['HomeDirectory'] + '/interface/condor.template' ,'r')
template = Template(tfile.read())
tfile.close()
cfile = open(scratch+"/condor.submit", 'w')
cfile.write(template.safe_substitute(file = scratch+'/cmsruncondor'+file.split('.')[0].split('__')[0], scratch = scratch))
cfile.close()
tfile = open (Configuration.variables['HomeDirectory'] + '/interface/cmsruncondor.template' ,'r')
template = Template(tfile.read())
tfile.close()
cfile = open(scratch+"/cmsruncondor"+file.split('.')[0].split('__')[0], 'w')
cfile.write(template.safe_substitute(directory=scratch, cfg = scratch+'/'+cfg_filename, CMSSW=CMS_dir ))
cfile.close()
os.chmod(scratch+"/cmsruncondor"+file.split('.')[0].split('__')[0], 0755)
status, output = commands.getstatusoutput("condor_submit " + scratch+"/condor.submit")
for line in output.split('\n'):
if "submitted" in line:
self.__jobNumber[file.split('.')[0].split('__')[0]].append(line.split(' ')[5])
if status != 0:
print file + " wasn't submitted properly"
if batch.upper() == "CERN":
tfile = open (Configuration.variables['HomeDirectory'] + '/interface/cmsrunbsub.template' ,'r')
template = Template(tfile.read())
tfile.close()
cfile = open(scratch+"/cmsrunbsub"+file.split('.')[0].split('__')[0], 'w')
cfile.write(template.safe_substitute(directory=scratch, cfg = scratch+'/'+cfg_filename, CMSSW=CMS_dir ))
cfile.close()
os.chmod(scratch+"/cmsrunbsub"+file.split('.')[0].split('__')[0], 0755)
status, output = commands.getstatusoutput("bsub " + scratch+"/cmsrunbsub"+file.split('.')[0].split('__')[0])
for line in output.split('\n'):
if "submitted" in line:
self.__jobNumber[file.split('.')[0].split('__')[0]].append(line.split(' ')[2].lstrip('<').rstrip('>'))
if status != 0:
print file + " wasn't submitted properly"
def batch_run(self, run, process, compare, batch):
generator_List = []
self.__release_List = {}
self.__jobNumber = {}
process_List = []
os.chdir(Configuration.variables["HomeDirectory"]+'bin/')
if run != None or process != None:
# Find release for each generator used
dir = os.listdir(Configuration.variables["HomeDirectory"]+'test/')
if run.upper() != "ALL":
temp_status, temp_output = commands.getstatusoutput('scramv1 tool info '+run )
if "Version" not in temp_output:
print "Error obtaining release, please make sure scramv1 is working"
self.__release_List[run] = run+'__external'
generator_List.append(run)
for line in temp_output.split('\n'):
if "Version" in line:
self.__release_List[run] = run+"__"+line.split(':')[-1].strip(' ')
generator_List.append(run)
if compare.upper() != "ALL" and compare.upper() != run.upper():
temp_status, temp_output = commands.getstatusoutput('scramv1 tool info '+compare )
if "Version" not in temp_output:
print "Error obtaining release, please make sure scramv1 is working"
self.__release_List[compare] = compare+'__external'
generator_List.append(compare)
for line in temp_output.split('\n'):
if "Version" in line:
self.__release_List[compare] = compare+"__"+line.split(':')[-1].strip(' ')
generator_List.append(compare)
if run.upper() == "ALL" or compare.upper() == "ALL":
for probe in dir:
if 'CVS' in probe:
continue
if len(probe.split('__')) == 1:
continue
if '~' in probe:
continue
if '.cfg' not in probe and '.py' not in probe:
continue
if probe.split('.')[0].split('__')[1] not in generator_List:
print 'scramv1 tool info '+probe.split('.')[0].split('__')[1]
temp_status, temp_output = commands.getstatusoutput('scramv1 tool info '+probe.split('.')[0].split('__')[1] )
if "Version" not in temp_output:
print "Error obtaining release, please make sure scramv1 is working"
self.__release_List[compare] = probe.split('.')[0].split('__')[1]+'__external'
generator_List.append(probe.split('.')[0].split('__')[1])
for line in temp_output.split('\n'):
if "Version" in line:
self.__release_List[probe.split('.')[0].split('__')[1]] = probe.split('.')[0].split('__')[1]+"__"+line.split(':')[-1].strip(' ')
generator_List.append(probe.split('.')[0].split('__')[1])
for file in dir:
if 'CVS' in file:
continue
if len(file.split('__')) == 1:
continue
if '~' in file:
continue
if '.cfg' not in file and '.py' not in file:
continue
if process.upper() != "ALL" and process.upper() not in file.upper():
continue
if file.split('.')[0].split('__')[0] not in process_List:
self.__jobNumber[file.split('.')[0].split('__')[0]] = []
process_List.append(file.split('.')[0].split('__')[0])
splitter = file.split('.')[0].split('__')
if len(splitter) == 3:
scratch = Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+file.split('.')[0].split('__')[0]+"/"+file.split('.')[0].split('__')[1]+'/'+file.split('.')[0].split('__')[2]+'-EDM'
else:
scratch = Configuration.variables["HomeDirectory"]+'DropBox/scratch/'+file.split('.')[0].split('__')[0]+"/"+file.split('.')[0].split('__')[1]+'/'+self.__release_List[file.split('.')[0].split('__')[1]]
if os.path.exists(scratch+'/') == False:
os.makedirs(scratch+'/')
CMSstatus, CMSoutput = commands.getstatusoutput('$CMSSW_BASE')
CMS_dir = CMSoutput.split(':')[1].strip(' ')
if 'CERN' not in batch.upper():
tfile = open (Configuration.variables['HomeDirectory'] + '/data/condor.template' ,'r')
template = Template(tfile.read())
tfile.close()
cfile = open(scratch+"/condor.submit", 'w')
cfile.write(template.safe_substitute(file = scratch+'/cmsrun'+file.split('.')[0].split('__')[0], scratch = scratch))
cfile.close()
tfile = open (Configuration.variables['HomeDirectory'] + '/data/cmsrun.template' ,'r')
template = Template(tfile.read())
tfile.close()
cfile = open(scratch+"/cmsrun"+file.split('.')[0].split('__')[0], 'w')
cfile.write(template.safe_substitute(directory=scratch, cfg = Configuration.variables["HomeDirectory"]+'test/'+file, CMSSW=CMS_dir ))
cfile.close()
os.chmod(scratch+"/cmsrun"+file.split('.')[0].split('__')[0], 0755)
status, output = commands.getstatusoutput("condor_submit " + scratch+"/condor.submit")
for line in output.split('\n'):
if "submitted" in line:
self.__jobNumber[file.split('.')[0].split('__')[0]].append(line.split(' ')[5])
jobID = line.split(' ')[5]
if status != 0:
print file + " wasn't submitted properly"
jobID = None
else:
tfile = open (Configuration.variables['HomeDirectory'] + '/interface/cmsrunbsub.template' ,'r')
template = Template(tfile.read())
tfile.close()
cfile = open(scratch+"/cmsrunbsub"+file.split('.')[0].split('__')[0], 'w')
cfile.write(template.safe_substitute(directory=scratch, cfg = Configuration.variables["HomeDirectory"]+'test/'+file, CMSSW=CMS_dir ))
cfile.close()
os.chmod(scratch+"/cmsrunbsub"+file.split('.')[0].split('__')[0], 0755)
os.chdir(scratch)
status, output = commands.getstatusoutput("bsub cmsrunbsub"+file.split('.')[0].split('__')[0])
for line in output.split('\n'):
if "submitted" in line:
self.__jobNumber[file.split('.')[0].split('__')[0]].append(line.split(' ')[2].lstrip('<').rstrip('>'))
jobID = line.split(' ')[2].lstrip('<').rstrip('>')
if status != 0:
print file + " wasn't submitted properly"
jobID = None
os.chdir(Configuration.variables["HomeDirectory"]+'/bin/')
return jobID