def _submit(cls, working_dir, njobs):
cwd = os.getcwd()
os.chdir(working_dir)
try:
submit_out, submit_ret = popen('condor_submit < cs_submit.jdl',
return_exit_code=True)
ok = False
cluster = None
schedd = None
if cls.schedds:
for line in submit_out.split('\n'):
if line.startswith('Attempting to submit jobs to '):
schedd = line.strip().replace(
'Attempting to submit jobs to ', '')
assert schedd in cls.schedds
for line in submit_out.split('\n'):
if 'job(s) submitted to cluster' in line:
ok = True
line = line.split()
try:
njobs_sub = int(line[0])
assert line[-1][-1] == '.'
cluster = int(line[-1][:-1])
open('njobs', 'wt').write(str(njobs_sub))
open('cluster',
'wt').write('%i %s' %
(cluster,
schedd) if schedd else str(cluster))
if njobs_sub != njobs:
ok = False
except ValueError:
ok = False
if ok and not cluster:
print '\033[1m problem, ok but no cluster? \033[0m'
if not ok:
print '\033[1m problem! \033[0m'
print submit_out
else:
if schedd:
cluster_s = '%s:%s' % (schedd, cluster)
if cls.links_dir:
cluster_link = os.path.join(cls.links_dir, schedd,
str(cluster))
else:
cluster_s = cluster
if cls.links_dir:
cluster_link = os.path.join(cls.links_dir,
str(cluster))
print '\x1b[92msuccess!\x1b[0m %s' % cluster_s
if cls.links_dir:
if os.path.islink(cluster_link):
print 'warning: clobbering old link:', os.readlink(
cluster_link)
os.unlink(cluster_link)
os.symlink(os.path.abspath(working_dir), cluster_link)
finally:
os.chdir(cwd)
def crab_need_renew_proxy(min_hours=144):
# JMTBAD should use the CRAB function if exists in v3
if os.system('voms-proxy-info -exists -valid %i:0' % min_hours) != 0:
return True
out = popen('myproxy-info -d -s myproxy.cern.ch')
for line in out.split('\n'):
if 'timeleft' in line:
return int(line.split(':')[1]) < min_hours
print '\033[36;7m warning: \033[m crab_need_renew_proxy could not parse output of myproxy-info'
return True
def crab_need_renew_proxy(min_hours=144):
# JMTBAD should use the CRAB function if exists in v3
if os.system('voms-proxy-info -exists -valid %i:0' % min_hours) != 0:
return True
out = popen('myproxy-info -d -s myproxy.cern.ch')
for line in out.split('\n'):
if 'timeleft' in line:
return int(line.split(':')[1]) < min_hours
print '\033[36;7m warning: \033[m crab_need_renew_proxy could not parse output of myproxy-info'
return True
def __init__(self, fn, trees=None):
self.fn = fn
if fn.startswith('/store/user'):
fn = 'root://cmseos.fnal.gov/' + fn
elif fn.startswith('/store'):
fn = 'root://cmsxrootd.fnal.gov/' + fn
if not trees:
trees = EdmFileInfo.event_tree_only
self.trees = [EdmTree(name, popen('edmEventSize -vn %s %s' % (name,fn))) for name in trees]
for t in self.trees:
setattr(self, t.name, t)
def __init__(self, fn, trees=None):
self.fn = fn
if fn.startswith('/store/user'):
fn = 'root://cmseos.fnal.gov/' + fn
elif fn.startswith('/store'):
fn = 'root://cmsxrootd.fnal.gov/' + fn
if not trees:
trees = EdmFileInfo.event_tree_only
self.trees = [
EdmTree(name, popen('edmEventSize -vn %s %s' % (name, fn)))
for name in trees
]
for t in self.trees:
setattr(self, t.name, t)
def submit(self, sample):
self.nsubmits += 1
print 'submit', self.batch_name, sample.name,
if self.dataset:
try:
sample.set_curr_dataset(self.dataset)
except KeyError:
print "\033[1m warning: \033[0m sample %s not submitted, doesn't have dataset %s" % (
sample.name, self.dataset)
return
if sample.split_by == 'events' and not sample.is_mc:
print "\033[1m warning: \033[0m sample %s not submitted because can't split by events on data sample"
return
working_dir = os.path.join(self.batch_dir, 'condor_%s' % sample.name)
if os.path.exists(working_dir):
print "\033[1m warning: \033[0m sample %s not submitted, working dir %s already exists" % (
sample.name, working_dir)
return
os.mkdir(working_dir)
touch(os.path.join(working_dir, 'cs_dir'))
open(os.path.join(working_dir, 'cs_ex'), 'wt').write(self.ex_str)
njobs = self.filelist(sample, working_dir)
pset_fn = self.pset(sample, working_dir)
jdl_fn = os.path.join(working_dir, 'cs_submit.jdl')
open(jdl_fn,
'wt').write(self.jdl_template.replace('__NJOBS__', str(njobs)))
if not self.testing:
self._submit(working_dir, njobs)
else:
print 'in testing mode, not submitting anything.'
if pset_fn:
diff_out, diff_ret = popen('diff -uN %s %s' %
(self.pset_template_fn, pset_fn),
return_exit_code=True)
if diff_ret != 0:
print '.py diff:\n---------'
print diff_out
raw_input('continue?')
print
def submit(self, sample):
self.nsubmits += 1
print 'batch', self.batch_name, 'sample', sample.name,
if self.dataset:
try:
sample.set_curr_dataset(self.dataset)
except KeyError:
print "\033[1m warning: \033[0m sample %s not submitted, doesn't have dataset %s" % (sample.name, self.dataset)
return
if sample.split_by == 'events' and not sample.is_mc:
print "\033[1m warning: \033[0m sample %s not submitted because can't split by events on data sample"
return
working_dir = os.path.join(self.batch_dir, 'condor_%s' % sample.name)
if os.path.exists(working_dir):
print "\033[1m warning: \033[0m sample %s not submitted, working dir %s already exists" % (sample.name, working_dir)
return
os.mkdir(working_dir)
touch(os.path.join(working_dir, 'cs_dir'))
open(os.path.join(working_dir, 'cs_ex'), 'wt').write(self.ex_str)
njobs = self.filelist(sample, working_dir)
pset_fn = self.pset(sample, working_dir)
jdl_fn = os.path.join(working_dir, 'cs_submit.jdl')
open(jdl_fn, 'wt').write(self.jdl_template.replace('__NJOBS__', str(njobs)))
if not self.testing:
self._submit(working_dir, njobs)
else:
print 'in testing mode, not submitting anything.'
if pset_fn:
diff_out, diff_ret = popen('diff -uN %s %s' % (self.pset_template_fn, pset_fn), return_exit_code=True)
if diff_ret != 0:
print '.py diff:\n---------'
print diff_out
raw_input('continue?')
print
def _submit(cls, working_dir, njobs):
cwd = os.getcwd()
os.chdir(working_dir)
try:
submit_out, submit_ret = popen('condor_submit < cs_submit.jdl', return_exit_code=True)
ok = False
schedd = None
for line in submit_out.split('\n'):
if line.startswith('Attempting to submit jobs to '):
schedd = line.strip().replace('Attempting to submit jobs to ', '')
assert schedd in cls.schedds
for line in submit_out.split('\n'):
if 'job(s) submitted to cluster' in line:
ok = True
line = line.split()
try:
njobs_sub = int(line[0])
assert line[-1][-1] == '.'
cluster = int(line[-1][:-1])
open('njobs', 'wt').write(str(njobs_sub))
open('cluster', 'wt').write('%i %s' % (cluster, schedd) if schedd else str(cluster))
if njobs_sub != njobs:
ok = False
except ValueError:
ok = False
if not ok:
print '\033[1m problem! \033[0m'
print submit_out
else:
print '\x1b[92msuccess!\x1b[0m cluster', cluster
if schedd:
cluster_link = os.path.join(cls.links_dir, schedd, str(cluster))
else:
cluster_link = os.path.join(cls.links_dir, str(cluster))
if os.path.islink(cluster_link):
print 'warning: clobbering old link:', os.readlink(cluster_link)
os.unlink(cluster_link)
os.symlink(os.path.abspath(working_dir), cluster_link)
finally:
os.chdir(cwd)
if 0:
from DBS import *
for s in data_samples + mfv_signal_samples:
n1, n2 = s.datasets['main'].nevents_orig, s.datasets['miniaod'].nevents_orig
if n1 != n2:
print s.name, n1, n2
if 0:
aod_strings = ['RunIIFall15DR76-PU25nsData2015v1', 'RunIISummer16DR80Premix-PUMoriond17']
miniaod_strings = ['RunIIFall15MiniAODv2-PU25nsData2015v1', 'RunIISummer16MiniAODv2-PUMoriond17']
no = 'Trains Material ALCA Flat RunIISpring16MiniAODv1'.split()
from JMTucker.Tools.general import popen
for s in xx4j_samples_2015: #qcd_samples + ttbar_samples + leptonic_background_samples:
print s.name
#print s.primary_dataset
output = popen('dasgoclient_linux -query "dataset=/%s/*/*AODSIM"' % s.primary_dataset).split('\n')
for y in 'AODSIM MINIAODSIM'.split():
for x in output:
ok = True
for n in no:
if n in x:
ok = False
if not ok:
continue
#print x
if x.endswith('/' + y):
nevents = None
for line in popen('dasgoclient_linux -query "dataset=%s | grep dataset.nevents"' % x).split('\n'):
try:
nevents = int(float(line))
except ValueError: