def _analysis(name, verbose, **kwargs):
a = root.pa.Analysis(name)
for k,v in kwargs.iteritems():
if not hasattr(a, k):
logger.error('PandaAnalysis.Flat.analysis','Could not set property %s'%k)
return None
setattr(a, k, bool(v))
setattr(a, 'dump', lambda : _dump(a))
if verbose:
a.dump()
return a
def run(self, f_out):
logger.info('fitting_forest.Process.run', 'Running ' + self.name)
branches = sorted(self.all_branches.values())
try:
xarr = root_interface.read_tree(tree=self.tree,
branches=branches,
cut=self.cut)
fields = self.variables.keys() + ['nominal']
self.__write_out(f_out, xarr, fields, '')
for shift, weight in self.weights.iteritems():
fields = self.variables.keys() + [shift]
self.__write_out(f_out, xarr, fields, '_' + shift)
except ValueError as e:
logger.error('fitting_forest.Process.run', str(e))
return
def read_r_model(mV, mDM=100, couplings='nominal'):
tmpl = getenv('PANDA_XSECS') + '/resonant/%i_%i.dat'
try:
fdat = open(tmpl % (mV, mDM))
except IOError:
logger.error('PandaCore.Tools.models.read_nr_model',
'Could not open %s' % (tmpl % (mV, mDM)))
return None
for line in fdat:
line_coupling, sigma = line.split(':')
if not (line_coupling == couplings):
continue
sigma = float(sigma)
p = ModelParams(mV, mDM, 1, 1, 0.25, 0.25, sigma, 0)
fdat.close()
return p
def kill(self, idle_only=False):
if not self.cluster_id:
logger.error(
self.__class__.__name__ + ".kill",
"This submission has not been executed yet (ClusterId not set)"
)
raise RuntimeError
if idle_only:
proc_ids = self.query_status(return_ids=True)['idle']
else:
proc_ids = self.proc_ids
N = self.schedd.act(htcondor.JobAction.Remove,
["%s.%s" % (self.cluster_id, p)
for p in proc_ids])['TotalSuccess']
if N:
logger.info(
self.__class__.__name__ + '.kill',
'Killed %i jobs in ClusterId=%i' % (N, self.cluster_id))
def query_status(self, return_ids=False):
if not self.cluster_id:
logger.error(
self.__class__.__name__ + ".query_status",
"This submission has not been executed yet (ClusterId not set)"
)
raise RuntimeError
jobs = {x: [] for x in ['T3', 'T2', 'idle', 'held', 'other']}
try:
results = self.schedd.query('ClusterId =?= %i' % (self.cluster_id))
except IOError: # schedd is down!
return jobs
for job in results:
proc_id = int(job['ProcId'])
status = job['JobStatus']
try:
if return_ids:
samples = [proc_id]
elif type(self.arguments) == dict:
samples = [self.arguments[self.proc_ids[proc_id]]]
else:
samples = self.proc_ids[proc_id].split()
except KeyError:
continue # sometimes one extra dummy job is created and not tracked, oh well
if job_status[status] == 'running':
try:
remote_host = job['RemoteHost']
if '@T3' in remote_host:
status = job_status_rev['T3']
else:
status = job_status_rev['T2']
except KeyError:
status = 1 # call it idle, job is probably moving between states
pass
if job_status[status] in jobs:
jobs[job_status[status]] += samples
else:
jobs['other'] += samples
return jobs
def read_nr_model(mV, mDM, couplings=None, path='non-resonant'):
tmpl = getenv('PANDA_XSECS') + '/' + path + '/%i_%i_xsec_gencut.dat'
try:
fdat = open(tmpl % (mV, mDM))
except IOError:
logger.error('PandaCore.Tools.models.read_nr_model',
'Could not open %s' % (tmpl % (mV, mDM)))
return None
for line in fdat:
if 'med dm' in line:
continue
p = ModelParams(*[float(x) for x in line.strip().split()])
if couplings:
if couplings == (p.gV_DM, p.gA_DM, p.gV_q, p.gA_q):
fdat.close()
return p
else:
continue
else:
# if not specified, take the first valid model (nominal)
fdat.close()
return p
def __init__(self,
cache_dir,
executable=None,
arglist=None,
arguments=None,
nper=1):
super(SimpleSubmission, self).__init__(cache_dir + '/submission.pkl')
self.cache_dir = cache_dir
if executable != None:
self.executable = executable
self.arguments = arguments
self.arglist = arglist
self.nper = nper
else:
try:
pkl = pickle.load(open(self.cache_filepath))
last_sub = pkl[-1]
self.executable = last_sub.executable
self.arguments = last_sub.arguments
self.arglist = last_sub.arglist
self.nper = last_sub.nper
except:
logger.error(self.__class__.__name__ + '.__init__',
'Must provide a valid cache or arguments!')
raise RuntimeError
self.cmssw = getenv('CMSSW_BASE')
self.workdir = cache_dir + '/workdir/'
self.logdir = cache_dir + '/logdir/'
for d in [self.workdir, self.logdir]:
system('mkdir -p ' + d)
if type(self.arglist) == list:
with open(cache_dir + '/workdir/args.list', 'w') as fargs:
fargs.write('\n'.join(self.arglist))
if not self.arguments:
self.arguments = range(1, len(self.arglist) + 1)
self.arglist = cache_dir + '/workdir/args.list'
def setup_schedd(config='T3'):
config = config.split(':') if config else []
global pool_server, schedd_server, base_job_properties, should_spool
os = '&& OpSysAndVer == "SL6"' if 'SL6' in config else ''
if 'T3' in config:
base_job_properties = {
"Iwd":
"WORKDIR",
"Cmd":
"WORKDIR/exec.sh",
"WhenToTransferOutput":
"ON_EXIT",
"REQUIRED_OS":
"rhel7",
"PYTHONHOME":
"/home/ceballos/.local/lib/python2.7/site-packages/",
"ShouldTransferFiles":
"YES",
"Requirements":
classad.ExprTree(
'UidDomain == "mit.edu" && Arch == "X86_64" %s && TARGET.Machine != "t3btchXXX.mit.edu"'
% os),
"AcctGroup":
acct_grp_t3,
"AccountingGroup":
'%s.USER' % (acct_grp_t3),
"X509UserProxy":
"/tmp/x509up_uUID",
"OnExitHold":
classad.ExprTree("( ExitBySignal == true ) || ( ExitCode != 0 )"),
"In":
"/dev/null",
"TransferInput":
"WORKDIR/cmssw.tgz,WORKDIR/skim.py,WORKDIR/x509up",
}
pool_server = None
schedd_server = getenv('HOSTNAME')
should_spool = False
query_owner = getenv('USER')
elif 'T2' in config:
base_job_properties = {
"Iwd":
"WORKDIR",
"Cmd":
"WORKDIR/exec.sh",
"WhenToTransferOutput":
"ON_EXIT",
"ShouldTransferFiles":
"YES",
"Requirements":
classad.ExprTree('(Arch == "X86_64" %s) && \
((GLIDEIN_Site =!= "MIT_CampusFactory") || (GLIDEIN_Site == "MIT_CampusFactory" && \
BOSCOCluster == "ce03.cmsaf.mit.edu" && BOSCOGroup == "bosco_cms" && HAS_CVMFS_cms_cern_ch))'
% os),
#classad.ExprTree('UidDomain == "cmsaf.mit.edu" && Arch == "X86_64" && OpSysAndVer == "SL6"'),
"REQUIRED_OS":
"rhel7",
"AcctGroup":
'group_cmsuser.USER',
"AccountingGroup":
'group_cmsuser.USER',
"X509UserProxy":
"/tmp/x509up_uUID",
"OnExitHold":
classad.ExprTree("( ExitBySignal == true ) || ( ExitCode != 0 )"),
"In":
"/dev/null",
"TransferInput":
"WORKDIR/cmssw.tgz,WORKDIR/skim.py,WORKDIR/x509up",
}
pool_server = None
schedd_server = getenv('HOSTNAME')
should_spool = False
query_owner = getenv('USER')
elif 'T2Only' in config:
base_job_properties = {
"Iwd":
"WORKDIR",
"Cmd":
"WORKDIR/exec.sh",
"WhenToTransferOutput":
"ON_EXIT",
"ShouldTransferFiles":
"YES",
"Requirements":
classad.ExprTree('(Arch == "X86_64" %s) && \
((GLIDEIN_Site == "MIT_CampusFactory" && \
BOSCOCluster == "ce03.cmsaf.mit.edu" && BOSCOGroup == "bosco_cms" && HAS_CVMFS_cms_cern_ch))'
% os),
# classad.ExprTree('UidDomain == "cmsaf.mit.edu" && Arch == "X86_64" && OpSysAndVer == "SL6"'),
"REQUIRED_OS":
"rhel7",
"AcctGroup":
'group_cmsuser.USER',
"AccountingGroup":
'group_cmsuser.USER',
"X509UserProxy":
"/tmp/x509up_uUID",
"OnExitHold":
classad.ExprTree("( ExitBySignal == true ) || ( ExitCode != 0 )"),
"In":
"/dev/null",
"TransferInput":
"WORKDIR/cmssw.tgz,WORKDIR/skim.py,WORKDIR/x509up",
}
pool_server = None
schedd_server = getenv('HOSTNAME')
should_spool = False
query_owner = getenv('USER')
elif 'SubMIT' in config:
base_job_properties = {
"Iwd":
"WORKDIR",
"Cmd":
"WORKDIR/exec.sh",
"WhenToTransferOutput":
"ON_EXIT",
"ShouldTransferFiles":
"YES",
"Requirements":
classad.ExprTree(
'Arch == "X86_64" && TARGET.OpSys == "LINUX" && TARGET.HasFileTransfer && ( isUndefined(IS_GLIDEIN) || ( OSGVO_OS_STRING == "RHEL 6" && HAS_CVMFS_cms_cern_ch == true ) || GLIDEIN_REQUIRED_OS == "rhel7" || HAS_SINGULARITY == true || ( Has_CVMFS_cms_cern_ch == true && ( BOSCOGroup == "bosco_cms" ) ) ) && %s'
% (submit_exclude)),
"AcctGroup":
"analysis",
"AccountingGroup":
"analysis.USER",
"X509UserProxy":
"/tmp/x509up_uUID",
"OnExitHold":
classad.ExprTree("( ExitBySignal == true ) || ( ExitCode != 0 )"),
"In":
"/dev/null",
"TransferInput":
"WORKDIR/cmssw.tgz,WORKDIR/skim.py",
"ProjectName":
"CpDarkMatterSimulation",
"Rank":
"Mips",
'SubMITOwner':
'USER',
"REQUIRED_OS":
"rhel7",
"DESIRED_OS":
"rhel7",
"RequestDisk":
3000000,
"SingularityImage":
"/cvmfs/singularity.opensciencegrid.org/bbockelm/cms:rhel7",
}
pool_server = 'submit.mit.edu:9615'
schedd_server = 'submit.mit.edu'
query_owner = getenv('USER')
should_spool = False
else:
logger.error('job_management.setup_schedd',
'Unknown config "%s"' % (':'.join(config)))
raise ValueError
name = argv[2]
singletons = None
outdir = getenv('SUBMIT_NPY')
datadir = getenv('CMSSW_BASE') + '/src/PandaAnalysis/data/deep/'
submit_name = getenv('SUBMIT_NAME')
me = argv[0].split('/')[-1]
argv = []
system('mkdir -p tmp/')
data = {}
for fpath in fcfg.readlines():
print fpath.strip()
try:
d = np.load(fpath.strip())
except [IOError, AttributeError] as e:
logger.error(me, str(e))
continue
for k, v in d.iteritems():
if k == 'singleton_branches':
data[k] = v
continue
if v.shape[0]:
if k not in data:
data[k] = []
data[k].append(v)
if not len(data):
logger.info(me, 'This was an empty config!')
exit(0)
for k, v in data.iteritems():
flist = glob(argv[2])
target_evt = np.int64(argv[1])
lumis = []
pts = []
msds = []
pfs = []
for f in flist:
arr = np.load(f)
evt = arr['eventNumber']
mask = (evt == target_evt)
if np.sum(mask):
idx = np.argmax(mask)
pfs.append(arr['pf'][idx])
pts.append(arr['pt'][idx])
msds.append(arr['msd'][idx])
lumis.append(arr['lumi'])
logger.info(argv[0], 'Found %i in %s' % (target_evt, f))
if lumis:
np.savez('sliced.npz',
pf=np.array(pfs),
msd=np.array(msds),
pt=np.array(pts),
lumi=np.array(lumis))
else:
logger.error(argv[0], 'Could not find %i in %s' % (target_evt, argv[2]))
exit(1)