def find_gr(run, grdir, hhmmMin=None, quiet=False):
d = "%s/000/%03d/%03d/00000/" % (grdir, run/1000, run % 1000)
stat = "%s stat %s" % (eos(), d)
ls = stat.replace(" stat ", " ls -l ")
if utils.commandOutputFull(stat)["returncode"]:
return
listings = filter(lambda x: x, utils.commandOutputFull(ls)["stdout"].split("\n"))
month_num = {"Jan": 1, "Feb": 2, "Mar": 3, "Apr": 4, "May": 5, "Jun": 6,
"Jul": 7, "Aug": 8, "Sep": 9, "Oct": 10, "Nov": 11, "Dec": 12}
coords = []
for listing in listings:
fields = listing.split()
month, day, hhmm, fileName = fields[-4:]
hh, mm = hhmm.split(":")
coords.append((month_num[month], int(day), int(hh), int(mm), fileName))
coords.sort()
if hhmmMin:
mmMin = hhmmMin % 100
hhMin = hhmmMin / 100
coords = filter(lambda x: hhMin < x[2] or (hhMin == x[2] and mmMin <= x[3]), coords)
if not quiet:
for c in coords:
print c
files = [c[-1] for c in coords]
if files:
l = ",".join(["%s/%s%s" % (eosprefix, d, f) for f in files])
return l
def global_eos(run, hhmmMin=None, quiet=False):
month_num = {"Jan": 1, "Feb": 2, "Mar": 3, "Apr": 4, "May": 5, "Jun": 6,
"Jul": 7, "Aug": 8, "Sep": 9, "Oct": 10, "Nov": 11, "Dec": 12}
for d in sw.dirs_global(run):
shortName = d[d.find(".ch/") + 4:]
stat = utils.commandOutputFull("eos stat %s" % shortName)
if stat["returncode"]:
continue
ll = utils.commandOutputFull("eos ls -l %s" % d)
listings = filter(lambda x: x, ll["stdout"].split("\n"))
coords = []
for listing in listings:
fields = listing.split()
month, day, hhmm, fileName = fields[-4:]
hh, mm = hhmm.split(":")
coords.append((month_num[month], int(day), int(hh), int(mm), fileName))
coords.sort()
if hhmmMin:
mmMin = hhmmMin % 100
hhMin = hhmmMin / 100
coords = filter(lambda x: hhMin < x[2] or (hhMin == x[2] and mmMin <= x[3]), coords)
if not quiet:
for c in coords:
print c
files = []
for c in coords:
files.append(d + c[-1])
return files
def find_gr(run, grdir, hhmmMin=None, quiet=False):
d = "%s/000/%03d/%03d/00000/" % (grdir, run / 1000, run % 1000)
stat = "%s stat %s" % (eos(), d)
ls = stat.replace(" stat ", " ls -l ")
if utils.commandOutputFull(stat)["returncode"]:
return
listings = filter(lambda x: x,
utils.commandOutputFull(ls)["stdout"].split("\n"))
month_num = {
"Jan": 1,
"Feb": 2,
"Mar": 3,
"Apr": 4,
"May": 5,
"Jun": 6,
"Jul": 7,
"Aug": 8,
"Sep": 9,
"Oct": 10,
"Nov": 11,
"Dec": 12
}
coords = []
for listing in listings:
fields = listing.split()
month, day, hhmm, fileName = fields[-4:]
hh, mm = hhmm.split(":")
coords.append((month_num[month], int(day), int(hh), int(mm), fileName))
coords.sort()
if hhmmMin:
mmMin = hhmmMin % 100
hhMin = hhmmMin / 100
coords = filter(
lambda x: hhMin < x[2] or (hhMin == x[2] and mmMin <= x[3]),
coords)
if not quiet:
for c in coords:
print c
files = [c[-1] for c in coords]
if files:
l = ",".join(["%s/%s%s" % (eosprefix, d, f) for f in files])
return l
def local_eos(run):
out = []
for filename in sw.files_eos_local(run):
shortName = filename[filename.find("/store"):]
if not utils.commandOutputFull("eos stat %s" % shortName)["returncode"]:
out.append(filename)
return out
def go(
baseDir="",
runs=[],
process=lambda inputFile, outputDir, run: {},
dependsUpon=[],
urlPrefix="http://cmsdoc.cern.ch",
jobCheck="oneRun",
nProcMax=5,
debug=False,
):
not_found = []
for run in runs:
processes = stdout("ps -ef | grep %s | grep %s" %
(os.environ["USER"], jobCheck))
if nProcMax < len(processes):
msg = "Already %d processes:" % len(processes)
msg += "\n".join(processes)
sys.exit(msg)
runDir = makeDir(baseDir, run)
suffix = process.__name__
ready, procFlag, doneFlag = flags(runDir=runDir,
run=run,
suffix=suffix,
dependsUpon=dependsUpon)
if debug:
print run, ready, procFlag, doneFlag, process
if ready:
fileName = "/store/group/dpg_hcal/comm_hcal/USC/USC_%d.root" % run
if utils.commandOutputFull("%s stat %s" %
(eos(), fileName))["returncode"]:
not_found.append(run)
continue
else:
fileName = "root://eoscms.cern.ch/%s" % fileName
stdout("touch %s" % procFlag)
d = process(inputFile=fileName, outputDir=runDir, run=run)
stdout("rm %s" % procFlag)
urlSuffix = runDir[7 + runDir.find("public/"):]
url = "%s/~%s/%s" % (urlPrefix, os.environ["USER"], urlSuffix)
report(d, subject='Run %d: %s (%s)' % (run, suffix, url))
if not d["returncode"]:
with open(doneFlag, "w") as f:
print >> f, blob(d)
if not_found:
print "Runs not found in EOS:"
print not_found
def go(baseDir="",
runs=[],
process=lambda inputFile, outputDir, run: {},
dependsUpon=[],
urlPrefix="http://cmsdoc.cern.ch",
jobCheck="oneRun",
nProcMax=5,
debug=False,
):
not_found = []
for run in runs:
processes = stdout("ps -ef | grep %s | grep %s" % (os.environ["USER"], jobCheck))
if nProcMax < len(processes):
msg = "Already %d processes:" % len(processes)
msg += "\n".join(processes)
sys.exit(msg)
runDir = makeDir(baseDir, run)
suffix = process.__name__
ready, procFlag, doneFlag = flags(runDir=runDir,
run=run,
suffix=suffix,
dependsUpon=dependsUpon)
if debug:
print(run, ready, procFlag, doneFlag, process)
if ready:
fileName = "/store/group/dpg_hcal/comm_hcal/USC/USC_%d.root" % run
if utils.commandOutputFull("eos stat %s" % fileName)["returncode"]:
not_found.append(run)
continue
else:
fileName = "root://eoscms.cern.ch/%s" % fileName
stdout("touch %s" % procFlag)
d = process(inputFile=fileName,
outputDir=runDir,
run=run)
stdout("rm %s" % procFlag)
urlSuffix = runDir[7+runDir.find("public/"):]
url = "%s/~%s/%s" % (urlPrefix, os.environ["USER"], urlSuffix)
report(d, subject='Run %d: %s (%s)' % (run, suffix, url))
if not d["returncode"]:
with open(doneFlag, "w") as f:
print(blob(d), file=f)
if not_found:
print("Runs not found in EOS:")
print(not_found)
def find2(run):
LS1 = "/store/group/dpg_hcal/comm_hcal/LS1/USC_%d.root" % run
stat = "%s stat %s" % (eos(), LS1)
if not utils.commandOutputFull(stat)["returncode"]:
return "%s/%s" % (eosprefix, LS1)
def commandOutput(command):
d = utils.commandOutputFull(command)
d["command"] = command
return d
def commandOutput(command):
d = utils.commandOutputFull(command)
d["command"] = command
return d
def find2(run):
USC = "/store/group/dpg_hcal/comm_hcal/USC/USC_%d.root" % run
stat = "%s stat %s" % (eos(), USC)
if not utils.commandOutputFull(stat)["returncode"]:
return "%s/%s" % (eosprefix, USC)
