def query_local(cfg, **wargs):
"""
Returns the run type
"""
def parse(out):
out = out.split("\n")[0]
logging.debug(out)
v = out[:-4].split("_")
v = v[1:]
t = ""
for x in v:
t+=x.lower()
logging.debug(t)
return t
run_number = wargs["run_number"]
quantity = "CONFIG_NEW"
selector_type = cfg.quantitymap[quantity][0]
selector_name = cfg.quantitymap[quantity][1]
cmd = "sqlplus -S %s@cms_rcms @%s %s %s %s" % (cfg.runinfo_db_name,
cfg.sql_template, selector_type, selector_name, str(run_number))
logging.debug(cmd)
out, err, rt = shell.execute(cmd.split(" "))
logging.debug(out); logging.debug(rt); logging.debug(err)
return parse(out)
def process_cmssw(*kargs, **wargs):
logging.debug("Runing process_cmssw()")
runType = wargs["runType"]
filepath = wargs["filepath"]
cmd_as_list = cfg.cmsRun_cmd_template.format(cmssw_config=cfg.cmssw_config,
pathToFileName=filepath, runType=runType).split(" ")
logging.debug(cmd_as_list)
out, err, rt = shell.execute(cmd_as_list)
return rt
def process_cmssw(*kargs, **wargs):
logging.debug("Runing process_cmssw()")
runType = wargs["runType"]
filepath = wargs["filepath"]
cmd_as_list = cfg.cmsRun_cmd_template.format(cmssw_config=cfg.cmssw_config,
pathToFileName=filepath, runType=runType).split(" ")
logging.debug(cmd_as_list)
out, err, rt = shell.execute(cmd_as_list)
logging.info(out)
logging.info(err)
return rt
def listFullFilePath(cfg, f):
cmd = "%s ls %s/" % (cfg.eosexe, os.path.join(cfg.poolsource, f))
logging.debug(cmd)
out, err, rt = shell.execute(cmd.split())
logging.debug(err); logging.debug(out), logging.debug(rt)
out = out.split("\n")
l = []
for o in out:
if ".root" in o:
l.append(os.path.join(cfg.poolsource, f, o))
return l[0]
def listRuns(cfg):
"""
List all the runs
"""
if cfg.ptype=="904":
cmd = "%s ls %s" % (cfg.eosexe, cfg.poolsource)
out, err, rt = shell.execute(cmd.split(" "))
logging.debug(err)
logging.debug(rt)
return out.split("\n")[:-1]
else: return None
def upload_dqmgui(*kargs, **wargs):
logging.debug("Running process_dqmgui()")
runNumber = wargs["runNumber"]
runType = wargs["runType"]
fileToUpload = os.path.join(cfg.cmssw_outputpool,
cfg.cmssw_output_template.format(runNumber=runNumber, runType=runType))
cmd_as_list = cfg.dqmupload_cmd_template.format(
server_name=cfg.dqmgui_server_name,
pathnameToFile=fileToUpload
).split(" ")
logging.debug(cmd_as_list)
out,err,rt = shell.execute(cmd_as_list)
return rt
def upload_dqmgui(*kargs, **wargs):
logging.debug("Running process_dqmgui()")
runNumber = wargs["runNumber"]
runType = wargs["runType"]
fileToUpload = os.path.join(cfg.cmssw_outputpool,
cfg.cmssw_output_template.format(runNumber=runNumber, runType=runType))
cmd_as_list = cfg.dqmupload_cmd_template.format(
server_name=cfg.dqmgui_server_name,
pathnameToFile=fileToUpload
).split(" ")
logging.debug(cmd_as_list)
out,err,rt = shell.execute(cmd_as_list)
logging.info(out)
logging.info(err)
return rt
def execute(filepath):
cmd = "%s %s" % (cfg.executable, filepath)
cmd_as_list = cmd.split()
out, err, rt = shell.execute(cmd_as_list)
logging.error(err)
return rt
def process():
"""
process something
"""
# external try
try:
# configure the current processing
runlist = res.listRuns(ptype)
logging.debug("RunList: " + str(runlist))
(conn, cur) = dbscripts.open()
if alreadyLocked(cfg.process_lock):
logging.info("process(): Lockfile exists")
return
# build the CMSSW
shell.execute(["cd", cfg.cmssw_src_directory])
shell.execute(["scram", "b", "-j", "8"])
# internal try
for f in runlist:
try:
fstripped = f.split("/")[-1]
run_number = res.getRunNumber(cfg.ptype, fstripped)
run_type = res.getRunType(cfg.ptype, f)
size = shell.getsize(f)
if not res.shouldProcess(run_number=run_number,
run_type=run_type, size=size, cfg=cfg): continue
logging.debug((run_number, run_type, size))
# query our db
q = '''
SELECT * FROM Runs WHERE run_number=%d;
''' % run_number
logging.debug(q)
results = cur.execute(q).fetchall()
logging.debug(results)
if len(results)==1:
logging.debug("Run %d is already in DB" % run_number)
continue
elif len(results)>1:
raise Exception("More than 1 Record for run_number=%d" % run_number)
else: # if it's ==0
q = '''
INSERT INTO Runs Values(%d, '%s', %d, %d);
''' % (run_number, run_type, 0, dbcfg.processing)
logging.debug(q)
cur.execute(q)
rt = process_cmssw(runType=run_type,
filepath=f)
if rt==0:
q = '''
UPDATE Runs
SET status=%d
WHERE run_number=%d;
''' % (dbcfg.processed, run_number)
cur.execute(q)
else:
raise dbscripts.SQLException("CMSSW Processing failed for record=%s" % str(run_number))
except Exception as exc:
q = '''
UPDATE Runs
SET status=%d
WHERE run_number=%d;
''' % (dbcfg.processing_failed, run_number)
cur.execute(q)
logging.error("process(): Error %s with message: %s" % (
type(exc).__name__, exc.msg))
finally:
conn.commit()
except Exception as exc:
logging.error("process(): Exception caught: %s %s" % (
type(exc).__name__, exc.args))
finally:
conn.commit()
conn.close()
shell.rm(cfg.process_lock)
def process():
"""
process something
"""
# external try
try:
# configure the current processing
runlist = listRuns(cfg)
logging.debug("RunList: " + str(runlist))
# build the CMSSW
shell.execute(["cd", cfg.cmssw_src_directory])
#shell.execute(["scram", "b", "-j", "8"])
# internal try
for f in runlist:
try:
logging.debug(f)
f = listFullFilePath(cfg, f)
logging.debug(f)
fstripped = f.split("/")[-1]
logging.debug(fstripped)
run_number = res.getRunNumber(cfg.ptype, fstripped)
run_type = getRunType(cfg, run_number=run_number)
if not shouldProcess(run_number=run_number,
run_type=run_type, cfg=cfg): continue
logging.debug((run_number, run_type))
#
# query our db
# for MT, each query is wrapped into a separate connection
# inefficient - but given processing time of cmssw, it's negligible
#
q = '''
SELECT * FROM %s WHERE run_number=%d;
''' % (cfg.table_name, run_number)
logging.debug(q)
results = dbscripts.query_select(dbpathname=cfg.dbpathname,
query=q)
if results==None: continue
logging.debug(results)
if len(results)==1:
logging.debug("Run %d is already in DB" % run_number)
continue
elif len(results)>1:
raise Exception("More than 1 Record for run_number=%d" % run_number)
else: # if it's ==0
q = '''
INSERT INTO %s Values(%d, '%s', %d, %d);
''' % (cfg.table_name, run_number, run_type, 0, dbcfg.processing)
logging.debug(q)
dbscripts.query_update(dbpathname=cfg.dbpathname,
query=q)
rt = process_cmssw(runType=run_type,
filepath=f)
if rt==0:
q = '''
UPDATE %s
SET status=%d
WHERE run_number=%d;
''' % (cfg.table_name, dbcfg.processed, run_number)
dbscripts.query_update(dbpathname=cfg.dbpathname,
query=q)
else:
raise dbscripts.SQLException("CMSSW Processing failed for record=%s" % str(run_number))
except Exception as exc:
q = '''
UPDATE %s
SET status=%d
WHERE run_number=%d;
''' % (cfg.table_name, dbcfg.processing_failed, run_number)
dbscripts.query_update(dbpathname=cfg.dbpathname,
query=q)
logging.error("process(): Error %s with message: %s" % (
type(exc).__name__, exc.args))
finally:
pass
except Exception as exc:
logging.error("process(): Exception caught: %s %s" % (
type(exc).__name__, exc.args))
finally:
pass
def execute(filepath):
cmd = "%s %s" % (cfg.executable, filepath)
cmd_as_list = cmd.split()
out, err, rt = shell.execute(cmd_as_list)
logging.error(err)
return rt