def upload():
"""
upload files that are processed
"""
try:
(conn, cur) = dbscripts.open()
if alreadyLocked(cfg.upload_lock):
logging.info("upload(): Lockfile exists")
return
q = '''
SELECT * FROM Runs WHERE status=%d
''' % dbcfg.processed
runsToUpload = cur.execute(q).fetchall()
for run in runsToUpload:
try:
rt = upload_dqmgui(runNumber=run[0],
runType=run[1])
run_number = run[0]
if rt==0:
q = '''
UPDATE Runs
SET status=%d
WHERE run_number=%d;
''' % (dbcfg.uploaded, run_number)
logging.debug(q)
cur.execute(q)
else:
raise dbscripts.SQLException("DQMGUI Uploading failed for record=%s" % str(run))
except Exception as exc:
q = '''
UPDATE Runs
SET status=%d
WHERE run_number=%d;
''' % (dbcfg.uploading_failed, run_number)
logging.debug(q)
cur.execute(q)
logging.error("upload(): Error %s with message: %s" % (
type(exc).__name__, exc.args))
finally:
conn.commit()
except Exception as exc:
pass
finally:
conn.commit()
conn.close()
shell.rm(cfg.upload_lock)
def reupload():
try:
(conn, cur) = dbscripts.open()
q = '''
SELECT * FROM Runs WHERE status=%d
''' % dbcfg.uploading_failed
runsToReupload = cur.execute(q).fetchall()
for run in runsToReupload:
try:
rt = upload_dqmgui(runNumber=run[0],
runType=run[1])
runNumber = run[0]
if rt==0:
q = '''
UPDATE Runs
SET status=%d
WHERE run_number=%d
''' % (dbcfg.uploaded, runNumber)
logging.debug(q)
cur.execute(q)
else:
raise dbscripts.SQLException("DQMGUI Uploading failed for record=%s" % str(run))
except Exception as exc:
q = '''
UPDATE Runs
SET status=%d
WHERE run_number=%d;
''' % (dbcfg.uploading_failed, runNumber)
logging.debug(q)
cur.execute(q)
logging.error("upload(): Error %s with message: %s" % (
type(exc).__name__, exc.args))
finally:
conn.commit()
except Exception as exc:
logging.error("reupload(): Error %s with message: %s" % (
type(exc).__name__, str(exc.args)))
finally:
conn.commit()
conn.close()
def reupload():
try:
(conn, cur) = dbscripts.open()
q = '''
SELECT * FROM %s WHERE status=%d
''' % (cfg.table_name, dbcfg.uploading_failed)
runsToReupload = cur.execute(q).fetchall()
for run in runsToReupload:
try:
rt = upload_dqmgui(runNumber=run[0],
runType=run[1])
runNumber = run[0]
if rt==0:
q = '''
UPDATE %s
SET status=%d
WHERE run_number=%d
''' % (cfg.table_name, dbcfg.uploaded, runNumber)
logging.debug(q)
cur.execute(q)
else:
raise dbscripts.SQLException("DQMGUI Uploading failed for record=%s" % str(run))
except Exception as exc:
q = '''
UPDATE %s
SET status=%d
WHERE run_number=%d;
''' % (cfg.table_name, dbcfg.uploading_failed, runNumber)
logging.debug(q)
cur.execute(q)
logging.error("upload(): Error %s with message: %s" % (
type(exc).__name__, exc.args))
finally:
conn.commit()
except Exception as exc:
logging.error("reupload(): Error %s with message: %s" % (
type(exc).__name__, str(exc.args)))
finally:
conn.commit()
conn.close()
def process():
"""
process something
"""
logging.info("Running function process")
try:
runlist = listRuns(cfg)
logging.debug("RunList: " + str(runlist))
(conn, cur) = dbscripts.open(cfg.dbpathname)
if alreadyLocked(cfg.process_lock):
logging.info("process(): Lockfile exists")
return
logging.debug("Starting the Run Loop")
# do initialization before looping over all files
for f in runlist:
try:
fstripped = f.split("/")[-1]
logging.debug(fstripped)
run_number = res.getRunNumber(cfg.ptype, fstripped)
logging.debug(run_number)
run_type = getRunType(cfg, run_number=run_number)
logging.debug((run_number, run_type))
if not shouldProcess(run_number=run_number,
run_type=run_type, cfg=cfg): continue
q = '''
SELECT * FROM %s WHERE run_number=%d;
''' % (cfg.table_name, run_number)
logging.debug(q)
results = cur.execute(q).fetchall()
logging.debug(results)
if len(results)==1:
logging.debug("Run %s is already in DB" % run_number)
continue
elif len(results)>1:
raise Exception("More than 1 Record for run_number=%d" % run_number)
else:
q = '''
INSERT INTO %s Values(%d, '%s', %d, %d);
''' % (cfg.table_name, run_number, run_type, 0, dbcfg.processing)
logging.debug(q)
cur.execute(q)
rt = execute(filepath=f)
if rt==0:
q = '''
UPDATE %s
SET status=%d
WHERE run_number=%d;
''' % (cfg.table_name, dbcfg.processed, run_number)
cur.execute(q)
else:
raise dbscripts.SQLException("Processing failed for record=%s" % str(run_number))
except Exception as exc:
q = '''
UPDATE %s
SET status=%d
WHERE run_number=%d;
''' % (cfg.table_name, dbcfg.processing_failed, run_number)
cur.execute(q)
logging.error("process(): Error %s with message: %s" % (
type(exc).__name__, exc.args))
finally:
conn.commit()
except Exception as exc:
logging.error("process(): Exception caught: %s %s" % (
type(exc).__name__, exc.args))
finally:
conn.commit()
conn.close()
shell.rm(cfg.process_lock)
def process():
"""
process something
"""
# external try
try:
# configure the current processing
runlist = res.listRuns(ptype)
logging.debug("RunList: " + str(runlist))
(conn, cur) = dbscripts.open()
if alreadyLocked(cfg.process_lock):
logging.info("process(): Lockfile exists")
return
# build the CMSSW
shell.execute(["cd", cfg.cmssw_src_directory])
shell.execute(["scram", "b", "-j", "8"])
# internal try
for f in runlist:
try:
fstripped = f.split("/")[-1]
run_number = res.getRunNumber(cfg.ptype, fstripped)
run_type = res.getRunType(cfg.ptype, f)
size = shell.getsize(f)
if not res.shouldProcess(run_number=run_number,
run_type=run_type, size=size, cfg=cfg): continue
logging.debug((run_number, run_type, size))
# query our db
q = '''
SELECT * FROM Runs WHERE run_number=%d;
''' % run_number
logging.debug(q)
results = cur.execute(q).fetchall()
logging.debug(results)
if len(results)==1:
logging.debug("Run %d is already in DB" % run_number)
continue
elif len(results)>1:
raise Exception("More than 1 Record for run_number=%d" % run_number)
else: # if it's ==0
q = '''
INSERT INTO Runs Values(%d, '%s', %d, %d);
''' % (run_number, run_type, 0, dbcfg.processing)
logging.debug(q)
cur.execute(q)
rt = process_cmssw(runType=run_type,
filepath=f)
if rt==0:
q = '''
UPDATE Runs
SET status=%d
WHERE run_number=%d;
''' % (dbcfg.processed, run_number)
cur.execute(q)
else:
raise dbscripts.SQLException("CMSSW Processing failed for record=%s" % str(run_number))
except Exception as exc:
q = '''
UPDATE Runs
SET status=%d
WHERE run_number=%d;
''' % (dbcfg.processing_failed, run_number)
cur.execute(q)
logging.error("process(): Error %s with message: %s" % (
type(exc).__name__, exc.msg))
finally:
conn.commit()
except Exception as exc:
logging.error("process(): Exception caught: %s %s" % (
type(exc).__name__, exc.args))
finally:
conn.commit()
conn.close()
shell.rm(cfg.process_lock)
def process():
"""
process something
"""
logging.info("Running function process")
try:
runlist = listRuns(cfg)
logging.debug("RunList: " + str(runlist))
(conn, cur) = dbscripts.open(cfg.dbpathname)
if alreadyLocked(cfg.process_lock):
logging.info("process(): Lockfile exists")
return
logging.debug("Starting the Run Loop")
# do initialization before looping over all files
for f in runlist:
try:
fstripped = f.split("/")[-1]
logging.debug(fstripped)
run_number = res.getRunNumber(cfg.ptype, fstripped)
logging.debug(run_number)
run_type = getRunType(cfg, run_number=run_number)
logging.debug((run_number, run_type))
if not shouldProcess(
run_number=run_number, run_type=run_type, cfg=cfg):
continue
q = '''
SELECT * FROM %s WHERE run_number=%d;
''' % (cfg.table_name, run_number)
logging.debug(q)
results = cur.execute(q).fetchall()
logging.debug(results)
if len(results) == 1:
logging.debug("Run %s is already in DB" % run_number)
continue
elif len(results) > 1:
raise Exception("More than 1 Record for run_number=%d" %
run_number)
else:
q = '''
INSERT INTO %s Values(%d, '%s', %d, %d);
''' % (cfg.table_name, run_number, run_type, 0,
dbcfg.processing)
logging.debug(q)
cur.execute(q)
rt = execute(filepath=f)
if rt == 0:
q = '''
UPDATE %s
SET status=%d
WHERE run_number=%d;
''' % (cfg.table_name, dbcfg.processed, run_number)
cur.execute(q)
else:
raise dbscripts.SQLException(
"Processing failed for record=%s" %
str(run_number))
except Exception as exc:
q = '''
UPDATE %s
SET status=%d
WHERE run_number=%d;
''' % (cfg.table_name, dbcfg.processing_failed, run_number)
cur.execute(q)
logging.error("process(): Error %s with message: %s" %
(type(exc).__name__, exc.args))
finally:
conn.commit()
except Exception as exc:
logging.error("process(): Exception caught: %s %s" %
(type(exc).__name__, exc.args))
finally:
conn.commit()
conn.close()
shell.rm(cfg.process_lock)