def parse_files():
# We will use this to identify this process in logs. Is done for investigation of double messages
script_start_datetime = datetime.now()
script_start_time = time.time()
script_name = os.urandom(8).encode('hex')
script_pid = os.getpid()
script_ppid = os.getppid()
script_uid = os.getuid()
script_start_clock = time.clock()
description = "The script updates RCDB gathering different sources given in --update flag:" \
" coda - information from coda file (file is required anyway to get run)" \
" config - run configuration file in HallD format" \
" roc - roc configuration files (taken from run configuration file)"\
" this option is run only if config is given"\
" epics - epics variables" \
"So now update of everything looks like: --update=coda,config,roc,epics" \
parser = argparse.ArgumentParser(description=description,
usage=get_usage())
parser.add_argument("coda_xml_log_file", help="Path to CODA run log file")
parser.add_argument("-v",
"--verbose",
help="increase output verbosity",
action="store_true")
parser.add_argument(
"--reason",
help="Reason of the update. 'start', 'update', 'end' or ''",
default="")
parser.add_argument("--update",
help="Comma separated, modules to update",
default="")
parser.add_argument(
"-c",
"--connection",
help="The connection string (like mysql://rcdb@localhost/rcdb)")
parser.add_argument("--udl",
help="UDL link to send messages to UDL logging")
parser.add_argument("--ipl",
help="Use inter-process lock, that allows ",
action="store_true")
parser.add_argument("--run-config-file",
help="Set custom path to run config file",
default="")
args = parser.parse_args()
# Figure out the parameters
log.setLevel(logging.DEBUG if args.verbose else logging.INFO)
# coda xml file name
coda_xml_log_file = args.coda_xml_log_file
log.debug(F("coda_xml_log_file = '{}'", coda_xml_log_file))
# Connection string
if args.connection:
con_string = args.connection
elif "RCDB_CONNECTION" in os.environ:
con_string = os.environ["RCDB_CONNECTION"]
else:
print(
"ERROR! RCDB_CONNECTION is not set and is not given as a script parameter (-c)"
)
parser.print_help()
sys.exit(2)
log.debug(F("con_string = '{}'", con_string))
# What to update
update_parts = []
if args.update:
update_parts = args.update.split(',')
log.debug(F("update_parts = {}", update_parts))
# Update reason
update_reason = args.reason
log.debug(F("update_reason = '{}'", update_reason))
# Use interprocess lock?
use_interprocess_lock = args.ipl
script_info = "'{script_start_datetime}', reason: '{reason}', parts: '{parts}, " \
"'pid: '{pid}', ppid: '{ppid}', uid: '{uid}', " \
.format(
script_start_datetime=script_start_datetime,
reason=update_reason,
parts=args.update,
pid=script_pid,
ppid=script_ppid,
uid=script_uid)
# Open DB connection
db = ConfigurationProvider(con_string)
# Ensure only one such process is running, to avoid duplicated records. See issues #25 #20 #19 on GitHub
if use_interprocess_lock:
lock_success = try_set_interprocess_lock()
wait_count = 0
while not lock_success:
# We failed to obtain the lock. Some other instance of this script is running now.
if update_reason == UpdateReasons.UPDATE:
log.info(
"The other instance is running. Since update_reason = update we just exit"
)
exit(0)
time.sleep(1)
wait_count += 1
log.debug(
F("{script_name}: Waiting lock for {waited}s",
script_name=script_name,
waited=wait_count))
if wait_count > 10:
log.error(
F(
"The other instance is running. Since this update reason is '{}', "
"this instance waited > 10s for the other one to end. But it still holds the lock",
update_reason))
# this is major problem. We'll try send it to DB before exit
db.add_log_record(
"",
"'{}': Exit!. The other instance is running. This instance waited > 10s! {}"
.format(script_name, script_info), 0)
exit(1)
lock_success = try_set_interprocess_lock()
# >oO DB logging
db.add_log_record("", "'{}': Start. {}".format(script_name, script_info),
0)
# Create update context
update_context = rcdb.UpdateContext(db, update_reason)
# CODA
# Parse coda xml and save to DB
log.debug(F("Parsing coda_xml_log_file='{}'", coda_xml_log_file))
coda_parse_result = coda_parser.parse_file(coda_xml_log_file)
run_number = coda_parse_result.run_number
run_config_file = coda_parse_result.run_config_file
log.debug(
F("Parsed coda_xml_log_file='{}'. run='{}', run_config_file='{}'",
coda_xml_log_file, run_number, run_config_file))
# >oO DEBUG log message
now_clock = time.clock()
db.add_log_record(
"",
"'{}':Parsed coda_xml_log_file='{}'. run='{}', run_config_file='{}', clocks='{}', time: '{}'"
.format(script_name, coda_xml_log_file, run_number,
run_config_file, run_number, now_clock - script_start_clock,
datetime.now()), run_number)
# Conditions from coda file save to DB
if "coda" in update_parts:
log.debug(F("Adding coda conditions to DB", ))
update_coda_conditions(update_context, coda_parse_result)
else:
log.debug(
F(
"Skipping to add coda conditions to DB. Use --update=...,coda to update it",
))
update_context.run = db.get_run(run_number)
if update_context.run is None:
log.warning(
F("No DB record for run '{}' is found! Further updates look impossible"
))
update_context.run = run_number
# Save coda file to DB
log.debug(F("Adding coda_xml_log_file to DB", ))
db.add_configuration_file(run_number,
coda_xml_log_file,
overwrite=True,
importance=ConfigurationFile.IMPORTANCE_HIGH)
# CONFIGURATION FILES
# Add run configuration file to DB... if it is run-start update
if args.run_config_file:
log.debug(
F(
"Flag --run-config-file is provided. Using this as a path to run_config_file: '{}'",
args.run_config_file))
run_config_file = args.run_config_file
if update_reason in [UpdateReasons.START, UpdateReasons.UNKNOWN
] and "config" in update_parts and run_config_file:
if os.path.isfile(run_config_file) and os.access(
run_config_file, os.R_OK):
# mmm just save for now
log.debug(F("Adding run_config_file to DB", ))
db.add_configuration_file(
run_number,
run_config_file,
importance=ConfigurationFile.IMPORTANCE_HIGH)
log.debug("Parsing run_config_file")
run_config_parse_result = parse_run_config_file(run_config_file)
log.debug("Parsed run_config_file. Updating conditions")
update_run_config_conditions(update_context,
run_config_parse_result)
log.debug("Updated run_config_file conditions")
if "roc" in update_parts:
log.debug("Adding ROC config files...")
add_roc_configuration_files(update_context,
run_config_parse_result)
log.debug("Done ROC config files!")
else:
log.warn("Config file '{}' is missing or is not readable".format(
run_config_file))
if "roc" in update_parts:
log.warn(
"Can't parse roc configs because there is no main config")
# Parse run configuration file and save to DB
# EPICS
# Get EPICS variables
epics_start_clock = time.clock()
if 'epics' in update_parts and run_number:
log.debug(F("Performing update_epics.py", ))
# noinspection PyBroadException
try:
import update_epics
conditions = update_epics.update_rcdb_conds(
db, run_number, update_reason)
epics_end_clock = time.clock()
# >oO DEBUG log message
if "beam_current" in conditions:
db.add_log_record(
"",
"'{}': Update epics. beam_current:'{}', epics_clocks:'{}' clocks:'{}', time: '{}'"
.format(script_name, conditions["beam_current"],
epics_end_clock - epics_start_clock,
epics_end_clock - script_start_clock,
datetime.now()), run_number)
except Exception as ex:
log.warn(
"update_epics.py failure. Impossible to run the script. Internal exception is:\n"
+ str(ex))
epics_end_clock = time.clock()
# >oO DEBUG log message
db.add_log_record(
"",
"'{}': ERROR update epics. Error type: '{}' message: '{}' trace: '{}' "
"||epics_clocks:'{}' clocks:'{}' time: '{}'".format(
script_name, type(ex), ex.message, traceback.format_exc(),
epics_end_clock - epics_start_clock,
epics_end_clock - script_start_clock, datetime.now()),
run_number)
log.debug("End of update")
# >oO DEBUG log message
now_clock = time.clock()
db.add_log_record(
"",
"'{}': End of update. Script proc clocks='{}', wall time: '{}', datetime: '{}'"
.format(script_name, now_clock - script_start_clock,
time.time() - script_start_time, datetime.now()), run_number)
def parse_start_run_info():
script_start_clock = time.clock()
script_start_time = time.time()
log = logging.getLogger('pvdb') # create run configuration standard logger
log.addHandler(logging.StreamHandler(
sys.stdout)) # add console output for logger
log.setLevel(
logging.INFO
) # DEBUG: print everything. Changed to logging. INFO for less output
parser = argparse.ArgumentParser(description="Update PVDB")
# parser.add_argument("config_xml_file", type=str, help="full path to configID.xml file")
parser.add_argument("session_xml_file",
type=str,
help="full path to controlSessions.xml file")
parser.add_argument("-v",
"--verbose",
help="increase output verbosity",
action="store_true")
parser.add_argument(
"--update",
help="Comma separated, modules to update such as coda,epics",
default="coda,epics")
parser.add_argument(
"--reason",
help="Reason of the udpate: 'start', 'udpate', 'end' or ''",
default="start")
parser.add_argument("--exp", help="Experiment name", default="CREX")
parser.add_argument(
"-c",
"--connection",
help="connection string (eg, mysql://pvdb@localhost/pvdb)")
args = parser.parse_args()
# Set log level
log.setLevel(logging.DEBUG if args.verbose else logging.INFO)
# coda xml files
# config_xml_file = args.config_xml_file
# log.debug(Lf("config_xml_file = '{}'", config_xml_file))
session_xml_file = args.session_xml_file
log.debug(Lf("session_xml_file = '{}' ", session_xml_file))
# Connection
if args.connection:
con_string = args.connection
elif "RCDB_CONNECTION" in os.environ:
con_string = os.environ["RCDB_CONNECTION"]
else:
print(
"ERROR! RCDB_CONNECTION is not set and is not given as a script parameter (-c)"
)
parser.print_help()
sys.exit(2)
log.debug(Lf("con_string = '{}'", con_string))
# What to update
update_parts = []
if args.update:
update_parts = args.update.split(',')
log.debug(Lf("update_parts = {}", update_parts))
# Update reason
update_reason = args.reason
log.debug(Lf("update_reason = '{}'", update_reason))
# Open DB connection
db = ConfigurationProvider(con_string)
# Create update context
update_context = rcdb.UpdateContext(db, update_reason)
# Parse coda files and save to DB
log.debug(Lf("Parsing coda__xml_files='{}'", session_xml_file))
coda_parse_result = parity_coda_parser.parse_start_run_data(
session_xml_file)
run_number = coda_parse_result.run_number
# >oO DEBUG log message
now_clock = time.clock()
db.add_log_record(
"", "Start Run Script: Parsed xml_files='{}'. run='{}'".format(
session_xml_file, run_number), run_number)
# Parse runstart.info (Run type, comment)
dict_info = parity_coda_parser.runinfo_parser()
if bool(dict_info["Run"]["type"]):
coda_parse_result.run_type = dict_info["Run"]["type"]
if bool(dict_info["comment"]["text"]):
coda_parse_result.user_comment = dict_info["comment"]["text"]
if bool(dict_info["parity"]["bmw"]):
coda_parse_result.bmw = dict_info["parity"]["bmw"]
if bool(dict_info["parity"]["feedback"]):
coda_parse_result.feedback = dict_info["parity"]["feedback"]
# Set experiment name
coda_parse_result.experiment_name = args.exp
# Coda conditions to DB
if "coda" in update_parts:
log.debug(Lf("Adding coda conditions to DB", ))
if test_mode:
print "Experiment name:\t", coda_parse_result.experiment_name
print "Run number:\t", coda_parse_result.run_number
print "Start time:\t", coda_parse_result.start_time
print "Run config:\t", coda_parse_result.run_config
print "Run type:\t", coda_parse_result.run_type
print "Comment:\t", coda_parse_result.user_comment
print "BMW:\t", coda_parse_result.bmw
print "Feedback:\t", coda_parse_result.feedback
else:
update_parity_coda_conditions(update_context, coda_parse_result)
else:
log.debug(
Lf(
"Skipping to add coda conditions to DB. Use --update=...,coda to update it",
))
"""
# Do we want to save files to DB?
log.debug(Lf("Adding coda_xml_log_file to DB", ))
db.add_configuration_file(run_number, coda_xml_log_file, overwrite=True, importance=ConfigurationFile.IMPORTANCE_HIGH)
"""
# EPICS Update
# Get EPICS variables
epics_start_clock = time.clock()
if test_mode:
conditions = epics_helper.get_run_conds()
print conditions
else:
if 'epics' in update_parts and run_number:
# noinspection PyBroadException
try:
conditions = epics_helper.update_db_conds(
db, run_number, update_reason)
epics_end_clock = time.clock()
# >oO DEBUG log message
if "beam_current" in conditions:
db.add_log_record(
"",
"Update epics. beam_current:'{}', time: '{}'".format(
conditions["beam_current"],
datetime.now()), run_number)
except Exception as ex:
log.warn(
"update_epics.py failure. Impossible to run the script. Internal exception is:\n"
+ str(ex))
epics_end_clock = time.clock()
# >oO DEBUG log message
db.add_log_record(
"",
"ERROR update epics. Error type: '{}' message: '{}' trace: '{}' "
"||epics_clocks:'{}' clocks:'{}' time: '{}'".format(
type(ex), ex.message, traceback.format_exc(),
epics_end_clock - epics_start_clock,
epics_end_clock - script_start_clock, datetime.now()),
run_number)
log.debug("End of update")
# >oO DEBUG log message
now_clock = time.clock()
if not test_mode:
db.add_log_record(
"",
"End of update. Script proc clocks='{}', wall time: '{}', datetime: '{}'"
.format(now_clock - script_start_clock,
time.time() - script_start_time,
datetime.now()), run_number)
def update():
description = "Script for DB udpate after CODA run end"
parser = argparse.ArgumentParser(description=description, usage=get_usage())
parser.add_argument("--run", type=str, help="Run number")
parser.add_argument("-v", "--verbose", help="increase output verbosity", action="store_true")
parser.add_argument("--update", help="Comma separated, modules to update such as coda,epics", default="")
parser.add_argument("--reason", help="Reason of the udpate: 'start', 'udpate', 'end' or ''", default="")
parser.add_argument("-c", "--connection", help="connection string (eg, mysql://pvdb@localhost/pvdb)")
args = parser.parse_args()
# Connection
if args.connection:
con_string = args.connection
elif "RCDB_CONNECTION" in os.environ:
con_string = os.environ["RCDB_CONNECTION"]
else:
print ("ERROR! RCDB_CONNECTION is not set and is not given as a script parameter (-c)")
parser.print_help()
sys.exit(2)
log.debug(Lf("con_string = '{}'", con_string))
# What to update
update_parts = []
if args.update:
update_parts = args.update.split(',')
log.debug(Lf("update_parts = {}", update_parts))
# Update reason
update_reason = args.reason
log.debug(Lf("update_reason = '{}'", update_reason))
# Open DB connection
db = ConfigurationProvider(con_string)
# Create update context
update_context = rcdb.UpdateContext(db, update_reason)
coda_path = None
coda_file_name = None
host = socket.gethostname()
if "adaq" in host:
coda_path = "/adaq2/data1/apar/"
elif "aonl" in host:
coda_path = "/adaq2/data1/apar/"
else:
coda_path = "/cache/halla/parity/raw/"
for files in glob.glob(coda_path+"*"+str(run)+"*.dat"):
if not "parity18" in files:
coda_file_name = files
if coda_file_name is None:
print "CODA file is not found", run
sys.exit()
coda_parse_result = parity_coda_parser.parse_coda_data_file(coda_file_name)
if "coda" in update_parts:
run_start.update_parity_coda_conditions(update_context, coda_parse_result)
def main():
# DB Connection
#------------------------------------------
update_parts = ["coda"]
update_reason = "update"
# Connection
if "RCDB_CONNECTION" in os.environ:
con_string = os.environ["RCDB_CONNECTION"]
else:
print ("ERROR! RCDB_CONNECTION is not set and is not given as a script parameter (-c)")
sys.exit(2)
# Open DB connection
db = ConfigurationProvider(con_string)
# Create update context
update_context = rcdb.UpdateContext(db, update_reason)
#------------------------------------------
# Read the list
"""
format:
Run, #Config, Date/Time, RunType, Beam Mode, Beam Current, Beam Energy, Target, Log entries (HALOG, HAPLOG), Run analyzed?, Comment, Raster X/Y, Helicity, IHWP setting
SAM 1 HVSAM 2 HVSAM 3 HVSAM 4 HVSAM 5 HVSAM 6 HVSAM 7 HVSAM 8 HV
"""
run_list = sys.argv[1]
with open('%s' % run_list, 'rb') as f:
for line in f:
values = []
# ignore the first line
if "Run #" in line:
continue
for item in line.split('\t'):
if item != '\t':
values.append(item)
run_type = values[3]
comment = values[10]
if '-' in values[0]:
# Loop over the range
run1=int(values[0].split('-')[0])
run2=int(values[0].split('-')[1])
for run in range(run1, run2+1):
print run
try:
post_update.update(str(run), update_parts, update_context)
db.add_condition(db.get_run(run), DefaultConditions.USER_COMMENT, comment)
if "junk" in run_type.lower():
db.add_condition(db.get_run(run), DefaultConditions.RUN_TYPE, "Junk", replace=True)
except Exception as ex:
print str(ex)
else:
if not values[0].isdigit():
continue
else:
try:
run=values[0]
print run
post_update.update(run, update_parts, update_context)
db.add_condition(db.get_run(run), DefaultConditions.USER_COMMENT, comment)
if "junk" in run_type.lower():
db.add_condition(db.get_run(run), DefaultConditions.RUN_TYPE, "Junk", replace=True)
except Exception as ex:
print str(ex)
# Connection
if args.connection:
con_string = args.connection
elif "RCDB_CONNECTION" in os.environ:
con_string = os.environ["RCDB_CONNECTION"]
else:
print(
"ERROR! RCDB_CONNECTION is not set and is not given as a script parameter (-c)"
)
sys.exit(2)
log.debug(Lf("con_string = '{}'", con_string))
# What to update
update_parts = []
if args.update:
update_parts = args.update.split(',')
log.debug(Lf("update_parts = {}", update_parts))
# Update reason
update_reason = args.reason
log.debug(Lf("update_reason = '{}'", update_reason))
# Open DB connection
db = ConfigurationProvider(con_string)
# Create update context
update_context = rcdb.UpdateContext(db, update_reason)
update(run_number, update_parts, update_context)