def update_run(db, run_number, reason):
if not db.get_run(run_number):
log.warn(Lf("Run '{}' is not found in DB", run_number))
return
run = db.get_run(run_number)
start_time = None
try:
start_time = run.start_time
except Exception as ex:
log.warn(
Lf("Error in start_time request : run='{}', '{}'", run_number, e))
if start_time is not None:
try:
conditions = epics_helper.update_db_conds(db, run, reason)
# conditions = epics_helper.mya_get_run_conds(run, log)
db.add_log_record(
"", "Update epics. beam_current:'{}', time: '{}'".format(
conditions["beam_current"], datetime.now()), run_number)
except Exception as ex:
log.warn("Update epics failed :" + str(ex))
db.add_log_record(
"",
"ERROR update epics. Error type: '{}' message: '{}' trace: '{}' "
"||time: '{}'".format(type(ex), ex.message,
traceback.format_exc(),
datetime.now()), run_number)
log.debug("End of Update")
def update_rcdb_conds(db, run, reason):
log = logging.getLogger(
'rcdb.update.epics') # create run configuration standard logger
log.debug(Lf("Running 'update_rcdb_conds(db={}, run={})'", db, run))
TOLERANCE = 1.e-5 # constant used for comparisons
# Run can be a rcdb.Run object or a run number
if not isinstance(run, Run):
log.debug(Lf("Getting run by number={}", run))
run = db.get_run(run)
# Build mapping of conditions to add to the RCDB, key is name of condition
conditions = {}
if reason == "start":
conditions.update(setup_run_conds(run))
if reason == "update" or reason == "end":
conditions.update(update_beam_conditions(run, log))
# Add all the values that we've determined to the RCDB
for (key, value) in conditions.items():
log.debug(Lf("Adding cnd '{}'='{}'", key, value))
db.add_conditions(run, conditions, True)
log.debug("Committed to DB. End of update_rcdb_conds()")
return conditions
def update_conditions(input_data):
script_start_clock = time.clock()
script_start_time = time.time()
last_run = None
con_str = os.environ["RCDB_CONNECTION"] \
if "RCDB_CONNECTION" in os.environ.keys() \
else "mysql://[email protected]:3306/a-rcdb"
db = RCDBProvider(con_str)
with open('%s' % input_data, 'rb') as f:
for line in f:
#parse data
run_num = (line.split(None,2)[0]).rstrip()
condition_name = (line.split(None,2)[1]).rstrip()
value = (line.split(None,2)[2]).rstrip()
if not run_num.isdigit():
log.warn(Lf("ERROR: wrong format, run should be number! Your input: '{}'", run_num))
continue
# check db entry and update
run = db.get_run(run_num)
if not run:
log.warn(Lf("Run '{}' is not found in DB.. skip this run", run_num))
continue
else:
last_run = run_num
try:
cnd_type = db.get_condition_type(condition_name)
if not TESTMODE:
db.add_condition(run, condition_name, value, True)
db.add_log_record("",
"Update condition:'{}'. time: '{}'"
.format(condition_name,
datetime.now()), run_num)
else:
print run_num, condition_name, value
except Exception as ex:
log.warn("ERROR: wrong condition name. Internal exception:\n" + str(ex))
if not TESTMODE:
now_clock = time.clock()
db.add_log_record("",
"End of update. Script proc clocks='{}', wall time: '{}', datetime: '{}'"
.format(condition_name,
now_clock - script_start_clock,
time.time() - script_start_time,
datetime.now()), last_run)
def parse_components(parse_result, xml_components):
"""
Parses <components> section fo coda log file
:param parse_result: CodaRunLogParseResult that holds all available update context
:type parse_result: CodaRunLogParseResult
:param xml_components: ElementTree <components> section of xml file
:type xml_components: xml.etree.ElementTree
:return: context
"""
if xml_components is not None:
components = {}
component_stats = {}
for xml_component in xml_components.findall('component'):
stats = {}
def find_stat(name, cast):
xml_field = xml_component.find(name)
if xml_field is not None:
stats[name] = cast(xml_field.text)
find_stat("evt-rate", float) # <evt-rate>7.541666</evt-rate>
find_stat("data-rate",
float) # <data-rate>19.369333333333334</data-rate>
find_stat("evt-number", int) # <evt-number>181</evt-number>
find_stat("min-evt-size", float) # <min-evt-size>0</min-evt-size>
find_stat("max-evt-size", float) # <max-evt-size>0</max-evt-size>
find_stat("average-evt-size",
float) # <average-evt-size>0</average-evt-size>
component_type = xml_component.attrib['type']
components[xml_component.attrib['name']] = component_type
component_stats[xml_component.attrib['name']] = stats
if component_type == 'ER':
last_file_xml = xml_component.find('out-file')
if last_file_xml is not None and last_file_xml.text:
last_file = last_file_xml.text
# the last file is something like: hd_rawdata_011410_055.evio
u_pos = last_file.rfind('_')
d_pos = last_file.find('.')
# noinspection PyBroadException
try:
parse_result.evio_files_count = int(
last_file[u_pos + 1:d_pos]) + 1
except:
log.warning(
Lf("Can't parse file index for '{}' file",
last_file))
parse_result.evio_last_file = last_file
parse_result.components = components
parse_result.component_stats = component_stats
return parse_result
def print_conds(db, run_number):
if not db.get_run(run_number):
log.warn(Lf("Run '{}' is not found in DB", run_number))
return
run = db.get_run(run_number)
start_time = None
try:
start_time = run.start_time
except Exception as ex:
log.warn(
Lf("Error in start_time request : run='{}', '{}'", run_number, e))
if start_time is not None:
try:
conditions = epics_helper.mya_get_run_conds(run, log)
print run_number, conditions
except Exception as ex:
log.warn("Getting conditions failed :" + str(ex))
def parse_file(filename):
"""
Opens and parses coda file
:return: context (it is filled after parsing the file)
"""
# read xml file and get root and run-start element
result = CodaRunLogParseResult()
result.coda_log_file = filename
log.debug(Lf("Parsing CODA file '{0}'", filename))
xml_root = Et.parse(filename).getroot()
return parse_xml(result, xml_root)
def parse_start_run_data(config_file, session_file):
"""
Parse coda config and session files
Use current time as temporary start time
Results to be filled from here:
start_time, coda_session, run_config, run_type, run_number, coda_config_file, coda_session_file
Return result
"""
# start time
script_start_time = datetime.now()
result = ParityCodaRunLogParseResult()
try:
temp_start_time = script_start_time.strftime("%Y-%m-%d %H:%M:%S")
result.start_time = temp_start_time
result.has_run_start = True
except Exception as ex:
log.warning("Error with temp run start time " + str(ex))
result.coda_config_file = config_file
result.coda_session_file = session_file
# parse configID.xml
log.debug(Lf("Parsing CODA Config file '{0}'", config_file))
parse_coda_config_file(result, config_file)
# parse controlSessions.xml
log.debug(Lf("Parsing controlSessions file '{0}'", session_file))
parse_coda_session_file(result, session_file)
return result
def parse_end_comment(parse_result, xml_root):
""" Parses end comment
It is assumed that the comment is in form:
<end-comment>
Add run comments here ....
-------------------------------------------
Date : Mon Feb 16 16:53:09 EST 2015
RUN_NUMBER : 002472 (2472)
RUN TYPE : hd_bcal_n.ti
RUN CONFIG : led_upstream_mode8.cnf
RAID DIR :
-------------------------------------------
</end-comment>
:param parse_result: CodaRunLogParseResult that holds all available update context
:type parse_result: CodaRunLogParseResult
:param xml_root: ElementTree parsed coda xml file
:type xml_root: xml.etree.ElementTree
:return: context
"""
# User comments
xml_end_comment = xml_root.find("end-comment")
if xml_end_comment is None:
log.info("Unable to find <end-comment> section")
return parse_result
end_comment = xml_end_comment.text
if "-------------" in end_comment:
user_comment = end_comment[:end_comment.find("-------------")].strip()
# "Add run comments here ...." - means user didn't add anything
if user_comment == "Add run comments here ....":
user_comment = ""
else:
log.warning(
Lf(
"Log comment is in unknown format. "
"No '-----' separated part found. The comment is '{}'",
end_comment))
user_comment = end_comment
parse_result.user_comment = user_comment
return parse_result
def update_db_conds(db, run, reason):
"""
add_conditions(run, name_values, replace=True/False)
:name_values: dictionary or list of name-value pairs
:replace: default is False?
:Defined in provider.py, takes care of incorrect ConditionType
"""
log = logging.getLogger('pvdb.udpate.epics')
log.debug(Lf("Running 'update_rcdb_conds(db={}, run={})'", db, run))
conditions = {}
conditions.update(get_run_conds())
db.add_conditions(run, conditions, True)
log.debug("Commited to DB. End of update_db_conds()")
return conditions
def parse_coda_session_file(parse_result, filename):
"""
Open and parse controlSessions.xml file
Return parse_result
Example structure:
<control>
<session>
<name>par1</name>
<config>ALL_PREX</config>
<runnumber>####</runnumber>
</session>
</control>
"""
xml_root = Et.parse(filename).getroot()
xml_result = xml_root.find("session").text
if xml_result is None:
log.warning("No <session> section found in controlSessions.xml")
return parse_result
# coda session name
parse_result.coda_session = xml_root.find("session").find("name").text
# coda config name, check consistency with configID.xml
config_name = xml_root.find("session").find("config").text
if config_name != parse_result.run_config:
log.warning(
Lf("config name mismatch {},{}", config_name,
parser_result.run_config))
# coda run number
parse_result.run_number = int(
xml_root.find("session").find("runnumber").text)
return parse_result
def mya_get_run_conds(run, log):
conditions = {}
start_time_str = datetime.strftime(run.start_time, "%Y-%m-%d %H:%M:%S")
for epics_name, cond_name in epics_list.iteritems():
end_time_str = datetime.strftime(run.end_time, "%Y-%m-%d %H:%M:%S")
# skip period when APEXPOS was not available
if cond_name == "target_encoder" and start_time_str < "2019-02-18 08:00:00":
continue
if "current" in cond_name:
try:
cmds = [
"myStats", "-b", start_time_str, "-e", end_time_str, "-c",
epics_name, "-r", "1:80", "-l", epics_name
]
cond_out = subprocess.Popen(cmds, stdout=subprocess.PIPE)
n = 0
for line in cond_out.stdout:
n += 1
if n == 1: # skip header
continue
tokens = line.strip().split()
if len(tokens) < 3:
continue
key = tokens[0]
value = tokens[2]
if value == "N/A":
value = 0
if key == epics_name:
conditions[cond_name] = value
except Exception as ex:
log.warn(Lf("Error in beam_current request : '{}'", e))
conditions["beam_current"] = "-999"
else:
try:
cmds = ["myget", "-c", epics_name, "-t", start_time_str]
cond_out = subprocess.Popen(cmds, stdout=subprocess.PIPE)
for line in cond_out.stdout:
value = line.strip().split()[2]
if cond_name == "ihwp":
if value == "1":
conditions[cond_name] = "IN"
else:
conditions[cond_name] = "OUT"
elif cond_name == "helicity_pattern":
if value == "1":
conditions[cond_name] = "Quartet"
elif value == "2":
conditions[cond_name] = "Octet"
else:
conditions[cond_name] = "-999" # undefined
else:
conditions[cond_name] = value
except Exception as e:
log.warn(Lf("Error in epics request : '{}',{}'", cond_name, e))
conditions[cond_name] = "-999"
# Get target type condition
if start_time_str > "2019-02-18 08:00:00":
conditions["target_type"] = get_target_name(
conditions["target_encoder"])
return conditions
"--connection",
help="connection string (eg, mysql://pvdb@localhost/pvdb)")
args = parser.parse_args()
run_number = args.run
# Connection
if args.connection:
con_string = args.connection
elif "RCDB_CONNECTION" in os.environ:
con_string = os.environ["RCDB_CONNECTION"]
else:
print(
"ERROR! RCDB_CONNECTION is not set and is not given as a script parameter (-c)"
)
sys.exit(2)
log.debug(Lf("con_string = '{}'", con_string))
# What to update
update_parts = []
if args.update:
update_parts = args.update.split(',')
log.debug(Lf("update_parts = {}", update_parts))
# Update reason
update_reason = args.reason
log.debug(Lf("update_reason = '{}'", update_reason))
# Open DB connection
db = ConfigurationProvider(con_string)
# Create update context
def update_beam_conditions(run, log):
# Build mapping of conditions to add to the RCDB, key is name of condition
conditions = {}
# Beam current - uses the primary BCM, IBCAD00CRCUR6
# We could also use the following: IPM5C11.VAL,IPM5C11A.VAL,IPM5C11B.VAL,IPM5C11C.VAL
try:
#conditions["beam_current"] = float(caget("IBCAD00CRCUR6")) # pull the value at beam start?
# save integrated beam current over the whole run
# use MYA archive commands to calculate average
# if the run has a set end time, then use that, otherwise use fallback
if run.end_time:
end_time = run.end_time
else:
time_delta = datetime.timedelta(minutes=10)
now = datetime.datetime.now()
end_time = now if now - run.start_time < time_delta else run.start_time + time_delta
log.debug(Lf("No 'run.end_time'. Using '{}' as end_time",
end_time))
# Format begin and end time
begin_time_str = datetime.datetime.strftime(run.start_time,
'%Y-%m-%d %H:%M:%S')
end_time_str = datetime.datetime.strftime(end_time,
'%Y-%m-%d %H:%M:%S')
log.debug(
Lf("Requesting beam_current between '{}' and '{}'",
begin_time_str, end_time_str))
# build myStats command
cmds = [
"myStats", "-b", begin_time_str, "-e", end_time_str, "-l",
"IBCAD00CRCUR6"
]
log.debug(Lf("Requesting beam_current subprocess flags: '{}'", cmds))
# execute external command
p = subprocess.Popen(cmds, stdout=subprocess.PIPE)
# iterate over output
n = 0
for line in p.stdout:
n += 1
if n == 1: # skip header
continue
tokens = line.strip().split()
if len(tokens) < 3:
continue
key = tokens[0]
value = tokens[2] # average value
if key == "IBCAD00CRCUR6":
conditions["beam_current"] = float(value)
except Exception as e:
log.warn(Lf("Error in a beam_current request : '{}'", e))
conditions["beam_current"] = -1.
try:
# also, get the current excluding periods when the beam is off
# we define this as the periods where the BCM reads 5 - 5000 nA
cmds = [
"myStats", "-b", begin_time_str, "-e", end_time_str, "-c",
"IBCAD00CRCUR6", "-r", "5:5000", "-l", "IBCAD00CRCUR6"
]
log.debug(Lf("Requesting beam_current subprocess flags: '{}'", cmds))
# execute external command
p = subprocess.Popen(cmds, stdout=subprocess.PIPE)
# iterate over output
n = 0
for line in p.stdout:
print(line.strip())
n += 1
if n == 1: # skip header
continue
tokens = line.strip().split()
if len(tokens) < 3:
continue
key = tokens[0]
value = tokens[2] # average value
if key == "IBCAD00CRCUR6":
conditions["beam_on_current"] = float(value)
except Exception as e:
log.warn(Lf("Error in a beam_current request : '{}'", e))
conditions["beam_on_current"] = -1.
try:
# also get the beam energy when current > 5 nA and beam energy > 10. MeV, to avoid problems
# where the CEBAF beam energy server fails and doesn't restart =(
avg_beam_energy = 0.
nentries = 0
cmds = [
"myData", "-b", begin_time_str, "-e", end_time_str,
"IBCAD00CRCUR6", "HALLD:p"
]
log.debug(Lf("Requesting beam_energy subprocess flags: '{}'", cmds))
# execute external command
p = subprocess.Popen(cmds, stdout=subprocess.PIPE)
# iterate over output
n = 0
for line in p.stdout:
print(line.strip())
n += 1
if n == 1: # skip header
continue
tokens = line.strip().split()
if len(tokens) < 4:
continue
the_beam_current = float(tokens[2])
the_beam_energy = float(tokens[3])
if (the_beam_current > 5.) and (the_beam_energy > 10.):
avg_beam_energy += the_beam_energy
nentries += 1
# experience has shown that the above myData command returns once or twice every second...
# so let's ignore the time periods and do a simple average
#avg_beam_energy /= float(n)
conditions["beam_energy"] = float("%7.1f" %
(avg_beam_energy / float(nentries)))
"""
cmds = ["myStats", "-b", begin_time_str, "-e", end_time_str, "-c", "IBCAD00CRCUR6", "-r", "30:5000", "-l", "HALLD:p"]
log.debug(Lf("Requesting beam_energy subprocess flags: '{}'", cmds))
# execute external command
p = subprocess.Popen(cmds, stdout=subprocess.PIPE)
# iterate over output
n = 0
for line in p.stdout:
print(line.strip())
n += 1
if n == 1: # skip header
continue
tokens = line.strip().split()
if len(tokens) < 3:
continue
key = tokens[0]
value = tokens[2] # average value
if key == "HALLD:p":
conditions["beam_energy"] = float(value)
"""
except Exception as e:
log.warn(Lf("Error in a beam_energy request : '{}'", e))
conditions["beam_energy"] = -1.
try:
# also, get the average CDC gas pressure
cmds = [
"myStats", "-b", begin_time_str, "-e", end_time_str, "-l",
"RESET:i:GasPanelBarPress1"
]
log.debug(
Lf("Requesting cdc_gas_pressure subprocess flags: '{}'", cmds))
# execute external command
p = subprocess.Popen(cmds, stdout=subprocess.PIPE)
# iterate over output
n = 0
for line in p.stdout:
print(line.strip())
n += 1
if n == 1: # skip header
continue
tokens = line.strip().split()
if len(tokens) < 3:
continue
key = tokens[0]
value = tokens[2] # average value
if key == "RESET:i:GasPanelBarPress1":
conditions["cdc_gas_pressure"] = float(value)
except Exception as e:
log.warn(Lf("Error in a cdc_gas_pressure request : '{}'", e))
conditions["cdc_gas_pressure"] = -1.
return conditions
def update(run_number, update_parts, context):
assert isinstance(context.db, RCDBProvider)
db = context.db
coda_path = None
coda_file_name = None
host = socket.gethostname()
if "adaq" in host:
coda_path = "/adaq2/data1/apar/"
elif "aonl" in host:
coda_path = "/adaq2/data1/apar/"
else:
coda_path = "/cache/halla/parity/raw/"
for files in glob.glob(coda_path + "*" + run_number + "*.dat"):
if not "parity18" in files:
coda_file_name = files
if "coda" in update_parts and coda_file_name is None:
print "CODA file is not found", run_number
log.info(Lf("Coda file is not found, run={}", run_number))
sys.exit()
# Udpate db (coda)
if "coda" in update_parts:
log.debug(Lf("Update run, run={}", run_number))
# Parse coda info (start, end time, ..)
coda_parse_result = parity_coda_parser.parse_coda_data_file(
coda_file_name)
run_start.update_parity_coda_conditions(context, coda_parse_result)
# Update epics
if "epics" in update_parts:
log.debug(Lf("Update epics, run={}", run_number))
if not "ops" in host:
print "You probably don't have myget available from your machine"
sys.exit()
if not db.get_run(run_number):
log.info(Lf("Run '{}' is not found in DB", run_number))
return
run = db.get_run(run_number)
conditions = {}
"""
myget -c[channel name] -t[time]
:if time is not specified, it returns data for last 1min
:Use myStats to get the average current where the BCM reads 1-70 uA
excluding periods when the beam is off
"""
import epics_helper
for epics_name, cond_name in epics_helper.epics_list.iteritems():
try:
start_time_str = datetime.strftime(run.start_time,
"%Y-%m-%d %H:%M:%S")
end_time_str = datetime.strftime(run.end_time,
"%Y-%m-%d %H:%M:%S")
if "current" in cond_name:
cmds = [
"myStats", "-b", start_time_str, "-e", end_time_str,
"-c", epics_name, "-r", "1:70", "-l", epics_name
]
cond_out = subprocess.Popen(cmds, stdout=subprocess.PIPE)
n = 0
for line in cond_out.stdout:
n += 1
if n == 1: # skip header
continue
tokens = line.strip().split()
if len(tokens) < 3:
continue
key = tokens[0]
value = tokens[2]
if value == "N/A":
value = 0
if key == epics_name:
conditions[cond_name] = value
else:
cmds = ["myget", "-c", epics_name, "-t", start_time_str]
cond_out = subprocess.Popen(cmds, stdout=subprocess.PIPE)
for line in cond_out.stdout:
value = line.strip().split()[2]
if cond_name == "ihwp":
if value == "1":
conditions[cond_name] = "IN"
else:
conditions[cond_name] = "OUT"
elif cond_name == "helicity_pattern":
if value == "1":
conditions[cond_name] = "Quartet"
elif value == "2":
conditions[cond_name] = "Octet"
else:
conditions[cond_name] = "-999" # undefined
else:
conditions[cond_name] = value
except Exception as e:
log.warn(Lf("Error in epics request : '{}',{}'", cond_name, e))
# Get target type condition
conditions["target_type"] = epics_helper.get_target_name(
conditions["target_encoder"])
db.add_conditions(run_number, conditions, True)
# >oO DEBUG log message
db.add_log_record("",
"End of update. datetime: '{}'".format(datetime.now()),
run_number)
def parse_start_run_data(parse_result, xml_root):
"""
Parses ElementTree element that should contain whole coda file
:param parse_result: CodaRunLogParseResult that holds all available update context
:type parse_result: CodaRunLogParseResult
:param xml_root: ElementTree parsed coda xml file
:type xml_root: xml.etree.ElementTree
:return: context
"""
assert isinstance(parse_result, CodaRunLogParseResult)
xml_run_start = xml_root.find("run-start")
if xml_run_start is None:
log.warning("No <run-start> section found!")
return parse_result
parse_result.has_run_start = True
# Run number
parse_result.run_number = int(
xml_root.find("run-start").find("run-number").text)
log.info(Lf("Run number '{}'", parse_result.run_number))
# Run type condition
parse_result.run_type = xml_root.attrib["runtype"]
# Session
parse_result.session = xml_root.attrib["session"]
# Start time
try:
start_time = datetime.strptime(
xml_run_start.find("start-time").text, "%m/%d/%y %H:%M:%S")
parse_result.start_time = start_time
log.info(Lf("Run start time is '{}'", start_time))
except Exception as ex:
log.warning("Error parsing <start-time> section: " + str(ex))
# Update time
try:
update_time = datetime.strptime(
xml_run_start.find("update-time").text, "%m/%d/%y %H:%M:%S")
parse_result.update_time = update_time
log.info(Lf("Update time is '{}'", update_time))
except Exception as ex:
log.warning("Error parsing <update-time> section: " + str(ex))
# Event number
xml_event_count = xml_run_start.find('total-evt')
if xml_event_count is not None:
parse_result.event_count = int(xml_event_count.text)
# Components used
xml_components = xml_run_start.find('components')
parse_components(parse_result, xml_components)
# RTVs
xml_rtvs = xml_run_start.find('rtvs')
if xml_rtvs is not None:
rtvs = {
rtv.attrib['name']: rtv.attrib['value']
for rtv in xml_rtvs.findall('rtv')
}
parse_result.rtvs = rtvs
# run_config_file
if RUN_CONFIG_RTV in rtvs.keys():
parse_result.run_config_file = rtvs[RUN_CONFIG_RTV]
parse_result.run_config = os.path.basename(
parse_result.run_config_file)
log.debug(
Lf("Run config file extracted from rtvs '{}'",
parse_result.run_config))
return parse_result
def update():
description = "Script for DB udpate after CODA run end"
parser = argparse.ArgumentParser(description=description, usage=get_usage())
parser.add_argument("--run", type=str, help="Run number")
parser.add_argument("-v", "--verbose", help="increase output verbosity", action="store_true")
parser.add_argument("--update", help="Comma separated, modules to update such as coda,epics", default="")
parser.add_argument("--reason", help="Reason of the udpate: 'start', 'udpate', 'end' or ''", default="")
parser.add_argument("-c", "--connection", help="connection string (eg, mysql://pvdb@localhost/pvdb)")
args = parser.parse_args()
# Connection
if args.connection:
con_string = args.connection
elif "RCDB_CONNECTION" in os.environ:
con_string = os.environ["RCDB_CONNECTION"]
else:
print ("ERROR! RCDB_CONNECTION is not set and is not given as a script parameter (-c)")
parser.print_help()
sys.exit(2)
log.debug(Lf("con_string = '{}'", con_string))
# What to update
update_parts = []
if args.update:
update_parts = args.update.split(',')
log.debug(Lf("update_parts = {}", update_parts))
# Update reason
update_reason = args.reason
log.debug(Lf("update_reason = '{}'", update_reason))
# Open DB connection
db = ConfigurationProvider(con_string)
# Create update context
update_context = rcdb.UpdateContext(db, update_reason)
coda_path = None
coda_file_name = None
host = socket.gethostname()
if "adaq" in host:
coda_path = "/adaq2/data1/apar/"
elif "aonl" in host:
coda_path = "/adaq2/data1/apar/"
else:
coda_path = "/cache/halla/parity/raw/"
for files in glob.glob(coda_path+"*"+str(run)+"*.dat"):
if not "parity18" in files:
coda_file_name = files
if coda_file_name is None:
print "CODA file is not found", run
sys.exit()
coda_parse_result = parity_coda_parser.parse_coda_data_file(coda_file_name)
if "coda" in update_parts:
run_start.update_parity_coda_conditions(update_context, coda_parse_result)
def update_parity_coda_conditions(context, parse_result):
# create run configuration standard logger
log = logging.getLogger('pvdb.update_coda')
# Some assertions in the beginning
assert isinstance(parse_result, ParityCodaRunLogParseResult)
assert isinstance(context, UpdateContext)
assert isinstance(context.db, RCDBProvider)
db = context.db
if parse_result.run_number is None:
log.warn(
"parse_result.run_number is None. (!) Run. Number. Is. None!!!")
return
"""
# enable later
if context.reason == UpdateReasons.END and not db.get_run(parse_result.run_number):
log.info(Lf("Run '{}' is not found in DB. But the update reason is end of run. "
"Considering there where no GO. Only prestart and then Stop ", parse_result.run_number))
return
"""
run = db.create_run(parse_result.run_number)
conditions = []
# Experiment name
if parse_result.experiment_name is not None:
conditions.append(
(ParityConditions.EXPERIMENT, parse_result.experiment_name))
# Arm flag (default set to 0)
conditions.append((ParityConditions.ARM_FLAG, 0))
# Run type condition
if parse_result.run_type is not None:
conditions.append((DefaultConditions.RUN_TYPE, parse_result.run_type))
# Session
if parse_result.coda_session is not None:
conditions.append(
(DefaultConditions.SESSION, parse_result.coda_session))
# config name
if parse_result.run_config is not None:
conditions.append(
(DefaultConditions.RUN_CONFIG, parse_result.run_config))
# slug number
if parse_result.slug is not None:
conditions.append((ParityConditions.SLUG, parse_result.slug))
#These need to be udpated by Run END
# Set the run as not properly finished (We hope that the next section will
if parse_result.has_run_end is not None:
conditions.append(
(DefaultConditions.IS_VALID_RUN_END, parse_result.has_run_end))
# The number of events in the run
if parse_result.event_count is not None:
conditions.append(
(DefaultConditions.EVENT_COUNT, parse_result.event_count))
# Daq comment by user
if parse_result.user_comment is not None:
conditions.append(
(DefaultConditions.USER_COMMENT, parse_result.user_comment))
if parse_result.bmw is not None:
conditions.append((ParityConditions.BMW, parse_result.bmw))
if parse_result.feedback is not None:
conditions.append((ParityConditions.FEEDBACK, parse_result.feedback))
# Run prestart time
if parse_result.prestart_time is not None:
conditions.append(
(ParityConditions.RUN_PRESTART_TIME, parse_result.prestart_time))
# Run length
if parse_result.has_run_end == True and parse_result.has_run_start == True:
total_run_time = datetime.strptime(
parse_result.end_time, "%Y-%m-%d %H:%M:%S") - datetime.strptime(
parse_result.start_time, "%Y-%m-%d %H:%M:%S")
conditions.append(
(DefaultConditions.RUN_LENGTH, total_run_time.seconds))
#start and end time is read from run table, so no need to add these conditions here
# conditions.append((DefaultConditions.RUN_START_TIME, datetime.strptime(parse_result.start_time,"%m/%d/%y %H:%M:%S")))
# conditions.append((DefaultConditions.RUN_END_TIME, datetime.strptime(parse_result.end_time, "%m/%d/%y %H:%M:%S")))
"""
# Conditions not added currently
# Filename of the last evio file written by CODA ER
if parse_result.coda_last_file is not None:
conditions.append(('evio_last_file', parse_result.evio_last_file))
# The number of evio files written by CODA Event Recorder
if parse_result.coda_files_count is not None:
conditions.append(('evio_files_count', parse_result.evio_files_count))
"""
# SAVE CONDITIONS
db.add_conditions(run, conditions, replace=True)
log.info(Lf("update_coda: Saved {} conditions to DB", len(conditions)))
# Start and end times
if parse_result.start_time is not None:
run.start_time = parse_result.start_time # Time of the run start
log.info(Lf("Run start time is {}", parse_result.start_time))
if parse_result.end_time is not None:
run.end_time = parse_result.end_time # Time of the run end
log.info(
Lf("Run end time is {}. Set from end_time record",
parse_result.end_time))
"""
# update_time is not currently included in ParityCodaRunLogParseResult
else:
if parse_result.update_time is not None:
run.end_time = parse_result.update_time # Fallback, set time when the coda log file is written as end time
log.info(Lf("Run end time is {}. Set from update_time record", parse_result.update_time))
"""
db.session.commit() # Save run times
def parse_end_run_info(run_number):
# for the record
script_start_clock = time.clock()
script_start_time = time.time()
# Run end time
end_time_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
# DB Connection
con_str = os.environ["RCDB_CONNECTION"] \
if "RCDB_CONNECTION" in os.environ.keys() \
else "mysql://[email protected]:3306/a-rcdb"
db = rcdb.RCDBProvider(con_str)
# Set data_path
if "QW_DATA" in os.environ:
data_path = os.environ["QW_DATA"]
else:
print ("QW_DATA is not set, force to /adaq1/data1/apar")
data_path = "/adaq1/data1/apar"
# when data path was set to /adaq2/data1/apar
if int(run_number) < 2143:
data_path = "/adaq2/data1/apar"
# dat file counts
num_files = len([files for files in glob.glob(data_path+"/*"+run_number+".dat*")])
run = db.get_run(run_number)
if not db.get_run(run_number):
log.info(Lf("run_end: Run '{}' is not found in DB."
"Considering there was no GO, only prestart and then Stop ", run_number))
# FIXME: add to error list
return
run.end_time = end_time_str
nevts = parity_coda_parser.GetTotalEvents()
# FIXME: total number of events? (maybe add at the end of prompt analysis)
# epics info: consistency check and update
epics_conditions = {}
epics_conditions = epics_helper.get_end_run_conds(run)
# total run length
total_run_time = datetime.strptime(run.end_time, "%Y-%m-%d %H:%M:%S") - run.start_time
# rough estimation of total charge = run length * average beam current
charge = float(total_run_time.total_seconds()) * float(epics_conditions["beam_current"])
epics_conditions["total_charge"] = charge
# Get dpp information
start_time_str = datetime.strftime(run.start_time, "%Y-%m-%d %H:%M:%S")
dpp_mean, dpp_sig = get_dpp_parity.get_dpp(start_time_str, end_time_str)
epics_conditions["dpp"] = dpp_mean;
epics_conditions["dpp_stdev"] = dpp_sig;
p_mean, p_sig = get_dpp_parity.get_p(start_time_str, end_time_str)
epics_conditions["p"] = p_mean;
epics_conditions["p_stdev"] = p_sig;
if nevts is None:
nevts = -1
event_rate = -1
else:
event_rate = float(nevts) / float(total_run_time.total_seconds())
if test_mode:
print("Run Start time:\t %s" % run.start_time)
print("Run End time:\t %s" % run.end_time)
print("Run length:\t %d" % (int(total_run_time.total_seconds())))
print("Total event counts %d" % (int(nevts)))
print("Event Rate %.2f" % (float(event_rate)))
print("Avg.Beam Current:\t %.2f" % (float(epics_conditions["beam_current"])))
print("Total charge:\t %.2f" % charge)
print("Beam energy:\t %.2f" % (float(epics_conditions["beam_energy"])))
#print("Target type:\t %s" % (epics_conditions["target_type"]))
print("Helicity pattern:\t %s" % (epics_conditions["helicity_pattern"]))
print("Helicity frequency:\t %s Hz" % (epics_conditions["helicity_frequency"]))
print("IHWP:\t %s" % (epics_conditions["ihwp"]))
print("Wien angle (H,V):\t %s, %s" % (epics_conditions["horizontal_wien"], epics_conditions["vertical_wien"]))
print("DPP:\t %s" % (epics_conditions["dpp"]))
print("DPP sigma:\t %s" % (epics_conditions["dpp_stdev"]))
print("P:\t %s" % (epics_conditions["p"]))
print("P sigma:\t %s" % (epics_conditions["p_stdev"]))
else:
# Add conditions to DB
conditions = []
conditions.append((DefaultConditions.IS_VALID_RUN_END, True))
conditions.append((DefaultConditions.RUN_LENGTH, total_run_time.total_seconds()))
conditions.append((ParityConditions.BEAM_CURRENT, epics_conditions["beam_current"]))
conditions.append((ParityConditions.TOTAL_CHARGE, epics_conditions["total_charge"]))
conditions.append((DefaultConditions.EVENT_COUNT, nevts))
conditions.append((DefaultConditions.EVENT_RATE, event_rate))
if dpp_mean is not None:
conditions.append((ParityConditions.DPP, epics_conditions["dpp"]))
if dpp_sig is not None:
conditions.append((ParityConditions.DPP_STDEV, epics_conditions["dpp_stdev"]))
if p_mean is not None:
conditions.append((ParityConditions.ENERGY_AVG, epics_conditions["p"]))
if p_sig is not None:
conditions.append((ParityConditions.ENERGY_SIG, epics_conditions["p_stdev"]))
# Disabled (conditions not added to DB)
# conditions.append(('evio_last_file', files))
# conditions.append(('evio_file_count', num_files))
# Save conditions
db.add_conditions(run, conditions, replace=True)
db.session.commit()
now_clock = time.clock()
db.add_log_record("",
"End of udpate. Script proc clocks='{}', wall time: '{}', datetime: '{}'"
.format(now_clock - script_start_clock,
time.time() - script_start_time,
datetime.now()), run_number)
def parse_start_run_info():
script_start_clock = time.clock()
script_start_time = time.time()
log = logging.getLogger('pvdb') # create run configuration standard logger
log.addHandler(logging.StreamHandler(
sys.stdout)) # add console output for logger
log.setLevel(
logging.INFO
) # DEBUG: print everything. Changed to logging. INFO for less output
parser = argparse.ArgumentParser(description="Update PVDB")
# parser.add_argument("config_xml_file", type=str, help="full path to configID.xml file")
parser.add_argument("session_xml_file",
type=str,
help="full path to controlSessions.xml file")
parser.add_argument("-v",
"--verbose",
help="increase output verbosity",
action="store_true")
parser.add_argument(
"--update",
help="Comma separated, modules to update such as coda,epics",
default="coda,epics")
parser.add_argument(
"--reason",
help="Reason of the udpate: 'start', 'udpate', 'end' or ''",
default="start")
parser.add_argument("--exp", help="Experiment name", default="CREX")
parser.add_argument(
"-c",
"--connection",
help="connection string (eg, mysql://pvdb@localhost/pvdb)")
args = parser.parse_args()
# Set log level
log.setLevel(logging.DEBUG if args.verbose else logging.INFO)
# coda xml files
# config_xml_file = args.config_xml_file
# log.debug(Lf("config_xml_file = '{}'", config_xml_file))
session_xml_file = args.session_xml_file
log.debug(Lf("session_xml_file = '{}' ", session_xml_file))
# Connection
if args.connection:
con_string = args.connection
elif "RCDB_CONNECTION" in os.environ:
con_string = os.environ["RCDB_CONNECTION"]
else:
print(
"ERROR! RCDB_CONNECTION is not set and is not given as a script parameter (-c)"
)
parser.print_help()
sys.exit(2)
log.debug(Lf("con_string = '{}'", con_string))
# What to update
update_parts = []
if args.update:
update_parts = args.update.split(',')
log.debug(Lf("update_parts = {}", update_parts))
# Update reason
update_reason = args.reason
log.debug(Lf("update_reason = '{}'", update_reason))
# Open DB connection
db = ConfigurationProvider(con_string)
# Create update context
update_context = rcdb.UpdateContext(db, update_reason)
# Parse coda files and save to DB
log.debug(Lf("Parsing coda__xml_files='{}'", session_xml_file))
coda_parse_result = parity_coda_parser.parse_start_run_data(
session_xml_file)
run_number = coda_parse_result.run_number
# >oO DEBUG log message
now_clock = time.clock()
db.add_log_record(
"", "Start Run Script: Parsed xml_files='{}'. run='{}'".format(
session_xml_file, run_number), run_number)
# Parse runstart.info (Run type, comment)
dict_info = parity_coda_parser.runinfo_parser()
if bool(dict_info["Run"]["type"]):
coda_parse_result.run_type = dict_info["Run"]["type"]
if bool(dict_info["comment"]["text"]):
coda_parse_result.user_comment = dict_info["comment"]["text"]
if bool(dict_info["parity"]["bmw"]):
coda_parse_result.bmw = dict_info["parity"]["bmw"]
if bool(dict_info["parity"]["feedback"]):
coda_parse_result.feedback = dict_info["parity"]["feedback"]
# Set experiment name
coda_parse_result.experiment_name = args.exp
# Coda conditions to DB
if "coda" in update_parts:
log.debug(Lf("Adding coda conditions to DB", ))
if test_mode:
print "Experiment name:\t", coda_parse_result.experiment_name
print "Run number:\t", coda_parse_result.run_number
print "Start time:\t", coda_parse_result.start_time
print "Run config:\t", coda_parse_result.run_config
print "Run type:\t", coda_parse_result.run_type
print "Comment:\t", coda_parse_result.user_comment
print "BMW:\t", coda_parse_result.bmw
print "Feedback:\t", coda_parse_result.feedback
else:
update_parity_coda_conditions(update_context, coda_parse_result)
else:
log.debug(
Lf(
"Skipping to add coda conditions to DB. Use --update=...,coda to update it",
))
"""
# Do we want to save files to DB?
log.debug(Lf("Adding coda_xml_log_file to DB", ))
db.add_configuration_file(run_number, coda_xml_log_file, overwrite=True, importance=ConfigurationFile.IMPORTANCE_HIGH)
"""
# EPICS Update
# Get EPICS variables
epics_start_clock = time.clock()
if test_mode:
conditions = epics_helper.get_run_conds()
print conditions
else:
if 'epics' in update_parts and run_number:
# noinspection PyBroadException
try:
conditions = epics_helper.update_db_conds(
db, run_number, update_reason)
epics_end_clock = time.clock()
# >oO DEBUG log message
if "beam_current" in conditions:
db.add_log_record(
"",
"Update epics. beam_current:'{}', time: '{}'".format(
conditions["beam_current"],
datetime.now()), run_number)
except Exception as ex:
log.warn(
"update_epics.py failure. Impossible to run the script. Internal exception is:\n"
+ str(ex))
epics_end_clock = time.clock()
# >oO DEBUG log message
db.add_log_record(
"",
"ERROR update epics. Error type: '{}' message: '{}' trace: '{}' "
"||epics_clocks:'{}' clocks:'{}' time: '{}'".format(
type(ex), ex.message, traceback.format_exc(),
epics_end_clock - epics_start_clock,
epics_end_clock - script_start_clock, datetime.now()),
run_number)
log.debug("End of update")
# >oO DEBUG log message
now_clock = time.clock()
if not test_mode:
db.add_log_record(
"",
"End of update. Script proc clocks='{}', wall time: '{}', datetime: '{}'"
.format(now_clock - script_start_clock,
time.time() - script_start_time,
datetime.now()), run_number)
def update_coda_conditions(context, parse_result):
"""
Opens and parses coda file
:return: None
"""
# read xml file and get root and run-start element
# Some assertions in the beginning
assert isinstance(parse_result, CodaRunLogParseResult)
assert isinstance(context, UpdateContext)
assert isinstance(context.db, RCDBProvider)
db = context.db
# Run! Run Lu.. I mean, run number is the major thing, starting with it
if parse_result.run_number is None:
log.warn(
"parse_result.run_number is None. (!) Run. Number. Is. None!!!")
return
if context.reason == UpdateReasons.END and not db.get_run(
parse_result.run_number):
log.info(
Lf(
"Run '{}' is not found in DB. But the update reason is end of run. "
"Considering there where no GO. Only prestart and then Stop ",
parse_result.run_number))
return
run = db.create_run(parse_result.run_number)
conditions = []
# Run type condition
if parse_result.run_type is not None:
conditions.append((DefaultConditions.RUN_TYPE, parse_result.run_type))
# Session (like hdops)
if parse_result.session is not None:
conditions.append((DefaultConditions.SESSION, parse_result.session))
# Set the run as not properly finished (We hope that the next section will
if parse_result.has_run_end is not None:
conditions.append(
(DefaultConditions.IS_VALID_RUN_END, parse_result.has_run_end))
# The number of events in the run
if parse_result.event_count is not None:
conditions.append(
(DefaultConditions.EVENT_COUNT, parse_result.event_count))
# a list of names of <components> section . E.g. ['ROCBCAL13', 'ROCFDC11', ...]
if parse_result.components is not None:
conditions.append((DefaultConditions.COMPONENTS,
json.dumps(parse_result.components)))
# dictionary with contents of the <components> section
if parse_result.component_stats is not None:
conditions.append((DefaultConditions.COMPONENT_STATS,
json.dumps(parse_result.component_stats)))
# RTVs
if parse_result.rtvs is not None:
conditions.append(
(DefaultConditions.RTVS, json.dumps(parse_result.rtvs)))
# Daq comment by user
if parse_result.user_comment is not None:
conditions.append(
(DefaultConditions.USER_COMMENT, parse_result.user_comment))
# config file name. E.g. TRG_COSMIC_BCAL_raw_cdc_b1
if parse_result.run_config is not None:
conditions.append(
(DefaultConditions.RUN_CONFIG, parse_result.run_config))
# Filename of the last evio file written by CODA ER
if parse_result.evio_last_file is not None:
conditions.append(('evio_last_file', parse_result.evio_last_file))
# The number of evio files written by CODA Event Recorder
if parse_result.evio_files_count is not None:
conditions.append(('evio_files_count', parse_result.evio_files_count))
# SAVE CONDITIONS
db.add_conditions(run, conditions, replace=True)
log.info(Lf("update_coda: Saved {} conditions to DB", len(conditions)))
# Start and end times
if parse_result.start_time is not None:
run.start_time = parse_result.start_time # Time of the run start
log.info(Lf("Run start time is {}", parse_result.start_time))
if parse_result.end_time is not None:
run.end_time = parse_result.end_time # Time of the run end
log.info(
Lf("Run end time is {}. Set from end_time record",
parse_result.end_time))
else:
if parse_result.update_time is not None:
run.end_time = parse_result.update_time # Fallback, set time when the coda log file is written as end time
log.info(
Lf("Run end time is {}. Set from update_time record",
parse_result.update_time))
db.session.commit() # Save run times