def update_conditions(input_data):
script_start_clock = time.clock()
script_start_time = time.time()
last_run = None
con_str = os.environ["RCDB_CONNECTION"] \
if "RCDB_CONNECTION" in os.environ.keys() \
else "mysql://[email protected]:3306/a-rcdb"
db = RCDBProvider(con_str)
with open('%s' % input_data, 'rb') as f:
for line in f:
#parse data
run_num = (line.split(None,2)[0]).rstrip()
condition_name = (line.split(None,2)[1]).rstrip()
value = (line.split(None,2)[2]).rstrip()
if not run_num.isdigit():
log.warn(Lf("ERROR: wrong format, run should be number! Your input: '{}'", run_num))
continue
# check db entry and update
run = db.get_run(run_num)
if not run:
log.warn(Lf("Run '{}' is not found in DB.. skip this run", run_num))
continue
else:
last_run = run_num
try:
cnd_type = db.get_condition_type(condition_name)
if not TESTMODE:
db.add_condition(run, condition_name, value, True)
db.add_log_record("",
"Update condition:'{}'. time: '{}'"
.format(condition_name,
datetime.now()), run_num)
else:
print run_num, condition_name, value
except Exception as ex:
log.warn("ERROR: wrong condition name. Internal exception:\n" + str(ex))
if not TESTMODE:
now_clock = time.clock()
db.add_log_record("",
"End of update. Script proc clocks='{}', wall time: '{}', datetime: '{}'"
.format(condition_name,
now_clock - script_start_clock,
time.time() - script_start_time,
datetime.now()), last_run)
def update_all():
# DB Connection
con_str = os.environ["RCDB_CONNECTION"] \
if "RCDB_CONNECTION" in os.environ.keys() \
else "mysql://[email protected]:3306/a-rcdb"
db = RCDBProvider(con_str)
run = db.get_run(run_number)
if not run:
print ("Run %s is not found in DB" % run_number)
sys.exit(1)
conditions = {}
conditions = db_fix_helper.get_run_end_info_from_data(run_number)
def update_slug():
# get run list to update
runs=sys.argv[1]
if "-" in runs:
brun = runs.split("-")[0]
erun = runs.split("-")[1]
else:
brun = runs
erun = runs
#slug number
slug=sys.argv[2]
print "Update SLUGNUMBER for run ", runs, " to SLUG = ", slug
"""
Ask user to enter Y or N (case-insensitive).
:return: True if the answer is Y.
:rtype: bool
"""
answer = ""
while answer not in ["y", "n"]:
answer = raw_input("OK to continue [Y/N]? ").lower()
print answer
if answer != "y":
return
if TESTMODE:
for runno in range(int(brun), int(erun)+1):
print runno, find_slug_number(runno)
else:
# DB Connection
con_str = os.environ["RCDB_CONNECTION"] \
if "RCDB_CONNECTION" in os.environ.keys() \
else "mysql://[email protected]:3306/a-rcdb"
db = RCDBProvider(con_str)
for run_number in range(int(brun), int(erun)+1):
run = db.get_run(run_number)
if not run:
print ("Run %s is not found in DB.. skip this run" % run_number)
continue
# slug = find_slug_number(run_number)
db.add_condition(run, ParityConditions.SLUG, slug, True)
def benchmark_conditions_add(cycles, auto_commit=True):
# Create RCDBProvider object that connects to DB and provide most of the functions
db = RCDBProvider("sqlite:///example.db")
# Create condition type
runs = [db.create_run(1), db.create_run(2)]
ct = db.create_condition_type("my_val", ConditionType.INT_FIELD,
"This is my value")
print("Start {} cycles in benchmark_conditions_add".format(cycles))
# cycle
for i in range(0, cycles):
print_cycle(i, cycles, "benchmark_conditions_add")
db.add_condition(random.choice(runs),
ct,
random.randrange(0, 1000),
replace=True,
auto_commit=auto_commit)
# commit in the end if needed
if not auto_commit:
db.session.commit()
print("Done benchmark_conditions_add")
def update_avg_current():
# get run list to update
runs = sys.argv[1]
if "-" in runs:
brun = runs.split("-")[0]
erun = runs.split("-")[1]
else:
brun = runs
erun = runs
print("Update beam_current for run {}".format(runs))
if TESTMODE:
for runno in range(int(brun), int(erun) + 1):
print("{} {}".format(runno, find_avg_current(runno)))
else:
# DB Connection
con_str = os.environ["RCDB_CONNECTION"] \
if "RCDB_CONNECTION" in os.environ.keys() \
else "mysql://[email protected]:3306/a-rcdb"
db = RCDBProvider(con_str)
for run_number in range(int(brun), int(erun) + 1):
run = db.get_run(run_number)
tmp = find_avg_current(run_number)
if is_number(tmp):
newCur = float(tmp)
else:
newCur = 0.0
if not run:
print("Run %s is not found in DB.. skip this run {}".format(
run_number))
continue
db.add_condition(run, ParityConditions.BEAM_CURRENT, newCur, True)
def update_arm_flag():
# get run list to update
runs = sys.argv[1]
if "-" in runs:
brun = runs.split("-")[0]
erun = runs.split("-")[1]
else:
brun = runs
erun = runs
#arm_flag number
arm_flag = sys.argv[2]
print "Update ARM_FLAG for run ", runs
if TESTMODE:
print "Would change arm flag to be ", arm_flag, " for runs ", brun, "-", erun
for runno in range(int(brun), int(erun) + 1):
print runno, arm_flag
else:
# DB Connection
con_str = os.environ["RCDB_CONNECTION"] \
if "RCDB_CONNECTION" in os.environ.keys() \
else "mysql://[email protected]:3306/a-rcdb"
db = RCDBProvider(con_str)
for run_number in range(int(brun), int(erun) + 1):
run = db.get_run(run_number)
if not run:
print("Run %s is not found in DB.. skip this run" % run_number)
continue
# arm_flag = find_arm_flag_number(run_number)
db.add_condition(run, ParityConditions.ARM_FLAG, arm_flag, True)
def benchmark_conditions_add(cycles, auto_commit=True):
# Create RCDBProvider object that connects to DB and provide most of the functions
db = RCDBProvider("sqlite:///example.db")
# Create condition type
runs = [db.create_run(1), db.create_run(2)]
ct = db.create_condition_type("my_val", ConditionType.INT_FIELD, "This is my value")
print ("Start {} cycles in benchmark_conditions_add".format(cycles))
# cycle
for i in range(0, cycles):
print_cycle(i, cycles, "benchmark_conditions_add")
db.add_condition(random.choice(runs), ct, random.randrange(0, 1000),
replace=True, auto_commit=auto_commit)
# commit in the end if needed
if not auto_commit:
db.session.commit()
print ("Done benchmark_conditions_add")
import sys
from rcdb import RCDBProvider
from rcdb.model import run_periods
from sqlalchemy import text, bindparam, Integer
if __name__ == "__main__":
# Get connection string from arguments
parser = argparse.ArgumentParser(description="This example shows select runs and put them by dates")
parser.add_argument("connection_string",
nargs='?',
help="RCDB connection string mysql://rcdb@localhost/rcdb",
default="mysql://[email protected]/rcdb")
args = parser.parse_args()
# Open DB connection
db = RCDBProvider(args.connection_string)
# Get runs from run period
run_min, run_max, description = run_periods["2016-02"]
print("Selecting runs {}-{} from run period: '{}'".format(run_min, run_max, description))
# print resulting array
# sql = text('SELECT * FROM runs WHERE number > :run')
# sql.bindparams(run_number=30000)
# result = db.session.connection().execute(sql, run=31000)
# for row in result:
# print row
result = db.select_values(['event_count', 'daq_run', 'beam_energy', 'beam_current'], "@is_production", 30000)
print (result.performance)
if __name__ == "__main__":
print sys.argv
# Get connection string from arguments
parser = argparse.ArgumentParser(
description="This example shows select runs and put them by dates")
parser.add_argument(
"connection_string",
nargs='?',
help="RCDB connection string mysql://rcdb@localhost/rcdb",
default="mysql://[email protected]/rcdb")
args = parser.parse_args()
# Open DB connection
db = RCDBProvider(args.connection_string)
# Get runs from run period
run_min, run_max, description = db.get_run_period("2016-02")
print("Selecting runs {}-{} from run period: '{}'".format(
run_min, run_max, description))
data_by_date, performance = get_runs_by_date(
db, "@status_approved and @is_production", run_min, run_max)
# Here we can see exact performance budget. All in seconds
print("Performance([sec]) is: ", performance)
# print resulting array
for run_date in sorted(data_by_date.keys()):
run_values = data_by_date[run_date]
def update_run(run_number):
# DB Connection
con_str = os.environ["RCDB_CONNECTION"] \
if "RCDB_CONNECTION" in os.environ.keys() \
else "mysql://[email protected]:3306/a-rcdb"
db = RCDBProvider(con_str)
run = db.get_run(run_number)
if not run:
print("Run %s is not found in DB" % run_number)
run = db.create_run(run_number)
conditions = {}
conditions = db_fix_helper.get_run_end_info_from_data(run_number)
#Replace with the start from the data file.. there are some cases where we have total time < 0
#because the run was too short... there is 30sec delay in start script
#delay was introduced to give enough time to the end run log before a new run log appears
#run_length = (datetime.strptime(conditions["run_end_time"], "%Y-%m-%d %H:%M:%S") - run.start_time).total_seconds()
run_length = (
datetime.strptime(conditions["run_end_time"], "%Y-%m-%d %H:%M:%S") -
datetime.strptime(conditions["run_start_time"],
"%Y-%m-%d %H:%M:%S")).total_seconds()
event_rate = None
if float(run_length) > 0 and conditions["event_count"] is not None:
event_rate = float(conditions["event_count"]) / float(run_length)
else:
if not float(run_length) > 0:
print("ERROR: run_length < 0....skip the update")
print "Reported start time:", conditions["run_start_time"]
print "Reported end time:", conditions["run_end_time"]
print
if conditions["event_count"] is None:
print("ERROR: event_count not available")
if TESTMODE:
print("Run start time:\t %s" % conditions["run_start_time"])
print("Run end time:\t %s" % conditions["run_end_time"])
print("Total Events:\t %s" % conditions["event_count"])
print("Run length :\t %s" % run_length)
print("Event rate :\t %s" % event_rate)
else:
print "Update coda run info", run.number
run.start_time = conditions["run_start_time"]
run.end_time = conditions["run_end_time"]
db.add_condition(run, DefaultConditions.RUN_LENGTH, run_length, True)
db.add_condition(run, DefaultConditions.IS_VALID_RUN_END,
conditions["has_run_end"], True)
if conditions["event_count"] is not None:
db.add_condition(run, DefaultConditions.EVENT_COUNT,
conditions["event_count"], True)
if event_rate is not None:
db.add_condition(run, DefaultConditions.EVENT_RATE, event_rate,
True)
db.session.commit()
"""
import argparse
import sys
from rcdb import RCDBProvider
if __name__ == "__main__":
print sys.argv
# Get connection string from arguments
parser = argparse.ArgumentParser(
description=
"This example shows basics of how to select runs using search queries")
parser.add_argument(
"connection_string",
help="RCDB connection string mysql://rcdb@localhost/rcdb")
args = parser.parse_args()
# Open DB connection
db = RCDBProvider(args.connection_string)
# Select production runs with event_count > 0.5M
result = db.select_runs("@is_production and event_count > 500000", 10000,
20000)
# Iterate through results
for run in result:
print run.number, run.get_condition_value("event_count")
# Another way of getting values
rows = result.get_values(
["event_count", "polarization_direction", "beam_current"])
from rcdb.stopwatch import StopWatchTimer
from sqlalchemy import desc
from sqlalchemy.orm import aliased
if __name__ == "__main__":
print sys.argv
# Get connection string from arguments
parser = argparse.ArgumentParser(description="This example shows basics of how to select runs using search queries")
parser.add_argument("connection_string", help="RCDB connection string mysql://rcdb@localhost/rcdb")
parser.add_argument('target_cnd_names', nargs='*')
args = parser.parse_args()
print "Conditions:", args.target_cnd_names
# Open DB connection
db = RCDBProvider(args.connection_string)
sw = StopWatchTimer()
sw.start()
# Select production runs with event_count > 0.5M
result = db.select_runs("event_count > 100", 0, 20000)
#result.runs = db.get_runs(0, 20000)
sw.stop()
print "Selection took ", sw.elapsed
print("Selected {} runs".format(len(result.runs)))
def update_run(run_number):
# DB Connection
con_str = os.environ["RCDB_CONNECTION"] \
if "RCDB_CONNECTION" in os.environ.keys() \
else "mysql://[email protected]:3306/a-rcdb"
db = RCDBProvider(con_str)
run = db.get_run(run_number)
if not run:
print("Run %s is not found in DB" % run_number)
sys.exit(1)
conditions = {}
conditions = db_fix_helper.get_run_end_info_from_data(run_number)
run_length = (
datetime.strptime(conditions["run_end_time"], "%Y-%m-%d %H:%M:%S") -
run.start_time).total_seconds()
event_rate = None
if float(run_length) > 0 and conditions["event_count"] is not None:
event_rate = float(conditions["event_count"]) / float(run_length)
else:
if not float(run_length) > 0:
print("ERROR: run_length < 0....skip the update\n")
if conditions["event_count"] is None:
print("ERROR: event_count not available")
if TESTMODE:
print("Run end time:\t %s" % conditions["run_end_time"])
print("Total Events:\t %s" % conditions["event_count"])
print("Run length :\t %s" % run_length)
print("Event rate :\t %s" % event_rate)
else:
run.end_time = conditions["run_end_time"]
db.add_condition(run, DefaultConditions.RUN_LENGTH, run_length, True)
db.add_condition(run, DefaultConditions.IS_VALID_RUN_END,
conditions["has_run_end"], True)
if conditions["event_count"] is not None:
db.add_condition(run, DefaultConditions.EVENT_COUNT,
conditions["event_count"], True)
if event_rate is not None:
db.add_condition(run, DefaultConditions.EVENT_RATE, event_rate,
True)
db.session.commit()
print "Committing to DB"
db.session.commit()
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Check RCDB data for errors and fixing it")
parser.add_argument(
"-e",
"--execute",
action="store_true",
help=
"Execute repairs. Fithout the flag script only shows, what is to be done",
default=False)
parser.add_argument("connection", help="RCDB connection string")
parser.add_argument("-a",
"--action",
choices=['dbl', 'mpl', 'no_daq_run'],
default='dbl')
args = parser.parse_args()
print(args)
# Create RCDBProvider object that connects to DB and provide most of the functions
db = RCDBProvider(args.connection)
if args.action == 'dbl':
fix_double_runs(db, args.execute)
elif args.action == 'no_daq_run':
fix_no_daq_run(db, args.execute)
if __name__ == "__main__":
print sys.argv
# Get connection string from arguments
parser = argparse.ArgumentParser(
description=
"This example shows basics of how to select runs using search queries")
parser.add_argument(
"connection_string",
help="RCDB connection string mysql://rcdb@localhost/rcdb",
nargs='?',
default='mysql://[email protected]/rcdb')
args = parser.parse_args()
# Open DB connection
db = RCDBProvider(args.connection_string)
# Select production runs with event_count > 0.5M
result = db.select_values(
["event_count", "polarization_direction", "beam_current"],
"@is_production and event_count > 500000", 10000, 20000)
# print title
print("{:>7} {:>15} {:>15} {:>15}".format('run', 'polarization_direction',
'beam_current', 'event_count'))
# Iterate through results
for row in result:
run, event_count, polarization_direction, beam_current = tuple(row)
print("{:>7} {:>15} {:>15} {:>15}".format(run, polarization_direction,
beam_current, event_count))
This example requires some live databse
Usage:
python example_simple_queries.py <connectionstring>
"""
import argparse
import sys
from rcdb import RCDBProvider
if __name__ == "__main__":
print sys.argv
# Get connection string from arguments
parser = argparse.ArgumentParser(description="This example shows basics of how to select runs using search queries")
parser.add_argument("connection_string", help="RCDB connection string mysql://rcdb@localhost/rcdb")
args = parser.parse_args()
# Open DB connection
db = RCDBProvider(args.connection_string)
# Select production runs with event_count > 0.5M
result = db.select_runs("@is_production and event_count > 500000", 10000, 20000)
# Iterate through results
for run in result:
print run.number, run.get_condition_value("event_count")
# Another way of getting values
rows = result.get_values(["event_count", "polarization_direction", "beam_current"])
"""
import argparse
import sys
from rcdb import RCDBProvider
from rcdb.model import ConditionType, Run
from rcdb import DefaultConditions
if __name__ == "__main__":
print sys.argv
# Get connection string from arguments
parser = argparse.ArgumentParser(description="This example shows select runs and put them by dates")
parser.add_argument("connection_string", nargs='?', default="mysql://[email protected]/rcdb")
args = parser.parse_args()
# Open DB connection
db = RCDBProvider(args.connection_string)
# add two conditions type. If they are already exist this function just does nothing
db.create_condition_type(DefaultConditions.RUN_START_TIME, ConditionType.TIME_FIELD, "Run start time by DAQ")
db.create_condition_type(DefaultConditions.RUN_END_TIME, ConditionType.TIME_FIELD, "Run end time by DAQ")
# get all runs
runs = db.get_runs(0, sys.maxint)
for run in runs:
print (run.number)
if run.start_time:
db.add_condition(run, DefaultConditions.RUN_START_TIME, run.start_time)
if run.end_time:
db.add_condition(run, DefaultConditions.RUN_END_TIME, run.end_time)
def __init__(self, home, connection_str):
self.home = home
self.db = RCDBProvider(connection_str)
self.config = {}
self.verbose = False
self.connection_str = connection_str
