Пример #1
0
def update_conditions(input_data):

    script_start_clock = time.clock()
    script_start_time = time.time()
    last_run = None    

    con_str = os.environ["RCDB_CONNECTION"] \
        if "RCDB_CONNECTION" in os.environ.keys() \
        else "mysql://[email protected]:3306/a-rcdb"

    db = RCDBProvider(con_str)

    with open('%s' % input_data, 'rb') as f:
        for line in f:
            #parse data
           run_num = (line.split(None,2)[0]).rstrip()
           condition_name = (line.split(None,2)[1]).rstrip()
           value = (line.split(None,2)[2]).rstrip()

           if not run_num.isdigit():
               log.warn(Lf("ERROR: wrong format, run should be number! Your input: '{}'", run_num))
               continue

           # check db entry and update
           run = db.get_run(run_num)
           if not run:
               log.warn(Lf("Run '{}' is not found in DB.. skip this run", run_num))
               continue
           else:
               last_run = run_num
               try:
                   cnd_type = db.get_condition_type(condition_name)
                   if not TESTMODE:
                       db.add_condition(run, condition_name, value, True)
                       db.add_log_record("",
                                         "Update condition:'{}'. time: '{}'"
                                         .format(condition_name, 
                                                 datetime.now()), run_num)
                   else:
                       print run_num, condition_name, value
               except Exception as ex:
                   log.warn("ERROR: wrong condition name. Internal exception:\n" + str(ex))

    if not TESTMODE:
        now_clock = time.clock()
        db.add_log_record("",
                          "End of update. Script proc clocks='{}', wall time: '{}', datetime: '{}'"
                          .format(condition_name, 
                                  now_clock - script_start_clock,
                                  time.time() - script_start_time,
                                  datetime.now()), last_run)
Пример #2
0
def update_all():
    # DB Connection
    con_str = os.environ["RCDB_CONNECTION"] \
              if "RCDB_CONNECTION" in os.environ.keys() \
        else "mysql://[email protected]:3306/a-rcdb"

    db = RCDBProvider(con_str)

    run = db.get_run(run_number)
    if not run:
        print ("Run %s is not found in DB" % run_number)
        sys.exit(1)

    conditions = {}
    conditions = db_fix_helper.get_run_end_info_from_data(run_number)
Пример #3
0
def update_slug():

    # get run list to update
    runs=sys.argv[1]
    if "-" in runs:
        brun = runs.split("-")[0]
        erun = runs.split("-")[1]
    else:
        brun = runs
        erun = runs

    #slug number
    slug=sys.argv[2]

    print "Update SLUGNUMBER for run ", runs, " to SLUG = ", slug
    """
    Ask user to enter Y or N (case-insensitive).
    :return: True if the answer is Y.
    :rtype: bool
    """
    answer = ""
    while answer not in ["y", "n"]:
        answer = raw_input("OK to continue [Y/N]? ").lower()
    print answer
    if answer != "y":
        return

    if TESTMODE:
        for runno in range(int(brun), int(erun)+1):
            print runno, find_slug_number(runno)
    else:
        # DB Connection
        con_str = os.environ["RCDB_CONNECTION"] \
            if "RCDB_CONNECTION" in os.environ.keys() \
            else "mysql://[email protected]:3306/a-rcdb"

        db = RCDBProvider(con_str)

        for run_number in range(int(brun), int(erun)+1):
            run = db.get_run(run_number)
            if not run:
                print ("Run %s is not found in DB.. skip this run" % run_number)
                continue
#            slug = find_slug_number(run_number)
            db.add_condition(run, ParityConditions.SLUG, slug, True)
Пример #4
0
def benchmark_conditions_add(cycles, auto_commit=True):
    # Create RCDBProvider object that connects to DB and provide most of the functions
    db = RCDBProvider("sqlite:///example.db")

    # Create condition type
    runs = [db.create_run(1), db.create_run(2)]
    ct = db.create_condition_type("my_val", ConditionType.INT_FIELD,
                                  "This is my value")

    print("Start {} cycles in benchmark_conditions_add".format(cycles))

    # cycle
    for i in range(0, cycles):
        print_cycle(i, cycles, "benchmark_conditions_add")
        db.add_condition(random.choice(runs),
                         ct,
                         random.randrange(0, 1000),
                         replace=True,
                         auto_commit=auto_commit)

    # commit in the end if needed
    if not auto_commit:
        db.session.commit()

    print("Done benchmark_conditions_add")
Пример #5
0
def update_avg_current():

    # get run list to update
    runs = sys.argv[1]
    if "-" in runs:
        brun = runs.split("-")[0]
        erun = runs.split("-")[1]
    else:
        brun = runs
        erun = runs

    print("Update beam_current for run {}".format(runs))

    if TESTMODE:
        for runno in range(int(brun), int(erun) + 1):
            print("{} {}".format(runno, find_avg_current(runno)))
    else:
        # DB Connection
        con_str = os.environ["RCDB_CONNECTION"] \
            if "RCDB_CONNECTION" in os.environ.keys() \
            else "mysql://[email protected]:3306/a-rcdb"

        db = RCDBProvider(con_str)

        for run_number in range(int(brun), int(erun) + 1):
            run = db.get_run(run_number)
            tmp = find_avg_current(run_number)
            if is_number(tmp):
                newCur = float(tmp)
            else:
                newCur = 0.0
            if not run:
                print("Run %s is not found in DB.. skip this run {}".format(
                    run_number))
                continue
            db.add_condition(run, ParityConditions.BEAM_CURRENT, newCur, True)
Пример #6
0
def update_arm_flag():

    # get run list to update
    runs = sys.argv[1]
    if "-" in runs:
        brun = runs.split("-")[0]
        erun = runs.split("-")[1]
    else:
        brun = runs
        erun = runs

    #arm_flag number
    arm_flag = sys.argv[2]
    print "Update ARM_FLAG for run ", runs

    if TESTMODE:
        print "Would change arm flag to be ", arm_flag, " for runs ", brun, "-", erun
        for runno in range(int(brun), int(erun) + 1):
            print runno, arm_flag
    else:
        # DB Connection
        con_str = os.environ["RCDB_CONNECTION"] \
            if "RCDB_CONNECTION" in os.environ.keys() \
            else "mysql://[email protected]:3306/a-rcdb"

        db = RCDBProvider(con_str)

        for run_number in range(int(brun), int(erun) + 1):
            run = db.get_run(run_number)
            if not run:
                print("Run %s is not found in DB.. skip this run" % run_number)
                continue


#            arm_flag = find_arm_flag_number(run_number)
            db.add_condition(run, ParityConditions.ARM_FLAG, arm_flag, True)
Пример #7
0
def benchmark_conditions_add(cycles, auto_commit=True):
    # Create RCDBProvider object that connects to DB and provide most of the functions
    db = RCDBProvider("sqlite:///example.db")

    # Create condition type
    runs = [db.create_run(1), db.create_run(2)]
    ct = db.create_condition_type("my_val", ConditionType.INT_FIELD, "This is my value")

    print ("Start {} cycles in benchmark_conditions_add".format(cycles))

    # cycle
    for i in range(0, cycles):
        print_cycle(i, cycles, "benchmark_conditions_add")
        db.add_condition(random.choice(runs), ct, random.randrange(0, 1000),
                         replace=True, auto_commit=auto_commit)

    # commit in the end if needed
    if not auto_commit:
        db.session.commit()

    print ("Done benchmark_conditions_add")
Пример #8
0
import sys
from rcdb import RCDBProvider
from rcdb.model import run_periods
from sqlalchemy import text, bindparam, Integer

if __name__ == "__main__":
    # Get connection string from arguments
    parser = argparse.ArgumentParser(description="This example shows select runs and put them by dates")
    parser.add_argument("connection_string",
                        nargs='?',
                        help="RCDB connection string mysql://rcdb@localhost/rcdb",
                        default="mysql://[email protected]/rcdb")
    args = parser.parse_args()

    # Open DB connection
    db = RCDBProvider(args.connection_string)

    # Get runs from run period
    run_min, run_max, description = run_periods["2016-02"]
    print("Selecting runs {}-{} from run period: '{}'".format(run_min, run_max, description))

    # print resulting array
    # sql = text('SELECT * FROM runs WHERE number > :run')
    # sql.bindparams(run_number=30000)
    # result = db.session.connection().execute(sql, run=31000)
    # for row in result:
    # print row

    result = db.select_values(['event_count', 'daq_run', 'beam_energy', 'beam_current'], "@is_production", 30000)

    print (result.performance)
Пример #9
0

if __name__ == "__main__":
    print sys.argv
    # Get connection string from arguments
    parser = argparse.ArgumentParser(
        description="This example shows select runs and put them by dates")
    parser.add_argument(
        "connection_string",
        nargs='?',
        help="RCDB connection string mysql://rcdb@localhost/rcdb",
        default="mysql://[email protected]/rcdb")
    args = parser.parse_args()

    # Open DB connection
    db = RCDBProvider(args.connection_string)

    # Get runs from run period
    run_min, run_max, description = db.get_run_period("2016-02")
    print("Selecting runs {}-{} from run period: '{}'".format(
        run_min, run_max, description))

    data_by_date, performance = get_runs_by_date(
        db, "@status_approved and @is_production", run_min, run_max)

    # Here we can see exact performance budget. All in seconds
    print("Performance([sec]) is: ", performance)

    # print resulting array
    for run_date in sorted(data_by_date.keys()):
        run_values = data_by_date[run_date]
Пример #10
0
def update_run(run_number):
    # DB Connection
    con_str = os.environ["RCDB_CONNECTION"] \
        if "RCDB_CONNECTION" in os.environ.keys() \
        else "mysql://[email protected]:3306/a-rcdb"

    db = RCDBProvider(con_str)

    run = db.get_run(run_number)
    if not run:
        print("Run %s is not found in DB" % run_number)
        run = db.create_run(run_number)

    conditions = {}
    conditions = db_fix_helper.get_run_end_info_from_data(run_number)

    #Replace with the start from the data file.. there are some cases where we have total time < 0
    #because the run was too short... there is 30sec delay in start script
    #delay was introduced to give enough time to the end run log before a new run log appears
    #run_length = (datetime.strptime(conditions["run_end_time"], "%Y-%m-%d %H:%M:%S") - run.start_time).total_seconds()
    run_length = (
        datetime.strptime(conditions["run_end_time"], "%Y-%m-%d %H:%M:%S") -
        datetime.strptime(conditions["run_start_time"],
                          "%Y-%m-%d %H:%M:%S")).total_seconds()

    event_rate = None
    if float(run_length) > 0 and conditions["event_count"] is not None:
        event_rate = float(conditions["event_count"]) / float(run_length)
    else:
        if not float(run_length) > 0:
            print("ERROR: run_length < 0....skip the update")
            print "Reported start time:", conditions["run_start_time"]
            print "Reported end time:", conditions["run_end_time"]
            print
        if conditions["event_count"] is None:
            print("ERROR: event_count not available")

    if TESTMODE:
        print("Run start time:\t %s" % conditions["run_start_time"])
        print("Run end time:\t %s" % conditions["run_end_time"])
        print("Total Events:\t %s" % conditions["event_count"])
        print("Run length  :\t %s" % run_length)
        print("Event rate  :\t %s" % event_rate)
    else:
        print "Update coda run info", run.number
        run.start_time = conditions["run_start_time"]
        run.end_time = conditions["run_end_time"]
        db.add_condition(run, DefaultConditions.RUN_LENGTH, run_length, True)
        db.add_condition(run, DefaultConditions.IS_VALID_RUN_END,
                         conditions["has_run_end"], True)

        if conditions["event_count"] is not None:
            db.add_condition(run, DefaultConditions.EVENT_COUNT,
                             conditions["event_count"], True)
        if event_rate is not None:
            db.add_condition(run, DefaultConditions.EVENT_RATE, event_rate,
                             True)
        db.session.commit()
Пример #11
0
"""
import argparse
import sys
from rcdb import RCDBProvider

if __name__ == "__main__":
    print sys.argv
    # Get connection string from arguments
    parser = argparse.ArgumentParser(
        description=
        "This example shows basics of how to select runs using search queries")
    parser.add_argument(
        "connection_string",
        help="RCDB connection string mysql://rcdb@localhost/rcdb")
    args = parser.parse_args()

    # Open DB connection
    db = RCDBProvider(args.connection_string)

    # Select production runs with event_count > 0.5M
    result = db.select_runs("@is_production and event_count > 500000", 10000,
                            20000)

    # Iterate through results
    for run in result:
        print run.number, run.get_condition_value("event_count")

    # Another way of getting values
    rows = result.get_values(
        ["event_count", "polarization_direction", "beam_current"])
Пример #12
0
from rcdb.stopwatch import StopWatchTimer
from sqlalchemy import desc
from sqlalchemy.orm import aliased

if __name__ == "__main__":
    print sys.argv
    # Get connection string from arguments
    parser = argparse.ArgumentParser(description="This example shows basics of how to select runs using search queries")
    parser.add_argument("connection_string", help="RCDB connection string mysql://rcdb@localhost/rcdb")
    parser.add_argument('target_cnd_names', nargs='*')
    args = parser.parse_args()

    print "Conditions:", args.target_cnd_names

    # Open DB connection
    db = RCDBProvider(args.connection_string)

    sw = StopWatchTimer()
    sw.start()

    # Select production runs with event_count > 0.5M
    result = db.select_runs("event_count > 100", 0, 20000)



    #result.runs = db.get_runs(0, 20000)

    sw.stop()
    print "Selection took ", sw.elapsed

    print("Selected {} runs".format(len(result.runs)))
Пример #13
0
def update_run(run_number):
    # DB Connection
    con_str = os.environ["RCDB_CONNECTION"] \
        if "RCDB_CONNECTION" in os.environ.keys() \
        else "mysql://[email protected]:3306/a-rcdb"

    db = RCDBProvider(con_str)

    run = db.get_run(run_number)
    if not run:
        print("Run %s is not found in DB" % run_number)
        sys.exit(1)

    conditions = {}
    conditions = db_fix_helper.get_run_end_info_from_data(run_number)

    run_length = (
        datetime.strptime(conditions["run_end_time"], "%Y-%m-%d %H:%M:%S") -
        run.start_time).total_seconds()
    event_rate = None
    if float(run_length) > 0 and conditions["event_count"] is not None:
        event_rate = float(conditions["event_count"]) / float(run_length)
    else:
        if not float(run_length) > 0:
            print("ERROR: run_length < 0....skip the update\n")
        if conditions["event_count"] is None:
            print("ERROR: event_count not available")

    if TESTMODE:
        print("Run end time:\t %s" % conditions["run_end_time"])
        print("Total Events:\t %s" % conditions["event_count"])
        print("Run length  :\t %s" % run_length)
        print("Event rate  :\t %s" % event_rate)
    else:
        run.end_time = conditions["run_end_time"]
        db.add_condition(run, DefaultConditions.RUN_LENGTH, run_length, True)
        db.add_condition(run, DefaultConditions.IS_VALID_RUN_END,
                         conditions["has_run_end"], True)

        if conditions["event_count"] is not None:
            db.add_condition(run, DefaultConditions.EVENT_COUNT,
                             conditions["event_count"], True)
        if event_rate is not None:
            db.add_condition(run, DefaultConditions.EVENT_RATE, event_rate,
                             True)
        db.session.commit()
Пример #14
0
        print "Committing to DB"
        db.session.commit()


if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="Check RCDB data for errors and fixing it")
    parser.add_argument(
        "-e",
        "--execute",
        action="store_true",
        help=
        "Execute repairs. Fithout the flag script only shows, what is to be done",
        default=False)
    parser.add_argument("connection", help="RCDB connection string")
    parser.add_argument("-a",
                        "--action",
                        choices=['dbl', 'mpl', 'no_daq_run'],
                        default='dbl')
    args = parser.parse_args()

    print(args)

    # Create RCDBProvider object that connects to DB and provide most of the functions
    db = RCDBProvider(args.connection)

    if args.action == 'dbl':
        fix_double_runs(db, args.execute)
    elif args.action == 'no_daq_run':
        fix_no_daq_run(db, args.execute)
Пример #15
0
if __name__ == "__main__":
    print sys.argv
    # Get connection string from arguments
    parser = argparse.ArgumentParser(
        description=
        "This example shows basics of how to select runs using search queries")
    parser.add_argument(
        "connection_string",
        help="RCDB connection string mysql://rcdb@localhost/rcdb",
        nargs='?',
        default='mysql://[email protected]/rcdb')
    args = parser.parse_args()

    # Open DB connection
    db = RCDBProvider(args.connection_string)

    # Select production runs with event_count > 0.5M
    result = db.select_values(
        ["event_count", "polarization_direction", "beam_current"],
        "@is_production and event_count > 500000", 10000, 20000)

    # print title
    print("{:>7} {:>15} {:>15} {:>15}".format('run', 'polarization_direction',
                                              'beam_current', 'event_count'))

    # Iterate through results
    for row in result:
        run, event_count, polarization_direction, beam_current = tuple(row)
        print("{:>7} {:>15} {:>15} {:>15}".format(run, polarization_direction,
                                                  beam_current, event_count))
Пример #16
0
This example requires some live databse

Usage:
   python example_simple_queries.py <connectionstring>

"""
import argparse
import sys
from rcdb import RCDBProvider


if __name__ == "__main__":
    print sys.argv
    # Get connection string from arguments
    parser = argparse.ArgumentParser(description="This example shows basics of how to select runs using search queries")
    parser.add_argument("connection_string", help="RCDB connection string mysql://rcdb@localhost/rcdb")
    args = parser.parse_args()

    # Open DB connection
    db = RCDBProvider(args.connection_string)

    # Select production runs with event_count > 0.5M
    result = db.select_runs("@is_production and event_count > 500000", 10000, 20000)

    # Iterate through results
    for run in result:
        print run.number, run.get_condition_value("event_count")

    # Another way of getting values
    rows = result.get_values(["event_count", "polarization_direction", "beam_current"])
Пример #17
0
"""
import argparse
import sys
from rcdb import RCDBProvider
from rcdb.model import ConditionType, Run
from rcdb import DefaultConditions

if __name__ == "__main__":
    print sys.argv
    # Get connection string from arguments
    parser = argparse.ArgumentParser(description="This example shows select runs and put them by dates")
    parser.add_argument("connection_string", nargs='?', default="mysql://[email protected]/rcdb")
    args = parser.parse_args()

    # Open DB connection
    db = RCDBProvider(args.connection_string)

    # add two conditions type. If they are already exist this function just does nothing
    db.create_condition_type(DefaultConditions.RUN_START_TIME, ConditionType.TIME_FIELD, "Run start time by DAQ")
    db.create_condition_type(DefaultConditions.RUN_END_TIME, ConditionType.TIME_FIELD, "Run end time by DAQ")

    # get all runs
    runs = db.get_runs(0, sys.maxint)
    for run in runs:
        print (run.number)
        if run.start_time:
            db.add_condition(run, DefaultConditions.RUN_START_TIME, run.start_time)
        if run.end_time:
            db.add_condition(run, DefaultConditions.RUN_END_TIME, run.end_time)

Пример #18
0
 def __init__(self, home, connection_str):
     self.home = home
     self.db = RCDBProvider(connection_str)
     self.config = {}
     self.verbose = False
     self.connection_str = connection_str