def tail_experiment(exp):
from src.dbms.utils import db_quick_connect
expdb = db_quick_connect("exp")
stdout = open(exp.stdout, "rb")
printed_until = 0
while True:
stdout.seek(printed_until)
for line in stdout:
print line, # do not print newline twice
printed_until = stdout.tell()
# Stop if experiment is not running
check = list(
expdb.sql_query(
"SELECT FROM exp WHERE status = 'running' AND id = %s",
exp.id))
if len(check) < 1:
break
time.sleep(0.5)
# Print last lines from experiment (if any)
for line in stdout:
print line
stdout.close()
def set_experiment_as_running(exp):
from src.dbms.utils import db_quick_connect
expdb = db_quick_connect("exp")
expdb.sql_update(
"UPDATE exp SET status = 'running', starttime = NOW() WHERE id = %s",
exp.id)
expdb.commit()
expdb.close()
def get_current_running_experiment():
from src.dbms.utils import db_quick_connect
expdb = db_quick_connect("exp")
exp = list(
expdb.sql_query(
"SELECT * FROM exp WHERE status = 'running' ORDER BY addtime, id LIMIT 1"
))
expdb.close()
if len(exp) >= 1:
return exp[0]
else:
return None
def proc_dump(args, port):
from src.dbms.utils import db_quick_connect
from src.experiments.settings import datasets_directory_path
dbname, schemaname, tablename = args
expdb = db_quick_connect("exp")
r = list(
expdb.sql_query(
"SELECT * FROM datasets WHERE "
"db_name = %s AND schema_name = %s AND table_name = %s", dbname,
schemaname, tablename))
if len(r) < 1:
raise Exception(
"Dataset table {}.{} not registered for database {}".format(
schemaname, tablename, dbname))
elif len(r) > 1:
raise Exception(
"Ambiguous table {}.{} in database {}. More than 1 entry.".format(
schemaname, tablename, dbname))
outfilename = schemaname + "." + tablename + ".sql"
dump_rel_path = os.path.join(datasets_directory_path, r[0].dump_rel_path,
outfilename)
newargs = ["pg_dump"] + (["-p", port] if port else []) + [
"-f",
dump_rel_path,
"-F",
"c", # "Custom" format for pg_dump,
"-n",
schemaname,
"-O", # Do not output ownership commands
"-t",
tablename,
"-v",
dbname
]
print " ".join(newargs)
return newargs
def main():
parser = ArgumentParser(description="Pachage Query Test Runner.")
parser.add_argument('experiment_folder')
parser.add_argument('experiment_file', nargs='?', default="main")
parser.add_argument("--run-now", action="store_true")
parser.add_argument("--expdb", dest="experiment_dbname", default=None)
parser.add_argument('--poolsize', default=1, type=int)
parser.add_argument("--logging-level", dest="logging_level", default=logging.INFO)
parser.add_argument("--debug",
help='Print lots of debugging statements',
action="store_const", dest="logging_level", const=logging.DEBUG, default=logging.INFO)
parser.add_argument("-V", "--verbose",
help='Be verbose',
action="store_const", dest="verbose", const=True, default=False)
parser.set_defaults(run_now=False)
# Parse arguments
args, other_args = parser.parse_known_args()
# Override experimental DB name
if args.experiment_dbname is not None:
exp_dbms_settings["dbname"] = args.experiment_dbname
if not args.run_now:
# Schedule experiment
expdb = db_quick_connect("exp")
expdb.sql_update(
"INSERT INTO exp(exp_dir, exp_file, args, expdb, poolsize, logging_level, set_verbose) "
"VALUES (%s, %s, %s, %s, %s, %s, %s)",
args.experiment_folder,
args.experiment_file,
" ".join(other_args),
args.experiment_dbname,
args.poolsize,
args.logging_level,
args.verbose,
)
expdb.commit()
expdb.close()
else:
# Set logging level
set_logging_level(logging_level=args.logging_level, verbose=args.verbose)
if args.poolsize > 1:
pool = Pool(processes=args.poolsize) # start a pool of parallel worker processes
else:
pool = None
exp_module_name = "src.experiments." + args.experiment_folder + "." + args.experiment_file
experiment_module = import_module(exp_module_name)
for ExperimentClass in experiment_module.tests:
experiment = ExperimentClass(pool, args.experiment_folder, args.experiment_file)
assert isinstance(experiment, Experiment)
print "#########################################################"
print "Experiment Name:"
print "\t", experiment.experiment_name
print "Experiment Description:"
print "\t", experiment.description
print "---------------------------------------------------------"
print "# Experiment Running..."
print "---------------------------------------------------------"
# Run the test experiment
experiment.start()
try:
experiment(other_args)
except Exception as e:
print "Exception during experiment"
traceback.print_exc(file=sys.stdout)
raise e
experiment.finish()
print "---------------------------------------------------------"
print "# Experiment Finished."
print "#########################################################"
print "End of All Experiments."
def proc_restore(args, port):
from src.experiments.settings import datasets_directory_path
from src.dbms.utils import db_quick_connect
from src.dbms.dbms_settings import data_dbms_settings, exp_dbms_settings
from_dbname, schemaname, tablename = args
to_dbname = data_dbms_settings["dbname"]
expdb = db_quick_connect("exp")
datadb = db_quick_connect("data")
r = list(
expdb.sql_query(
"SELECT * FROM datasets WHERE "
"db_name = %s AND schema_name = %s AND table_name = %s", to_dbname,
schemaname, tablename))
if len(r) >= 1:
raise Exception("Table {}.{} in database {} already exists.".format(
schemaname, tablename, to_dbname))
outfilename = schemaname + "." + tablename + ".sql"
dump_rel_path = os.path.join(
datasets_directory_path,
"..",
from_dbname,
"dumps", # NOTE: this assumes that folder "dumps" is always used
outfilename)
print datasets_directory_path
print os.path.join(datasets_directory_path, "dump")
newargs = ["pg_restore"] + (["-p", port] if port else []) + [
"-F",
"c", # "Custom" format for pg_dump
"-c", # Clean: clean (drop) database objects before recreating them
"-n",
schemaname,
"-O", # Do not output ownership commands
"-t",
tablename,
"-v",
"-d",
to_dbname,
dump_rel_path
]
def callback():
# Execute this after pg_restore has completed
table_size = datadb.sql_query("SELECT COUNT(*) FROM {}.{}".format(
schemaname, tablename)).next()[0]
expdb.sql_update(
"INSERT INTO datasets (db_name, schema_name, table_name, table_size, dump_rel_path) "
"VALUES (%s, %s, %s, %s, %s)", to_dbname, schemaname, tablename,
table_size, "dumps")
expdb.commit()
print " ".join(newargs)
return newargs, callback
def main():
expdb = db_quick_connect("exp")
running_processes = OrderedDict()
atexit.register(cleanup, running_processes)
while True:
expdb.commit()
# Currently running experiments
current_exps = check_current_running_experiments(expdb)
# Next-up experiment
next_exps = get_next_scheduled_experiments(expdb)
# If some processes died, remove them running processes
current_exps_set = set(e.id for e in current_exps)
for pid in running_processes.iterkeys():
if running_processes[pid]["id"] not in current_exps_set:
del running_processes[pid]
# Asserts
assert current_exps_set == set(p["id"] for p in running_processes.itervalues()), \
(current_exps_set, set(p["id"] for p in running_processes.itervalues()))
# Prints
if len(current_exps) or len(next_exps):
print
if len(current_exps):
print "There are {} currently running experiments{}".format(
len(current_exps), ":" if len(current_exps) else ".")
for i, e in enumerate(current_exps, start=1):
print "R{}) Experiment(id={}) has been running for {}".format(
i, e.id, e.time_elapsed)
if len(next_exps):
print "There are {} scheduled experiments{}".format(
len(next_exps), ":" if len(next_exps) else ".")
for i, e in enumerate(next_exps, start=1):
print "S{}) Experiment(id={}) {} {}".format(
i, e.id, e.exp_dir, e.exp_file)
if len(current_exps) == 0 and len(next_exps) >= 1:
exp = next_exps[0]
print "Preparing to run Experiment(id={}) {} {}...".format(
exp.id, exp.exp_dir, exp.exp_file)
args = [
"/usr/bin/env", "python", "-m",
"src.experiments.run_experiments", "--run-now", exp.exp_dir,
exp.exp_file
] + exp.args.split(" ") + [
("--expdb", exp.expdb) if exp.expdb is not None else "",
"--poolsize",
str(exp.poolsize),
# "--repetitions", str(exp.repetitions),
"--logging_level",
str(exp.logging_level),
"--verbose" if exp.set_verbose else ""
]
uid = uuid.uuid4().hex
stdout_file_id = "{}_{}".format(exp.id, uid)
stderr_file_id = "{}_{}".format(exp.id, uid)
stdout = open(
os.path.join(result_data_directory_path, "stdout",
stdout_file_id), "wb", 1)
stderr = open(
os.path.join(result_data_directory_path, "stderr",
stderr_file_id), "wb", 1)
print "Running Experiment(id={}):".format(exp.id)
print " ".join(args)
p = subprocess.Popen(args,
env=os.environ,
stdout=stdout,
stderr=stderr)
set_experiment_as_running(expdb, exp, p.pid, stdout, stderr)
running_processes[p.pid] = {
"id": exp.id,
"proc": p,
"stdout": stdout,
"stderr": stderr,
}
def callback(p):
while p.poll() is None:
try:
p.wait()
except (KeyboardInterrupt, SystemExit) as e:
print(KeyboardInterrupt, SystemExit)
p.send_signal(signal.SIGINT)
# sys.exit(-1)
set_experiment_as_error(expdb, exp, str(e))
except Exception as e:
print Exception
p.send_signal(signal.SIGTERM)
# raise e
set_experiment_as_error(expdb, exp, str(e))
else:
if p.returncode == 0:
set_experiment_as_success(expdb, exp)
else:
set_experiment_as_error(
expdb, exp,
"returncode = {}".format(p.returncode))
# finally:
# set_experiment_endtime(expdb, exp)
# Set callback function
thread = threading.Thread(target=callback, args=(p, ))
thread.start()
# else:
# print "No new experiments to run."
time.sleep(5)
expdb.close()