def tail_experiment(exp): from src.dbms.utils import db_quick_connect expdb = db_quick_connect("exp") stdout = open(exp.stdout, "rb") printed_until = 0 while True: stdout.seek(printed_until) for line in stdout: print line, # do not print newline twice printed_until = stdout.tell() # Stop if experiment is not running check = list( expdb.sql_query( "SELECT FROM exp WHERE status = 'running' AND id = %s", exp.id)) if len(check) < 1: break time.sleep(0.5) # Print last lines from experiment (if any) for line in stdout: print line stdout.close()
def set_experiment_as_running(exp): from src.dbms.utils import db_quick_connect expdb = db_quick_connect("exp") expdb.sql_update( "UPDATE exp SET status = 'running', starttime = NOW() WHERE id = %s", exp.id) expdb.commit() expdb.close()
def get_current_running_experiment(): from src.dbms.utils import db_quick_connect expdb = db_quick_connect("exp") exp = list( expdb.sql_query( "SELECT * FROM exp WHERE status = 'running' ORDER BY addtime, id LIMIT 1" )) expdb.close() if len(exp) >= 1: return exp[0] else: return None
def proc_dump(args, port): from src.dbms.utils import db_quick_connect from src.experiments.settings import datasets_directory_path dbname, schemaname, tablename = args expdb = db_quick_connect("exp") r = list( expdb.sql_query( "SELECT * FROM datasets WHERE " "db_name = %s AND schema_name = %s AND table_name = %s", dbname, schemaname, tablename)) if len(r) < 1: raise Exception( "Dataset table {}.{} not registered for database {}".format( schemaname, tablename, dbname)) elif len(r) > 1: raise Exception( "Ambiguous table {}.{} in database {}. More than 1 entry.".format( schemaname, tablename, dbname)) outfilename = schemaname + "." + tablename + ".sql" dump_rel_path = os.path.join(datasets_directory_path, r[0].dump_rel_path, outfilename) newargs = ["pg_dump"] + (["-p", port] if port else []) + [ "-f", dump_rel_path, "-F", "c", # "Custom" format for pg_dump, "-n", schemaname, "-O", # Do not output ownership commands "-t", tablename, "-v", dbname ] print " ".join(newargs) return newargs
def main(): parser = ArgumentParser(description="Pachage Query Test Runner.") parser.add_argument('experiment_folder') parser.add_argument('experiment_file', nargs='?', default="main") parser.add_argument("--run-now", action="store_true") parser.add_argument("--expdb", dest="experiment_dbname", default=None) parser.add_argument('--poolsize', default=1, type=int) parser.add_argument("--logging-level", dest="logging_level", default=logging.INFO) parser.add_argument("--debug", help='Print lots of debugging statements', action="store_const", dest="logging_level", const=logging.DEBUG, default=logging.INFO) parser.add_argument("-V", "--verbose", help='Be verbose', action="store_const", dest="verbose", const=True, default=False) parser.set_defaults(run_now=False) # Parse arguments args, other_args = parser.parse_known_args() # Override experimental DB name if args.experiment_dbname is not None: exp_dbms_settings["dbname"] = args.experiment_dbname if not args.run_now: # Schedule experiment expdb = db_quick_connect("exp") expdb.sql_update( "INSERT INTO exp(exp_dir, exp_file, args, expdb, poolsize, logging_level, set_verbose) " "VALUES (%s, %s, %s, %s, %s, %s, %s)", args.experiment_folder, args.experiment_file, " ".join(other_args), args.experiment_dbname, args.poolsize, args.logging_level, args.verbose, ) expdb.commit() expdb.close() else: # Set logging level set_logging_level(logging_level=args.logging_level, verbose=args.verbose) if args.poolsize > 1: pool = Pool(processes=args.poolsize) # start a pool of parallel worker processes else: pool = None exp_module_name = "src.experiments." + args.experiment_folder + "." + args.experiment_file experiment_module = import_module(exp_module_name) for ExperimentClass in experiment_module.tests: experiment = ExperimentClass(pool, args.experiment_folder, args.experiment_file) assert isinstance(experiment, Experiment) print "#########################################################" print "Experiment Name:" print "\t", experiment.experiment_name print "Experiment Description:" print "\t", experiment.description print "---------------------------------------------------------" print "# Experiment Running..." print "---------------------------------------------------------" # Run the test experiment experiment.start() try: experiment(other_args) except Exception as e: print "Exception during experiment" traceback.print_exc(file=sys.stdout) raise e experiment.finish() print "---------------------------------------------------------" print "# Experiment Finished." print "#########################################################" print "End of All Experiments."
def proc_restore(args, port): from src.experiments.settings import datasets_directory_path from src.dbms.utils import db_quick_connect from src.dbms.dbms_settings import data_dbms_settings, exp_dbms_settings from_dbname, schemaname, tablename = args to_dbname = data_dbms_settings["dbname"] expdb = db_quick_connect("exp") datadb = db_quick_connect("data") r = list( expdb.sql_query( "SELECT * FROM datasets WHERE " "db_name = %s AND schema_name = %s AND table_name = %s", to_dbname, schemaname, tablename)) if len(r) >= 1: raise Exception("Table {}.{} in database {} already exists.".format( schemaname, tablename, to_dbname)) outfilename = schemaname + "." + tablename + ".sql" dump_rel_path = os.path.join( datasets_directory_path, "..", from_dbname, "dumps", # NOTE: this assumes that folder "dumps" is always used outfilename) print datasets_directory_path print os.path.join(datasets_directory_path, "dump") newargs = ["pg_restore"] + (["-p", port] if port else []) + [ "-F", "c", # "Custom" format for pg_dump "-c", # Clean: clean (drop) database objects before recreating them "-n", schemaname, "-O", # Do not output ownership commands "-t", tablename, "-v", "-d", to_dbname, dump_rel_path ] def callback(): # Execute this after pg_restore has completed table_size = datadb.sql_query("SELECT COUNT(*) FROM {}.{}".format( schemaname, tablename)).next()[0] expdb.sql_update( "INSERT INTO datasets (db_name, schema_name, table_name, table_size, dump_rel_path) " "VALUES (%s, %s, %s, %s, %s)", to_dbname, schemaname, tablename, table_size, "dumps") expdb.commit() print " ".join(newargs) return newargs, callback
def main(): expdb = db_quick_connect("exp") running_processes = OrderedDict() atexit.register(cleanup, running_processes) while True: expdb.commit() # Currently running experiments current_exps = check_current_running_experiments(expdb) # Next-up experiment next_exps = get_next_scheduled_experiments(expdb) # If some processes died, remove them running processes current_exps_set = set(e.id for e in current_exps) for pid in running_processes.iterkeys(): if running_processes[pid]["id"] not in current_exps_set: del running_processes[pid] # Asserts assert current_exps_set == set(p["id"] for p in running_processes.itervalues()), \ (current_exps_set, set(p["id"] for p in running_processes.itervalues())) # Prints if len(current_exps) or len(next_exps): print if len(current_exps): print "There are {} currently running experiments{}".format( len(current_exps), ":" if len(current_exps) else ".") for i, e in enumerate(current_exps, start=1): print "R{}) Experiment(id={}) has been running for {}".format( i, e.id, e.time_elapsed) if len(next_exps): print "There are {} scheduled experiments{}".format( len(next_exps), ":" if len(next_exps) else ".") for i, e in enumerate(next_exps, start=1): print "S{}) Experiment(id={}) {} {}".format( i, e.id, e.exp_dir, e.exp_file) if len(current_exps) == 0 and len(next_exps) >= 1: exp = next_exps[0] print "Preparing to run Experiment(id={}) {} {}...".format( exp.id, exp.exp_dir, exp.exp_file) args = [ "/usr/bin/env", "python", "-m", "src.experiments.run_experiments", "--run-now", exp.exp_dir, exp.exp_file ] + exp.args.split(" ") + [ ("--expdb", exp.expdb) if exp.expdb is not None else "", "--poolsize", str(exp.poolsize), # "--repetitions", str(exp.repetitions), "--logging_level", str(exp.logging_level), "--verbose" if exp.set_verbose else "" ] uid = uuid.uuid4().hex stdout_file_id = "{}_{}".format(exp.id, uid) stderr_file_id = "{}_{}".format(exp.id, uid) stdout = open( os.path.join(result_data_directory_path, "stdout", stdout_file_id), "wb", 1) stderr = open( os.path.join(result_data_directory_path, "stderr", stderr_file_id), "wb", 1) print "Running Experiment(id={}):".format(exp.id) print " ".join(args) p = subprocess.Popen(args, env=os.environ, stdout=stdout, stderr=stderr) set_experiment_as_running(expdb, exp, p.pid, stdout, stderr) running_processes[p.pid] = { "id": exp.id, "proc": p, "stdout": stdout, "stderr": stderr, } def callback(p): while p.poll() is None: try: p.wait() except (KeyboardInterrupt, SystemExit) as e: print(KeyboardInterrupt, SystemExit) p.send_signal(signal.SIGINT) # sys.exit(-1) set_experiment_as_error(expdb, exp, str(e)) except Exception as e: print Exception p.send_signal(signal.SIGTERM) # raise e set_experiment_as_error(expdb, exp, str(e)) else: if p.returncode == 0: set_experiment_as_success(expdb, exp) else: set_experiment_as_error( expdb, exp, "returncode = {}".format(p.returncode)) # finally: # set_experiment_endtime(expdb, exp) # Set callback function thread = threading.Thread(target=callback, args=(p, )) thread.start() # else: # print "No new experiments to run." time.sleep(5) expdb.close()