def __init__(self, pool_path, config_path=None): """Init class""" cfg = config.read_config(config_path)["DEFAULT"] self.ippool = ippool.Ippool(pool_path) self.params = dict() keys = ["limit", "socket_num", "ssl_num", "socket_timeout", "ssl_timeout"] for k in keys: self.params[k] = int(cfg[k])
def test_read_config(self): config = io.StringIO(\ "foo='bar'\n"\ "bar=42.2\n"\ "#Hello this is a comment\n"\ "\n"\ "baz=True\n"\ ) expected = {\ "foo" : "bar", "bar" : 42.2, "baz" : True, } self.assertEquals(read_config(config), expected)
def test_svg_suite(cfg, args, silent=False): assert len(args) != 0 if 'all' in args: assert len(args) == 1 args = list(TARGET_INFO.keys()) cfg = config.read_config() svg_test_suite_root = cfg.get('test','svg_test_suite_root') faint_exe = cfg.get('test', 'faint_exe') svg_files = config.list_svg_files(svg_test_suite_root) for num, target_type in enumerate(sorted(args)): setup_test(target_type) run_test(faint_exe, svg_test_suite_root, target_type, svg_files, num, len(args), silent)
def test_svg_suite(cfg, args, silent=False): assert len(args) != 0 if 'all' in args: assert len(args) == 1 args = list(TARGET_INFO.keys()) cfg = config.read_config() svg_test_suite_root = cfg.get('test', 'svg_test_suite_root') faint_exe = cfg.get('test', 'faint_exe') svg_files = config.list_svg_files(svg_test_suite_root) for num, target_type in enumerate(sorted(args)): setup_test(target_type) run_test(faint_exe, svg_test_suite_root, target_type, svg_files, num, len(args), silent)
args = list(TARGET_INFO.keys()) cfg = config.read_config() svg_test_suite_root = cfg.get('test','svg_test_suite_root') faint_exe = cfg.get('test', 'faint_exe') svg_files = config.list_svg_files(svg_test_suite_root) for num, target_type in enumerate(sorted(args)): setup_test(target_type) run_test(faint_exe, svg_test_suite_root, target_type, svg_files, num, len(args), silent) if __name__ == '__main__': if config.maybe_create_config(): exit(1) opts, args = parse_options() if len(args) == 0: targets = "all " + " ".join(TARGET_INFO.keys()) print("No arguments specified. Expected: %s" % targets) exit(1) if 'all' in args: if len(args) != 1: print('Error: No additional arguments allowed with "all".') exit(1) cfg = config.read_config() test_svg_suite(cfg, args)
def main(): set_logging_level(logging.INFO) # Parse arguments parser = ArgumentParser() parser.add_argument("act", nargs="?", default=None) args, other_args = parser.parse_known_args() act = args.act callback = None sshtunnel = None shell = False args = [] if act is None or act == "shell": # With no arguments, simply run ipython with PB environment os.putenv("PYTHONSTARTUP", "src/utils/pb_startup_python_interpreter.py") args = ["ipython"] elif act == "help": print_help() exit(0) elif act == "set": config_name = other_args[0] if config_name.endswith(".cfg"): config_name = config_name[:-4] if not os.path.isfile( os.path.join(PB_HOME, "{}.cfg".format(config_name))): raise Exception( "Config file `{}.cfg' does not exist in PB_HOME.".format( config_name)) print "Setting config {}.cfg.".format(config_name) os.putenv("PB_CONF", config_name) os.system("/usr/bin/env bash") exit() elif act == "daemon": from src.pb import pb_daemon pb_daemon.main() elif act == "db": config = read_config() from src.dbms.dbms_settings import data_dbms_settings, exp_dbms_settings port = data_dbms_settings[ "port"] if data_dbms_settings["port"] != "" else "5432" host = data_dbms_settings["host"] orig_port = port db_data_folder = config.get("Folders", "dbms_folder") # If not localhost, perform SSH tunneling if host != "localhost" and host != "127.0.0.1" and host != "": sshtunnel = SSHTunnelForwarder( (host, 22), remote_bind_address=("127.0.0.1", int(port))) sshtunnel.start() host = "localhost" port = str(sshtunnel.local_bind_port) if len(other_args) > 0: if other_args[0] == "log": base_logfile = "{}/{}".format(db_data_folder, DB_LOGFILE_NAME) other_logfiles = glob.glob( "{}/pg_log/*".format(db_data_folder)) log("Tailing log files:\n{}".format("\n".join([base_logfile] + other_logfiles))) args = ["tail", "-n", "50", "-f", base_logfile ] + other_logfiles elif other_args[0] == "start": args = [ "pg_ctl", "-o", "-F", "-p " + orig_port, "-D", db_data_folder, "-l", "{}/{}".format(db_data_folder, DB_LOGFILE_NAME), "start" ] elif other_args[0] == "restart": opts = "-F" + \ (" -p " + port if port else "") + \ (" -h " + host if host else "") args = [ "pg_ctl", "-o", opts, "-D", db_data_folder, "-m", "fast", "restart" ] elif other_args[0] == "stop": args = ["pg_ctl", "-D", db_data_folder, "-m", "fast", "stop"] elif other_args[0] == data_dbms_settings["dbname"] or other_args[ 0] == "data": sql = prepare_sql(other_args[1:]) if sql: # Execute SQL on the db args = [ "psql" ] + \ ([ "-p", port ] if port else []) + \ ([ "-h", host ] if host else []) + \ [ "-c", sql, data_dbms_settings["dbname"] ] else: # Just connect to the db args = [ "psql"] + \ ([ "-p", port ] if port else []) + \ ([ "-h", host ] if host else []) + \ [ data_dbms_settings["dbname"] ] elif other_args[0] == exp_dbms_settings["dbname"] or other_args[ 0] == "exp": sql = prepare_sql(other_args[1:]) if sql: # Execute SQL on the db args = [ "psql" ] + \ ([ "-p", port ] if port else []) + \ ([ "-h", host ] if host else []) + \ [ "-c", sql, exp_dbms_settings["dbname"] ] else: # Just connect to the db args = [ "psql"] + \ ([ "-p", port ] if port else []) + \ ([ "-h", host ] if host else []) + \ [ exp_dbms_settings["dbname"] ] elif other_args[0] == "dump": args = proc_dump(other_args[1:], port) elif other_args[0] == "restore": args, callback = proc_restore(other_args[1:], port) else: # In this case, the command corresponds to a SQL query in the sql/ folder sql = prepare_sql(other_args + [ data_dbms_settings["dbname"], exp_dbms_settings["dbname"] ], folder=os.path.join(PB_HOME, "sql")) args = [ "psql" ] + \ ([ "-p", port ] if port else []) + \ ([ "-h", host ] if host else []) + \ [ "-c", sql, "postgres" ] else: args = [ "psql" ] + \ ([ "-p", port ] if port else []) + \ ([ "-h", host ] if host else []) + \ [ data_dbms_settings["dbname"] ] elif act == "ls": # See scheduled experiment sql = "SELECT id, addtime, exp_dir AS exp, exp_file, args " \ "FROM exp " \ "WHERE status = 'scheduled' " \ "ORDER BY addtime, id " args = ["pb", "db", "exp", sql] #+ other_args elif act == "top": # See running experiment sql = "SELECT id, NOW() - starttime AS time_elapsed, exp_dir AS exp, exp_file, args " \ "FROM exp " \ "WHERE status = 'running' " \ "ORDER BY starttime, id " args = ["pb", "db", "exp", sql] #+ other_args elif act == "tail": exp = get_current_running_experiment() # if exp is None: # args = [ "echo", "No running processes." ] # else: while True: if exp is not None: tail_experiment(exp) time.sleep(0.5) exp = get_current_running_experiment() elif act == "exp": if other_args[0] == "next": assert len(other_args) == 1 # Run next scheduled experiment exp = get_next_scheduled_experiment() print exp.args args = [ "pb", "exprun", exp.exp_dir, exp.exp_file ] + exp.args.split(" ") + [ ("--expdb", exp.expdb) if exp.expdb is not None else "", "--poolsize", str(exp.poolsize), "--repetitions", str(exp.repetitions), "--logging_level", str(exp.logging_level), "--verbose" if exp.set_verbose else "" ] # print args set_experiment_as_running(exp) else: # Schedule experiment args = [ "/usr/bin/env", "python", "-m", "src.experiments.run_experiments" ] + other_args elif act == "exprun": # Run experiment now args = [ "/usr/bin/env", "python", "-m", "src.experiments.run_experiments", "--run-now" ] + other_args elif act == "bench": # Run benchmark benchmark_name = other_args[0] args = [ "/usr/bin/env", "python", "-m", "benchmark.{}.run".format(benchmark_name) ] + other_args[1:] elif act.endswith(".py") and os.path.isfile(act): # Run Python script args = [ "/usr/bin/env", "python", "-m", act.replace("/", ".").replace(".py", "") ] + other_args else: raise Exception("Action '{}' not supported.".format(act)) print "RUNNING:" print " ".join(args) if len(args): p = subprocess.Popen(args, env=os.environ, shell=shell) while p.poll() is None: try: p.wait() except (KeyboardInterrupt, SystemExit) as e: print(KeyboardInterrupt, SystemExit) p.send_signal(signal.SIGINT) # p.wait() if sshtunnel is not None: sshtunnel.close() sshtunnel = None sys.exit(-1) except Exception as e: print Exception p.send_signal(signal.SIGTERM) # p.wait() if sshtunnel is not None: sshtunnel.close() sshtunnel = None raise e if callback is not None: callback() if sshtunnel is not None: sshtunnel.close() sshtunnel = None
#!/usr/bin/env python3 # -*- coding: utf-8 -*- import configparser import os import subprocess import shutil import src.config as config import sys def test_color_background(cfg, silent): faint_exe = cfg.get('test', 'faint_exe') cmd = [faint_exe, '--run', 'faint_scripts/run-test-color-background.py'] if silent: cmd.append('--silent') subprocess.call(cmd) if __name__ == '__main__': if config.maybe_create_config(): exit(1) silent = not 'stay' in sys.argv cfg = config.read_config() test_color_background(cfg, silent)
from src.config import read_config config = read_config() data_dbms_settings = { "dbname": config.get("Data DB", "dbname"), "username": config.get("Data DB", "username"), "password": config.get("Data DB", "password"), "host": config.get("Data DB", "host"), "port": config.get("Data DB", "port"), } exp_dbms_settings = { "dbname": config.get("Experiment DB", "dbname"), "username": config.get("Experiment DB", "username"), "password": config.get("Experiment DB", "password"), "host": config.get("Experiment DB", "host"), "port": config.get("Experiment DB", "port"), }
def __main__(): parser = argparse.ArgumentParser() parser.add_argument('--input', dest='input_dir', required=True, nargs=1, type=str, help='Input dir') parser.add_argument('--output', dest='output_dir', required=True, nargs=1, type=str, help='Output dir') parser.add_argument('--mode', dest='mode', required=True, choices=[constants.MODE_ENCODE, constants.MODE_DECODE]) parser.add_argument('--nolog', dest='nolog', required=False, action='store_true', default=False) parser.add_argument('--silent', '-s', dest='silent', required=False, action='store_true', default=False) args = parser.parse_args() input_dir = args.input_dir[0] output_dir = args.output_dir[0] mode = args.mode.upper() Log.should_log = args.nolog Log.silent = args.silent Log.find_next_log_file_path() # Find next path for log file config.read_config() files = find_files(mode, input_dir) Log.log_to_file(True, 'Found {} files\n'.format(len(files))) if mode == constants.MODE_DECODE: for ks_file in files: decoded = decode.decode(ks_file) if decoded is None: Log.log_to_file( True, 'Failed to decode input file \"{}\"?!'.format(ks_file)) continue write_file(output_dir, ks_file, mode, decoded) continue return elif mode == constants.MODE_ENCODE: for ks_json_file in files: encoded = encode.encode(ks_json_file) write_file(output_dir, ks_json_file, mode, encoded) continue return Log.close_log_file() return