def __init__(self, pool_path, config_path=None):
"""Init class"""
cfg = config.read_config(config_path)["DEFAULT"]
self.ippool = ippool.Ippool(pool_path)
self.params = dict()
keys = ["limit", "socket_num", "ssl_num",
"socket_timeout", "ssl_timeout"]
for k in keys:
self.params[k] = int(cfg[k])
def test_read_config(self):
config = io.StringIO(\
"foo='bar'\n"\
"bar=42.2\n"\
"#Hello this is a comment\n"\
"\n"\
"baz=True\n"\
)
expected = {\
"foo" : "bar",
"bar" : 42.2,
"baz" : True,
}
self.assertEquals(read_config(config), expected)
def test_svg_suite(cfg, args, silent=False):
assert len(args) != 0
if 'all' in args:
assert len(args) == 1
args = list(TARGET_INFO.keys())
cfg = config.read_config()
svg_test_suite_root = cfg.get('test','svg_test_suite_root')
faint_exe = cfg.get('test', 'faint_exe')
svg_files = config.list_svg_files(svg_test_suite_root)
for num, target_type in enumerate(sorted(args)):
setup_test(target_type)
run_test(faint_exe, svg_test_suite_root, target_type, svg_files, num,
len(args), silent)
def test_svg_suite(cfg, args, silent=False):
assert len(args) != 0
if 'all' in args:
assert len(args) == 1
args = list(TARGET_INFO.keys())
cfg = config.read_config()
svg_test_suite_root = cfg.get('test', 'svg_test_suite_root')
faint_exe = cfg.get('test', 'faint_exe')
svg_files = config.list_svg_files(svg_test_suite_root)
for num, target_type in enumerate(sorted(args)):
setup_test(target_type)
run_test(faint_exe, svg_test_suite_root, target_type, svg_files, num,
len(args), silent)
args = list(TARGET_INFO.keys())
cfg = config.read_config()
svg_test_suite_root = cfg.get('test','svg_test_suite_root')
faint_exe = cfg.get('test', 'faint_exe')
svg_files = config.list_svg_files(svg_test_suite_root)
for num, target_type in enumerate(sorted(args)):
setup_test(target_type)
run_test(faint_exe, svg_test_suite_root, target_type, svg_files, num,
len(args), silent)
if __name__ == '__main__':
if config.maybe_create_config():
exit(1)
opts, args = parse_options()
if len(args) == 0:
targets = "all " + " ".join(TARGET_INFO.keys())
print("No arguments specified. Expected: %s" % targets)
exit(1)
if 'all' in args:
if len(args) != 1:
print('Error: No additional arguments allowed with "all".')
exit(1)
cfg = config.read_config()
test_svg_suite(cfg, args)
def main():
set_logging_level(logging.INFO)
# Parse arguments
parser = ArgumentParser()
parser.add_argument("act", nargs="?", default=None)
args, other_args = parser.parse_known_args()
act = args.act
callback = None
sshtunnel = None
shell = False
args = []
if act is None or act == "shell":
# With no arguments, simply run ipython with PB environment
os.putenv("PYTHONSTARTUP",
"src/utils/pb_startup_python_interpreter.py")
args = ["ipython"]
elif act == "help":
print_help()
exit(0)
elif act == "set":
config_name = other_args[0]
if config_name.endswith(".cfg"): config_name = config_name[:-4]
if not os.path.isfile(
os.path.join(PB_HOME, "{}.cfg".format(config_name))):
raise Exception(
"Config file `{}.cfg' does not exist in PB_HOME.".format(
config_name))
print "Setting config {}.cfg.".format(config_name)
os.putenv("PB_CONF", config_name)
os.system("/usr/bin/env bash")
exit()
elif act == "daemon":
from src.pb import pb_daemon
pb_daemon.main()
elif act == "db":
config = read_config()
from src.dbms.dbms_settings import data_dbms_settings, exp_dbms_settings
port = data_dbms_settings[
"port"] if data_dbms_settings["port"] != "" else "5432"
host = data_dbms_settings["host"]
orig_port = port
db_data_folder = config.get("Folders", "dbms_folder")
# If not localhost, perform SSH tunneling
if host != "localhost" and host != "127.0.0.1" and host != "":
sshtunnel = SSHTunnelForwarder(
(host, 22), remote_bind_address=("127.0.0.1", int(port)))
sshtunnel.start()
host = "localhost"
port = str(sshtunnel.local_bind_port)
if len(other_args) > 0:
if other_args[0] == "log":
base_logfile = "{}/{}".format(db_data_folder, DB_LOGFILE_NAME)
other_logfiles = glob.glob(
"{}/pg_log/*".format(db_data_folder))
log("Tailing log files:\n{}".format("\n".join([base_logfile] +
other_logfiles)))
args = ["tail", "-n", "50", "-f", base_logfile
] + other_logfiles
elif other_args[0] == "start":
args = [
"pg_ctl", "-o", "-F", "-p " + orig_port, "-D",
db_data_folder, "-l",
"{}/{}".format(db_data_folder, DB_LOGFILE_NAME), "start"
]
elif other_args[0] == "restart":
opts = "-F" + \
(" -p " + port if port else "") + \
(" -h " + host if host else "")
args = [
"pg_ctl", "-o", opts, "-D", db_data_folder, "-m", "fast",
"restart"
]
elif other_args[0] == "stop":
args = ["pg_ctl", "-D", db_data_folder, "-m", "fast", "stop"]
elif other_args[0] == data_dbms_settings["dbname"] or other_args[
0] == "data":
sql = prepare_sql(other_args[1:])
if sql:
# Execute SQL on the db
args = [ "psql" ] + \
([ "-p", port ] if port else []) + \
([ "-h", host ] if host else []) + \
[ "-c", sql, data_dbms_settings["dbname"] ]
else:
# Just connect to the db
args = [ "psql"] + \
([ "-p", port ] if port else []) + \
([ "-h", host ] if host else []) + \
[ data_dbms_settings["dbname"] ]
elif other_args[0] == exp_dbms_settings["dbname"] or other_args[
0] == "exp":
sql = prepare_sql(other_args[1:])
if sql:
# Execute SQL on the db
args = [ "psql" ] + \
([ "-p", port ] if port else []) + \
([ "-h", host ] if host else []) + \
[ "-c", sql, exp_dbms_settings["dbname"] ]
else:
# Just connect to the db
args = [ "psql"] + \
([ "-p", port ] if port else []) + \
([ "-h", host ] if host else []) + \
[ exp_dbms_settings["dbname"] ]
elif other_args[0] == "dump":
args = proc_dump(other_args[1:], port)
elif other_args[0] == "restore":
args, callback = proc_restore(other_args[1:], port)
else:
# In this case, the command corresponds to a SQL query in the sql/ folder
sql = prepare_sql(other_args + [
data_dbms_settings["dbname"], exp_dbms_settings["dbname"]
],
folder=os.path.join(PB_HOME, "sql"))
args = [ "psql" ] + \
([ "-p", port ] if port else []) + \
([ "-h", host ] if host else []) + \
[ "-c", sql, "postgres" ]
else:
args = [ "psql" ] + \
([ "-p", port ] if port else []) + \
([ "-h", host ] if host else []) + \
[ data_dbms_settings["dbname"] ]
elif act == "ls":
# See scheduled experiment
sql = "SELECT id, addtime, exp_dir AS exp, exp_file, args " \
"FROM exp " \
"WHERE status = 'scheduled' " \
"ORDER BY addtime, id "
args = ["pb", "db", "exp", sql] #+ other_args
elif act == "top":
# See running experiment
sql = "SELECT id, NOW() - starttime AS time_elapsed, exp_dir AS exp, exp_file, args " \
"FROM exp " \
"WHERE status = 'running' " \
"ORDER BY starttime, id "
args = ["pb", "db", "exp", sql] #+ other_args
elif act == "tail":
exp = get_current_running_experiment()
# if exp is None:
# args = [ "echo", "No running processes." ]
# else:
while True:
if exp is not None:
tail_experiment(exp)
time.sleep(0.5)
exp = get_current_running_experiment()
elif act == "exp":
if other_args[0] == "next":
assert len(other_args) == 1
# Run next scheduled experiment
exp = get_next_scheduled_experiment()
print exp.args
args = [
"pb", "exprun", exp.exp_dir, exp.exp_file
] + exp.args.split(" ") + [
("--expdb", exp.expdb) if exp.expdb is not None else "",
"--poolsize",
str(exp.poolsize), "--repetitions",
str(exp.repetitions), "--logging_level",
str(exp.logging_level), "--verbose" if exp.set_verbose else ""
]
# print args
set_experiment_as_running(exp)
else:
# Schedule experiment
args = [
"/usr/bin/env", "python", "-m",
"src.experiments.run_experiments"
] + other_args
elif act == "exprun":
# Run experiment now
args = [
"/usr/bin/env", "python", "-m", "src.experiments.run_experiments",
"--run-now"
] + other_args
elif act == "bench":
# Run benchmark
benchmark_name = other_args[0]
args = [
"/usr/bin/env", "python", "-m",
"benchmark.{}.run".format(benchmark_name)
] + other_args[1:]
elif act.endswith(".py") and os.path.isfile(act):
# Run Python script
args = [
"/usr/bin/env", "python", "-m",
act.replace("/", ".").replace(".py", "")
] + other_args
else:
raise Exception("Action '{}' not supported.".format(act))
print "RUNNING:"
print " ".join(args)
if len(args):
p = subprocess.Popen(args, env=os.environ, shell=shell)
while p.poll() is None:
try:
p.wait()
except (KeyboardInterrupt, SystemExit) as e:
print(KeyboardInterrupt, SystemExit)
p.send_signal(signal.SIGINT)
# p.wait()
if sshtunnel is not None:
sshtunnel.close()
sshtunnel = None
sys.exit(-1)
except Exception as e:
print Exception
p.send_signal(signal.SIGTERM)
# p.wait()
if sshtunnel is not None:
sshtunnel.close()
sshtunnel = None
raise e
if callback is not None:
callback()
if sshtunnel is not None:
sshtunnel.close()
sshtunnel = None
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import configparser
import os
import subprocess
import shutil
import src.config as config
import sys
def test_color_background(cfg, silent):
faint_exe = cfg.get('test', 'faint_exe')
cmd = [faint_exe, '--run', 'faint_scripts/run-test-color-background.py']
if silent:
cmd.append('--silent')
subprocess.call(cmd)
if __name__ == '__main__':
if config.maybe_create_config():
exit(1)
silent = not 'stay' in sys.argv
cfg = config.read_config()
test_color_background(cfg, silent)
from src.config import read_config
config = read_config()
data_dbms_settings = {
"dbname": config.get("Data DB", "dbname"),
"username": config.get("Data DB", "username"),
"password": config.get("Data DB", "password"),
"host": config.get("Data DB", "host"),
"port": config.get("Data DB", "port"),
}
exp_dbms_settings = {
"dbname": config.get("Experiment DB", "dbname"),
"username": config.get("Experiment DB", "username"),
"password": config.get("Experiment DB", "password"),
"host": config.get("Experiment DB", "host"),
"port": config.get("Experiment DB", "port"),
}
def __main__():
parser = argparse.ArgumentParser()
parser.add_argument('--input',
dest='input_dir',
required=True,
nargs=1,
type=str,
help='Input dir')
parser.add_argument('--output',
dest='output_dir',
required=True,
nargs=1,
type=str,
help='Output dir')
parser.add_argument('--mode',
dest='mode',
required=True,
choices=[constants.MODE_ENCODE, constants.MODE_DECODE])
parser.add_argument('--nolog',
dest='nolog',
required=False,
action='store_true',
default=False)
parser.add_argument('--silent',
'-s',
dest='silent',
required=False,
action='store_true',
default=False)
args = parser.parse_args()
input_dir = args.input_dir[0]
output_dir = args.output_dir[0]
mode = args.mode.upper()
Log.should_log = args.nolog
Log.silent = args.silent
Log.find_next_log_file_path() # Find next path for log file
config.read_config()
files = find_files(mode, input_dir)
Log.log_to_file(True, 'Found {} files\n'.format(len(files)))
if mode == constants.MODE_DECODE:
for ks_file in files:
decoded = decode.decode(ks_file)
if decoded is None:
Log.log_to_file(
True,
'Failed to decode input file \"{}\"?!'.format(ks_file))
continue
write_file(output_dir, ks_file, mode, decoded)
continue
return
elif mode == constants.MODE_ENCODE:
for ks_json_file in files:
encoded = encode.encode(ks_json_file)
write_file(output_dir, ks_json_file, mode, encoded)
continue
return
Log.close_log_file()
return
