def test_combine_dbs(capsys):
common_test_helpers.skip_if_outside_sim_db()
path_db_1 = common_test_helpers.get_test_dir() + "/sim1_test_comb.db"
path_db_2 = common_test_helpers.get_test_dir() + "/sim2_test_comb.db"
path_comb_db = common_test_helpers.get_test_dir() + "/new_comb_sim.db"
if os.path.exists(path_comb_db):
os.remove(path_comb_db)
command_line_tool("sim_db",
["combine_dbs", path_db_1, path_db_2, path_comb_db])
comb_sim_db = helpers.connect_sim_db(path_comb_db)
comb_sim_db_cursor = comb_sim_db.cursor()
db_1 = helpers.connect_sim_db(path_db_1)
db_1_cursor = db_1.cursor()
db_2 = helpers.connect_sim_db(path_db_2)
db_2_cursor = db_2.cursor()
column_names_1, column_types_1 = helpers.get_db_column_names_and_types(
db_1_cursor)
column_names_2, column_types_2 = helpers.get_db_column_names_and_types(
db_2_cursor)
column_names_comb, column_types_comb = helpers.get_db_column_names_and_types( \
comb_sim_db_cursor)
old_run_commands = []
db_1_cursor.execute("SELECT run_command FROM runs WHERE id=1;")
old_run_commands.append(db_1_cursor.fetchall()[0][0])
db_1_cursor.execute("SELECT run_command FROM runs WHERE id=2;")
old_run_commands.append(db_1_cursor.fetchall()[0][0])
db_2_cursor.execute("SELECT run_command FROM runs WHERE id=1;")
old_run_commands.append(db_2_cursor.fetchall()[0][0])
db_2_cursor.execute("SELECT run_command FROM runs WHERE id=2;")
old_run_commands.append(db_2_cursor.fetchall()[0][0])
new_run_commands = []
comb_sim_db_cursor.execute("SELECT run_command FROM runs WHERE id=1;")
new_run_commands.append(comb_sim_db_cursor.fetchall()[0][0])
comb_sim_db_cursor.execute("SELECT run_command FROM runs WHERE id=2;")
new_run_commands.append(comb_sim_db_cursor.fetchall()[0][0])
comb_sim_db_cursor.execute("SELECT run_command FROM runs WHERE id=3;")
new_run_commands.append(comb_sim_db_cursor.fetchall()[0][0])
comb_sim_db_cursor.execute("SELECT run_command FROM runs WHERE id=4;")
new_run_commands.append(comb_sim_db_cursor.fetchall()[0][0])
with capsys.disabled():
print("\nTest combine_dbs...")
comb_sim_db.commit()
comb_sim_db_cursor.close()
comb_sim_db.close()
os.remove(path_comb_db)
for column_name in column_names_1:
assert column_name in column_names_comb
for column_name in column_names_2:
assert column_name in column_names_comb
for (old_command, new_command) in zip(old_run_commands, new_run_commands):
assert old_command == new_command
def add_column(name_command_line_tool="sim_db",
name_command="add_column",
argv=None):
db = helpers.connect_sim_db()
db_cursor = db.cursor()
column_names, column_types = helpers.get_db_column_names_and_types(
db_cursor)
args = command_line_arguments_parser(name_command_line_tool,
name_command).parse_args(argv)
if args.column not in column_names:
if args.type == 'int' or args.type == int:
db_cursor.execute("ALTER TABLE runs ADD COLUMN {0} INTEGER".format(
args.column))
elif args.type == 'float' or args.type == float:
db_cursor.execute("ALTER TABLE runs ADD COLUMN {0} REAL".format(
args.column))
else:
db_cursor.execute("ALTER TABLE runs ADD COLUMN {0} TEXT".format(
args.column))
db.commit()
db_cursor.close()
db.close()
def update_sim(name_command_line_tool="sim_db",
name_command="update_sim",
argv=None):
args = command_line_arguments_parser(name_command_line_tool,
name_command).parse_args(argv)
if args.id == None and args.where == "id > -1":
print("Nothing was updated. --id 'ID' or --where 'CONDITION' must be " \
+ "passed to the program.")
exit(0)
db = helpers.connect_sim_db(args.db_path)
db_cursor = db.cursor()
column_names, column_types = helpers.get_db_column_names_and_types(
db_cursor)
type_dict = dict(zip(column_names, column_types))
condition = args.where
if args.id:
condition = condition + " AND id = {0}".format(args.id)
for column, value in zip(args.columns, args.values):
if type_dict[column] == 'TEXT':
value = "'{0}'".format(value)
db_cursor.execute("UPDATE runs SET {0} = {1} WHERE {2}" \
.format(column, value, condition))
db.commit()
db_cursor.close()
db.close()
def get(name_command_line_tool="sim_db", name_command="get", argv=None):
args = command_line_arguments_parser(name_command_line_tool,
name_command).parse_args(argv)
db = helpers.connect_sim_db()
db_cursor = db.cursor()
if args.id != None:
db_cursor.execute("SELECT {0} FROM runs WHERE id={1}".format(
args.column, args.id))
value = db_cursor.fetchone()[0]
else:
db_cursor.execute("SELECT {0} FROM runs".format(args.column))
if args.n == None:
args.n = 1
value = db_cursor.fetchall()
if len(value) > args.n:
value = value[len(value) - args.n][0]
db.commit()
db_cursor.close()
db.close()
if value == None:
print("Specified simulation does not have a value in the '{0}' column."
.format(args.column))
exit()
value.replace(" ", "\ ")
return value
def delete_sim(name_command_line_tool="sim_db",
name_command="delete_sim",
argv=None):
db = helpers.connect_sim_db()
db_cursor = db.cursor()
args = command_line_arguments_parser(name_command_line_tool,
name_command).parse_args(argv)
if args.all:
args.where = "id > -1"
answer = 'n'
if len(args.id) == 0 and args.where == None:
print("--id 'ID' or --where 'CONDITION' must be passed to the "
"program.")
elif len(args.id) > 0 and not args.no_checks:
print("Do you really want to delete simulations with following ID's:")
for delete_id in args.id:
print(delete_id)
answer = helpers.user_input("? (y/n)")
elif args.where != None and not args.no_checks:
print("Do you really want to delete simulations with following "
"condition:")
print(args.where)
answer = helpers.user_input("? (y/n)")
if (answer == 'y' or answer == 'Y' or answer == 'yes' or answer == 'Yes'
or args.no_checks):
if len(args.id) > 0:
if args.no_checks:
delete_results_dir_params = ['--no_checks', '--id']
else:
delete_results_dir_params = ['--id']
for delete_id in args.id:
delete_results_dir_params.append(str(delete_id))
delete_results_dir.delete_results_dir(
argv=delete_results_dir_params)
for delete_id in args.id:
db_cursor.execute(
"DELETE FROM runs WHERE id = {0}".format(delete_id))
elif args.where:
if args.no_checks:
delete_results_dir.delete_results_dir(
argv=["--no_checks", "--where", args.where])
else:
delete_results_dir.delete_results_dir(
argv=["--where", args.where])
db_cursor.execute("DELETE FROM runs WHERE {0}".format(args.where))
else:
print("No simulations were deleted.")
db.commit()
db_cursor.close()
db.close()
def delete_empty_columns(name_command_line_tool="sim_db",
name_command="delete_empty_columns",
argv=None):
command_line_arguments_parser(name_command_line_tool,
name_command).parse_args(argv)
db = helpers.connect_sim_db()
db_cursor = db.cursor()
column_names, column_types = helpers.get_db_column_names_and_types(
db_cursor)
new_table_dict = OrderedDict()
for column_name, column_type in zip(column_names, column_types):
db_cursor.execute("SELECT {0} FROM runs;".format(column_name))
values = db_cursor.fetchall()
is_empty = True
for value in values:
if value != (None, ):
is_empty = False
break
if not is_empty or (column_name in helpers.default_db_columns):
new_table_dict[column_name] = column_type
new_columns_and_types = ""
new_columns = ""
for column_name in new_table_dict:
new_columns_and_types += column_name + " " + new_table_dict[
column_name] + ", "
new_columns += column_name + ", "
new_columns_and_types = new_columns_and_types[:-2]
new_columns = new_columns[:-2]
assert new_columns_and_types[0:2] == 'id', (
"Name of first column in database is not 'id'.")
new_columns_and_types = new_columns_and_types[0:10] + " PRIMARY KEY" \
+new_columns_and_types[10:] # Correct id type
db_cursor.execute("CREATE TABLE IF NOT EXISTS new_runs ({0});".format(
new_columns_and_types))
db_cursor.execute(
"INSERT INTO new_runs SELECT {0} FROM runs;".format(new_columns))
db_cursor.execute("DROP TABLE runs;")
db_cursor.execute("ALTER TABLE new_runs RENAME TO runs;")
db.commit()
db_cursor.close()
db.close()
def test_add_column_and_delete_empty_columns(capsys):
common_test_helpers.skip_if_outside_sim_db()
with capsys.disabled():
print("\nTest add_column and delete_empty_columns...")
command_line_tool("sim_db",
"add_column --column new_column --type 'TEXT'".split())
db = helpers.connect_sim_db()
db_cursor = db.cursor()
column_names, column_types = helpers.get_db_column_names_and_types(
db_cursor)
assert "new_column" in column_names
command_line_tool("sim_db", ["delete_empty_columns"])
column_names, column_types = helpers.get_db_column_names_and_types(
db_cursor)
assert "new_column" not in column_names
def duplicate_sim(name_command_line_tool="sim_db",
name_command="duplicate_sim",
argv=None):
args = command_line_arguments_parser(name_command_line_tool,
name_command).parse_args(argv)
db = helpers.connect_sim_db()
db_cursor = db.cursor()
db_cursor.execute("CREATE TEMPORARY TABLE tmp AS SELECT * FROM runs WHERE "
"id = {0};".format(args.id))
db_cursor.execute("UPDATE tmp SET id = NULL;")
db_cursor.execute("UPDATE tmp SET status = 'new';")
db_cursor.execute("INSERT INTO runs SELECT * FROM tmp;")
db_id = db_cursor.lastrowid
db_cursor.execute("DROP TABLE tmp")
db.commit()
db_cursor.close()
db.close()
return db_id
def add_empty_sim(store_metadata=False):
"""Add an empty entry into the database and SimDB connected to it.
:param store_metadata: If False, no metadata is added to the database.
Typically used when postprocessing (visualizing) data from a simulation.
:type store_metadata: bool
"""
db = helpers.connect_sim_db()
db_cursor = db.cursor()
default_db_columns = ""
for key in helpers.default_db_columns:
default_db_columns += key + " " + str(
helpers.default_db_columns[key]) + ", "
default_db_columns = default_db_columns[:-2]
db_cursor.execute(
"CREATE TABLE IF NOT EXISTS runs ({0});".format(default_db_columns))
db_cursor.execute("INSERT INTO runs DEFAULT VALUES")
db_id = db_cursor.lastrowid
db.commit()
db_cursor.close()
db.close()
return SimDB(db_id=db_id, store_metadata=store_metadata)
def add_comment(name_command_line_tool="sim_db",
name_command="add_comment",
argv=None):
args = command_line_arguments_parser(name_command_line_tool,
name_command).parse_args(argv)
if (args.comment == None and args.filename == None):
print(
"ERROR: Either '--comment'/'-c' or '--filename'/'-f' need to be provided."
)
exit()
comment = ""
if args.append:
db = helpers.connect_sim_db()
db_cursor = db.cursor()
db_cursor.execute("SELECT comment FROM runs")
fetched = db_cursor.fetchone()
if len(fetched) > 0:
comment = fetched[0]
db.commit()
db_cursor.close()
db.close()
if args.comment != None:
comment += args.comment
if args.filename != None:
with open(args.filename) as comment_file:
comment_content = comment_file.read()
if len(comment_content) > 3000:
warning = "WARNING: Comment limited to the last 3000 characters of the file."
print(warning)
comment_content = warning + '\n'
comment_content += comment_content[(len(comment_content) -
3000):]
comment += comment_content
update_sim.update_sim(argv=[
'--id',
str(args.id), '--columns', 'comment', '--values', comment
])
def run_serial_sims(name_command_line_tool="sim_db",
name_command="run_serial_sims",
argv=None):
"""Run multiple simulations in series."""
args = command_line_arguments_parser(name_command_line_tool,
name_command).parse_args(argv)
db = helpers.connect_sim_db()
db_cursor = db.cursor()
if len(args.id) > 0:
ids = args.id
elif len(args.where) > 0:
db_cursor.execute("SELECT id FROM runs WHERE status = '{0}';"
.format(args.where))
ids = db_cursor.fetchall()
ids = [i[0] for i in ids]
else:
db_cursor.execute("SELECT id FROM runs WHERE status = 'new';")
ids = db_cursor.fetchall()
ids = [i[0] for i in ids]
db.commit()
db_cursor.close()
db.close()
print("Running simulations with the following IDs one after another:")
print(ids)
for id_sim in ids:
db = helpers.connect_sim_db()
db_cursor = db.cursor()
run_command = helpers.get_run_command(db_cursor, id_sim)
db.commit()
db_cursor.close()
db.close()
update_sim.update_sim(argv=[
"--id",
str(id_sim), "--columns", "cpu_info", "--values",
helpers.get_cpu_and_mem_info()
])
update_sim.update_sim(argv=[
"--id",
str(id_sim), "--columns", "status", "--values", "running"
])
print("\nRunning simulation with ID: {0}".format(id_sim))
for command in run_command.split(';'):
process = subprocess.Popen(
command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
if sys.version_info[0] < 3:
for line in iter(process.stdout.readline, ''):
sys.stdout.write(line.decode('ascii', 'replace'))
sys.stdout.flush()
for line in iter(process.stderr.readline, ''):
sys.stdout.write(line.decode('ascii', 'replace'))
sys.stdout.flush()
else:
for line in iter(process.stdout.readline, b''):
sys.stdout.write(line.decode('ascii', 'replace'))
sys.stdout.flush()
for line in iter(process.stderr.readline, b''):
sys.stdout.write(line.decode('ascii', 'replace'))
sys.stdout.flush()
update_sim.update_sim(argv=[
"--id",
str(id_sim), "--columns", "status", "--values", "finished"
])
def add_sim(name_command_line_tool="sim_db", name_command="add", argv=None):
db = helpers.connect_sim_db()
args = command_line_arguments_parser(name_command_line_tool,
name_command).parse_args(argv)
sim_params_filename = args.filename
if sim_params_filename == None:
sim_params_filename = search_for_parameter_file_matching_settings()
if sim_params_filename == None:
print("No parameter files in the current directory matches the "
"ones under 'Parameter filenames'\nin settings.txt.\n"
"\nAdd the '--filename' flag to specify the filename of "
"the parameter file.")
exit(1)
elif (sim_params_filename[0:5] == 'root/'):
sim_params_filename = os.path.join(
os.path.join(helpers.get_dot_sim_db_dir_path(), os.pardir),
sim_params_filename[5:])
elif (sim_params_filename[0:6] == '"root/'):
sim_params_filename = '"' + os.path.join(
os.path.join(helpers.get_dot_sim_db_dir_path(), os.pardir),
sim_params_filename[6:])
try:
sim_params_file = open(sim_params_filename, 'r')
except:
print("Could NOT open {0}.".format(sim_params_filename))
exit(1)
sim_params_file_lines = sim_params_file.readlines()
sim_params_file.close()
sim_params_file_lines = add_included_parameter_files(sim_params_file_lines)
sim_params_file_lines = replace_aliases(sim_params_file_lines)
db_cursor = db.cursor()
default_db_columns = ""
for key in helpers.default_db_columns:
default_db_columns += key + " " + str(
helpers.default_db_columns[key]) + ", "
default_db_columns = default_db_columns[:-2]
db_cursor.execute(
"CREATE TABLE IF NOT EXISTS runs ({0});".format(default_db_columns))
column_names, column_types = helpers.get_db_column_names_and_types(
db_cursor)
initial_parameters = []
last_row_id = None
for i, line in enumerate(sim_params_file_lines):
if len(line.split(':')) > 1:
param_name, param_type, value = split_parameter_line(line, i)
initial_parameters.append(param_name)
if param_name == 'run_command':
value = make_path_relative_to_root(value, sim_params_filename)
try:
row_index = column_names.index(param_name)
except ValueError:
row_index = None
if row_index is None:
add_new_column(db_cursor, i, param_type, param_name, value,
column_names, column_types)
else:
check_type_matches(param_type, column_types[row_index], value,
i)
value = standardize_value(value, param_type)
if len(value) > 0:
last_row_id = insert_value(db_cursor, param_name, last_row_id,
value)
initial_parameters = standardize_value(str(initial_parameters),
"string array")
last_row_id = insert_value(db_cursor, 'initial_parameters', last_row_id,
initial_parameters)
db.commit()
db_cursor.close()
db.close()
return last_row_id
def delete_results_dir(name_command_line_tool="sim_db",
name_command="delete_results_dir",
argv=None):
db = helpers.connect_sim_db()
db_cursor = db.cursor()
args = command_line_arguments_parser(name_command_line_tool,
name_command).parse_args(argv)
if (len(args.id) == 0 and args.where == None
and args.not_in_db_but_in_dir == None):
print(
"No 'results_dir' was deleted. --id 'ID' or --where 'CONDITION' " +
"must be passed to the program.")
exit(0)
results_dirs = []
for delete_id in args.id:
db_cursor.execute(
"SELECT results_dir FROM runs WHERE id = {0}".format(delete_id))
results_dir = db_cursor.fetchone()
if results_dir != None and results_dir[0] != None:
results_dirs.append(results_dir[0])
if args.where:
db_cursor.execute("SELECT results_dir FROM runs WHERE {0}".format(
args.where))
for selected_output in db_cursor.fetchall():
if selected_output[0] != None:
results_dirs.append(selected_output[0])
if args.not_in_db_but_in_dir != None:
db_cursor.execute("SELECT results_dir FROM runs")
results_dir = db_cursor.fetchone()
while results_dir != None and results_dir[0] != None:
results_dirs.append(results_dir[0])
results_dir = db_cursor.fetchone()
db.commit()
db_cursor.close()
db.close()
if args.not_in_db_but_in_dir != None:
if not os.path.isabs(args.not_in_db_but_in_dir):
if args.not_in_db_but_in_dir[0] == '.':
args.not_in_db_but_in_dir = args.not_in_db_but_in_dir[2:]
if args.not_in_db_but_in_dir[-1] == '/':
args.not_in_db_but_in_dir = args.not_in_db_but_in_dir[:-1]
args.not_in_db_but_in_dir = os.path.join(os.getcwd(),
args.not_in_db_but_in_dir)
for path in os.listdir(args.not_in_db_but_in_dir):
path = os.path.join(args.not_in_db_but_in_dir, path)
if os.path.isdir(path) and (path not in results_dirs):
print("\nDo you really want to delete:")
print(path)
answer = helpers.user_input("? (y/n)")
if (answer == 'y' or answer == 'Y' or answer == 'yes'
or answer == 'Yes' or args.no_checks):
shutil.rmtree(path)
else:
print(path)
print("was NOT deleted.")
elif len(results_dirs) > 0:
answer = 'no'
if not args.no_checks:
print("Do you really want to delete the following directories and "
"everything in them:")
for results_dir in results_dirs:
print(results_dir)
answer = helpers.user_input("? (y/n)")
if (answer == 'y' or answer == 'Y' or answer == 'yes'
or answer == 'Yes' or args.no_checks):
for results_dir in results_dirs:
try:
shutil.rmtree(results_dir)
except Exception as e:
if e.errno == errno.ENOENT:
print("Results directory NOT found: {0}".format(
results_dir))
else:
raise e
elif not args.no_checks:
print("No results deleted.")
elif not args.no_checks:
print("No 'results_dir' to delete.")
def submit_sim(name_command_line_tool="sim_db",
name_command="submit_sim",
argv=None):
args = command_line_arguments_parser(name_command_line_tool,
name_command).parse_args(argv)
ids = args.id
db = helpers.connect_sim_db()
db_cursor = db.cursor()
job_script_name = None
job_id = None
if ids == None:
db_cursor.execute("SELECT id FROM runs WHERE status = 'new';")
ids = db_cursor.fetchall()
ids = [i[0] for i in ids]
answer = None
while (not args.no_confirmation and answer != 'y' and answer != 'Y'
and answer != 'yes' and answer != 'Yes'):
answer = helpers.user_input(
"Would you like to submit simulations "
"with the following ID's: {0}? (y/n) ".format(ids))
if (answer == 'n' or answer == 'N' or answer == 'no'
or answer == 'No'):
db.commit()
db_cursor.close()
db.close()
return (job_script_name, job_id)
elif not args.allow_reruns:
for i in ids:
db_cursor.execute(
"SELECT status FROM runs WHERE id = {0};".format(i))
status = db_cursor.fetchone()[0]
if status != "new":
print("No simulations was submitted.\nStatus of simulation "
"with 'ID' = {0} is {1}.\nEither:\n- Add "
"'--allow_reruns' flag to allow it to run.\n- Update "
"status to 'new'.".format(args.id, status))
exit()
if args.add_unique_results_dir:
for i in ids:
unique_results_dir = helpers.unique_results_dir(db_cursor, i)
os.mkdir(unique_results_dir)
update_sim.update_sim(argv=[
"--id",
str(i), "--columns", "results_dir", "--values",
unique_results_dir
])
for i, id_submit in enumerate(ids):
name_job_script = make_job_script(db_cursor, i, args, id_submit)
if not args.do_not_submit_job_script:
job_id = submit_job_script(name_job_script)
db_cursor.execute("UPDATE runs SET status='submitted' WHERE id={0}" \
.format(id_submit))
db_cursor.execute("UPDATE runs SET job_id={0} WHERE id={1}" \
.format(job_id, id_submit))
db.commit()
db_cursor.close()
db.close()
print("Job ID: {0}".format(job_id))
return (name_job_script, job_id)
def print_sim(name_command_line_tool="sim_db",
name_command="print_sim",
argv=None):
if argv == None:
argv = ['-p', 'default']
elif '--columns' not in argv and '-c' not in argv:
if len(argv) > 0 and argv[0][0] != '-':
# To print correct error message even though '-p default' is added.
command_line_arguments_parser(name_command_line_tool,
name_command).parse_args(argv)
argv = ['-p', 'default'] + argv
argv_p_replaced = []
i = 0
while i < len(argv):
if argv[i] != '-p':
argv_p_replaced.append(argv[i])
else:
while i + 1 < len(argv) and argv[i + 1][0] != '-':
i += 1
print_config_key = argv[i]
print_config = get_personalized_print_config(print_config_key)
if print_config == None:
print(
"No personalized print configuration with key string "
"'{0}' is found in settings.".format(print_config_key))
exit(1)
argv_p_replaced = argv_p_replaced + shlex.split(print_config)
i += 1
argv = argv_p_replaced
args = command_line_arguments_parser(name_command_line_tool,
name_command).parse_args(argv)
db = helpers.connect_sim_db()
db_cursor = db.cursor()
column_names, column_types = helpers.get_db_column_names_and_types(
db_cursor)
type_dict = dict(zip(column_names, column_types))
selected_output, column_names = select_command(name_command_line_tool,
name_command, db_cursor,
args, column_names)
if args.diff:
selected_output, column_names = remove_columns_with_only_same_values(
selected_output, column_names)
if not args.vertically:
print_selected_parameters(selected_output, column_names,
args.no_headers, args.max_width,
args.first_line)
else:
print_selected_parameters_vertically(selected_output, column_names,
args.no_headers, args.max_width,
args.first_line)
if args.column_names:
print("")
print("Column names and types in database:")
for col_name in column_names:
col_type = type_dict[col_name]
print("{0}, {1}".format(col_name, col_type))
db.commit()
db_cursor.close()
db.close()
def combine_dbs(name_command_line_tool="sim_db",
name_command="combine_dbs",
argv=None):
args = command_line_arguments_parser(name_command_line_tool,
name_command).parse_args(argv)
db_1 = helpers.connect_sim_db(args.path_db_1)
db_1_cursor = db_1.cursor()
db_2 = helpers.connect_sim_db(args.path_db_2)
db_2_cursor = db_2.cursor()
(columns_db_1,
types_db_1) = helpers.get_db_column_names_and_types(db_1_cursor)
(columns_db_2,
types_db_2) = helpers.get_db_column_names_and_types(db_2_cursor)
column_type_dict = OrderedDict(
[(col, typ) for col, typ in zip(columns_db_1, types_db_1)] +
[(col, typ) for col, typ in zip(columns_db_2, types_db_2)])
new_db = helpers.connect_sim_db(args.name_new_db)
new_db_cursor = new_db.cursor()
new_db_columns_string = ""
for col in column_type_dict:
new_db_columns_string += col + " " + column_type_dict[col] + ", "
new_db_columns_string = new_db_columns_string[:-2]
new_db_columns_string = new_db_columns_string.replace(
"id INTEGER", "id INTEGER PRIMARY KEY", 1)
new_db_cursor.execute(
"CREATE TABLE runs ({0});".format(new_db_columns_string))
db_1_cursor.execute("SELECT * FROM runs")
for row in db_1_cursor.fetchall():
column_tuple = ()
value_tuple = ()
for column, value in zip(columns_db_1, row):
column = helpers.if_unicode_convert_to_str(column)
value = helpers.if_unicode_convert_to_str(value)
if value != None and column != 'id':
column_tuple += (column, )
value_tuple += (value, )
new_db_cursor.execute("INSERT INTO runs {0} VALUES {1}".format(
column_tuple, value_tuple))
db_2_cursor.execute("SELECT * FROM runs")
for row in db_2_cursor.fetchall():
column_tuple = ()
value_tuple = ()
for column, value in zip(columns_db_2, row):
column = helpers.if_unicode_convert_to_str(column)
value = helpers.if_unicode_convert_to_str(value)
if value != None and column != 'id':
column_tuple += (column, )
value_tuple += (value, )
new_db_cursor.execute("INSERT INTO runs {0} VALUES {1}".format(
column_tuple, value_tuple))
db_1.commit()
db_2.commit()
new_db.commit()
db_1_cursor.close()
db_2_cursor.close()
new_db_cursor.close()
db_1.close()
db_2.close()
new_db.close()
def __init__(self,
store_metadata=True,
db_id=None,
rank=None,
only_write_on_rank=0):
"""Connect to the **sim_db** database.
:param store_metadata: If False, no metadata is added to the database.
Typically used when postprocessing (visualizing) data from a
simulation.
:type store_metadata: bool
:param db_id: ID number of the simulation parameters in the **sim_db**
database. If it is 'None', then it is read from the argument passed
to the program after option '--id'.
:type db_id: int
:param rank: Number identifing the calling process and/or thread.
(Typically the MPI or OpenMP rank.) If provided, only the 'rank'
matching 'only_write_on_rank' will write to the database to avoid
too much concurrent writing to the database. Single process and
threaded programs may ignore this, while
multithreading/multiprocessing programs need to provide it.
:type rank: int
:param only_write_on_rank: Number identifing the only process/thread
that will write to the database. Only used if 'rank' is provided.
:type only_write_on_rank: int
"""
self.rank = rank
self.only_write_on_rank = only_write_on_rank
self.start_time = time.time()
self.store_metadata = store_metadata
self.id, self.path_proj_root = self.__read_from_command_line_arguments(
db_id)
self.db = helpers.connect_sim_db()
self.db_cursor = self.db.cursor()
self.column_names = []
self.column_types = []
if (self.store_metadata and
(self.rank == None or self.rank == self.only_write_on_rank)):
self.write('status', 'running')
self.write('time_started', self.__get_date_and_time_as_string())
if (self.store_metadata and self.__is_a_git_project() and
(self.rank == None or self.rank == self.only_write_on_rank)):
proc = subprocess.Popen(
['cd "{0}"; git rev-parse HEAD'.format(self.path_proj_root)],
stdout=subprocess.PIPE,
stderr=open(os.devnull, 'w'),
shell=True)
(out, err) = proc.communicate()
self.write(column="git_hash", value=out.decode('ascii', 'replace'))
proc = subprocess.Popen([
'cd "{0}"; git log -n 1 --format=%B HEAD'.format(
self.path_proj_root)
],
stdout=subprocess.PIPE,
stderr=open(os.devnull, 'w'),
shell=True)
(out, err) = proc.communicate()
self.write(column="commit_message",
value=out.decode('ascii', 'replace'))
proc = subprocess.Popen(
['cd "{0}"; git diff HEAD --stat'.format(self.path_proj_root)],
stdout=subprocess.PIPE,
stderr=open(os.devnull, 'w'),
shell=True)
(out, err) = proc.communicate()
self.write(column="git_diff_stat",
value=out.decode('ascii', 'replace'))
proc = subprocess.Popen(
['cd "{0}"; git diff HEAD'.format(self.path_proj_root)],
stdout=subprocess.PIPE,
stderr=open(os.devnull, 'w'),
shell=True)
(out, err) = proc.communicate()
out = out.decode('ascii', 'replace')
if len(out) > 3000:
warning = "WARNING: Diff limited to first 3000 characters.\n"
out = warning + '\n' + out[0:3000] + '\n\n' + warning
self.write(column="git_diff", value=out)
def run_sim(name_command_line_tool="sim_db",
name_command="run_sim",
argv=None):
"""Run simulation with parameters with ID passed or the highest ID."""
args = command_line_arguments_parser(name_command_line_tool,
name_command).parse_args(argv)
db = helpers.connect_sim_db()
db_cursor = db.cursor()
if args.id == None:
db_cursor.execute("SELECT id FROM runs WHERE status = 'new';")
ids = db_cursor.fetchall()
ids = [i[0] for i in ids]
args.id = max(ids)
print("Start simulation with ID {0}.".format(args.id))
elif not args.allow_reruns:
db_cursor.execute("SELECT status FROM runs WHERE id = {0};".format(
args.id))
status = db_cursor.fetchone()
if status == None:
print("There exists no entry in the database with id = {0} and a "
"'run_command'.".format(args.id))
exit(1)
status = status[0]
if status != "new":
print(
"Status of simulation with 'ID' = {0} is {1}.\nEither:\n"
"- Add '--allow_reruns' flag to allow it to run.\n- Update "
"status to 'new'.\n- Duplicate it, delete the old and run it "
"with: 'ddr' / 'duplicate_delete_and_run'".format(
args.id, status))
exit(1)
if args.add_unique_results_dir:
unique_results_dir = helpers.unique_results_dir(db_cursor, args.id)
print(unique_results_dir)
os.mkdir(unique_results_dir)
update_sim.update_sim(argv=[
"--id",
str(args.id), "--columns", "results_dir", "--values",
unique_results_dir
])
run_command = helpers.get_run_command(db_cursor, args.id, args.n)
db.commit()
db_cursor.close()
db.close()
update_sim.update_sim(argv=[
"--id",
str(args.id), "--columns", "cpu_info", "--values",
helpers.get_cpu_and_mem_info()
])
update_sim.update_sim(argv=[
"--id",
str(args.id), "--columns", "status", "--values", "running"
])
for command in run_command.split(';'):
process = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
if sys.version_info[0] < 3:
for line in iter(process.stdout.readline, ''):
sys.stdout.write(line.decode('ascii', 'replace'))
sys.stdout.flush()
for line in iter(process.stderr.readline, ''):
sys.stdout.write(line.decode('ascii', 'replace'))
sys.stdout.flush()
else:
for line in iter(process.stdout.readline, b''):
sys.stdout.write(line.decode('ascii', 'replace'))
sys.stdout.flush()
for line in iter(process.stderr.readline, b''):
sys.stdout.write(line.decode('ascii', 'replace'))
sys.stdout.flush()
update_sim.update_sim(argv=[
"--id",
str(args.id), "--columns", "status", "--values", "finished"
])
def add_range_sim(name_command_line_tool="sim_db",
name_command="add_range_sim",
argv=None):
args = command_line_arguments_parser(name_command_line_tool,
name_command).parse_args(argv)
# Check command line arguments
n_cols = len(args.columns)
if ((len(args.lin_steps) != n_cols or len(args.exp_steps) != 0)
and (len(args.lin_steps) != 0 or len(args.exp_steps) != n_cols)
and
(len(args.lin_steps) != n_cols or len(args.exp_steps) != n_cols)):
print("ERROR: Either '--lin_steps', '--exp_steps' or both have to be provided "
"and they have to be same length as '--columns'.")
exit()
elif ((len(args.end_steps) != n_cols or len(args.n_steps) != 0)
and (len(args.end_steps) != 0 or len(args.n_steps) != n_cols)
and (len(args.end_steps) != n_cols or len(args.n_steps) != n_cols)):
print("ERROR: Either '--end_steps', '--n_steps' or both have to be provided "
"and they have to be same length as '--columns'.")
exit()
ids_added = []
if args.filename == None:
ids_added.append(add_sim.add_sim())
else:
ids_added.append(add_sim.add_sim(argv=['--filename', args.filename]))
# Get start value of each column that varies.
db = helpers.connect_sim_db()
db_cursor = db.cursor()
start_values = []
for column in args.columns:
db_cursor.execute("SELECT {0} FROM runs WHERE id = {1}".format(
column, ids_added[0]))
start_values.append(db_cursor.fetchone()[0])
db.commit()
db_cursor.close()
db.close()
# Make table of the cartisian product.
if len(args.lin_steps) == 0:
args.lin_steps = [0 for i in range(n_cols)]
if len(args.exp_steps) == 0:
args.exp_steps = [1 for i in range(n_cols)]
column_range = [[] for i in range(n_cols)]
for i in range(n_cols):
column_range[i].append(start_values[i])
if len(args.end_steps) == 0:
for j in range(args.n_steps[i]):
column_range[i].append(column_range[i][-1] *
args.exp_steps[i] + args.lin_steps[i])
elif len(args.lin_steps) == 0:
j = 0
while (start_values[i] != args.end_steps[i]
and (start_values[i] < args.end_steps[i]) == (
column_range[i][j] < args.end_steps[i])):
column_range[i].append(column_range[i][-1] *
args.exp_steps[i] + args.lin_steps[i])
j += 1
else:
j = 0
while (j < args.n_steps[i]
or (start_values[i] != args.end_steps[i] and
(start_values[i] < args.end_steps[i]) ==
(column_range[i][j] < args.end_steps[i]))):
column_range[i].append(column_range[i][-1] *
args.exp_steps[i] + args.lin_steps[i])
j += 1
cartisian_product = [[] for i in range(n_cols)]
for i in range(n_cols - 1, -1, -1):
cartisian_product = add_new_column_in_cartisian_product(
cartisian_product, column_range[i], i)
# Add all simulations.
for i in range(1, len(cartisian_product[0])):
if args.filename == None:
ids_added.append(add_sim.add_sim())
else:
ids_added.append(
add_sim.add_sim(argv=['--filename', args.filename]))
update_params = ['--id', str(ids_added[-1]), '--columns']
update_params.extend(args.columns)
update_params.append('--values')
for j in range(n_cols):
update_params.append(str(cartisian_product[j][i]))
update_sim.update_sim(argv=update_params)
return ids_added
def extract_params(name_command_line_tool="sim_db",
name_command="extract_params",
argv=None):
args = command_line_arguments_parser(name_command_line_tool,
name_command).parse_args(argv)
is_printing_parameters = True
if args.default_file:
param_files = helpers.Settings().read('parameter_files')
if len(param_files) == 0:
print("ERROR: No '--filename' provided and no 'Parameters files' "
"in settings.txt.")
exit()
else:
filename = param_files[0]
if os.path.exists(filename):
answer = helpers.user_input(
"Would you like to overwrite '{0}'? (y/n)".format(
filename))
if (answer != 'y' and answer != 'Y' and answer != 'yes'
and answer != 'Yes'):
exit()
print("Extracts parameters to '{0}'.".format(filename))
is_printing_parameters = False
elif args.filename != None:
filename = args.filename
is_printing_parameters = False
if not is_printing_parameters:
params_file = open(filename, 'w')
db = helpers.connect_sim_db()
db_cursor = db.cursor()
db_cursor.execute("SELECT * FROM runs WHERE id={0}".format(args.id))
extracted_row = db_cursor.fetchall()
column_names, column_types = helpers.get_db_column_names_and_types(
db_cursor)
db_cursor.execute("SELECT initial_parameters FROM runs WHERE id={0}"
.format(args.id))
initial_parameters = db_cursor.fetchone()[0]
initial_parameters, correct_type = helpers.convert_text_to_correct_type(
initial_parameters, 'string array')
for col_name, col_type, value in zip(column_names, column_types,
extracted_row[0]):
if (value != None
and (col_name in initial_parameters
or (args.also_results and col_name not in no_extract_columns)
or args.all)):
line = col_name
param_type = get_param_type_as_string(col_type, value)
line += " ({0}): ".format(param_type)
if param_type[-3:] == 'ray':
value = '[' + value.split('[')[1]
line += str(value).replace(':', ';') + '\n'
if is_printing_parameters:
print(line)
else:
params_file.write(line)
if not is_printing_parameters:
params_file.close()
db.commit()
db_cursor.close()
db.close()