def test_combine_dbs(capsys): common_test_helpers.skip_if_outside_sim_db() path_db_1 = common_test_helpers.get_test_dir() + "/sim1_test_comb.db" path_db_2 = common_test_helpers.get_test_dir() + "/sim2_test_comb.db" path_comb_db = common_test_helpers.get_test_dir() + "/new_comb_sim.db" if os.path.exists(path_comb_db): os.remove(path_comb_db) command_line_tool("sim_db", ["combine_dbs", path_db_1, path_db_2, path_comb_db]) comb_sim_db = helpers.connect_sim_db(path_comb_db) comb_sim_db_cursor = comb_sim_db.cursor() db_1 = helpers.connect_sim_db(path_db_1) db_1_cursor = db_1.cursor() db_2 = helpers.connect_sim_db(path_db_2) db_2_cursor = db_2.cursor() column_names_1, column_types_1 = helpers.get_db_column_names_and_types( db_1_cursor) column_names_2, column_types_2 = helpers.get_db_column_names_and_types( db_2_cursor) column_names_comb, column_types_comb = helpers.get_db_column_names_and_types( \ comb_sim_db_cursor) old_run_commands = [] db_1_cursor.execute("SELECT run_command FROM runs WHERE id=1;") old_run_commands.append(db_1_cursor.fetchall()[0][0]) db_1_cursor.execute("SELECT run_command FROM runs WHERE id=2;") old_run_commands.append(db_1_cursor.fetchall()[0][0]) db_2_cursor.execute("SELECT run_command FROM runs WHERE id=1;") old_run_commands.append(db_2_cursor.fetchall()[0][0]) db_2_cursor.execute("SELECT run_command FROM runs WHERE id=2;") old_run_commands.append(db_2_cursor.fetchall()[0][0]) new_run_commands = [] comb_sim_db_cursor.execute("SELECT run_command FROM runs WHERE id=1;") new_run_commands.append(comb_sim_db_cursor.fetchall()[0][0]) comb_sim_db_cursor.execute("SELECT run_command FROM runs WHERE id=2;") new_run_commands.append(comb_sim_db_cursor.fetchall()[0][0]) comb_sim_db_cursor.execute("SELECT run_command FROM runs WHERE id=3;") new_run_commands.append(comb_sim_db_cursor.fetchall()[0][0]) comb_sim_db_cursor.execute("SELECT run_command FROM runs WHERE id=4;") new_run_commands.append(comb_sim_db_cursor.fetchall()[0][0]) with capsys.disabled(): print("\nTest combine_dbs...") comb_sim_db.commit() comb_sim_db_cursor.close() comb_sim_db.close() os.remove(path_comb_db) for column_name in column_names_1: assert column_name in column_names_comb for column_name in column_names_2: assert column_name in column_names_comb for (old_command, new_command) in zip(old_run_commands, new_run_commands): assert old_command == new_command
def add_column(name_command_line_tool="sim_db", name_command="add_column", argv=None): db = helpers.connect_sim_db() db_cursor = db.cursor() column_names, column_types = helpers.get_db_column_names_and_types( db_cursor) args = command_line_arguments_parser(name_command_line_tool, name_command).parse_args(argv) if args.column not in column_names: if args.type == 'int' or args.type == int: db_cursor.execute("ALTER TABLE runs ADD COLUMN {0} INTEGER".format( args.column)) elif args.type == 'float' or args.type == float: db_cursor.execute("ALTER TABLE runs ADD COLUMN {0} REAL".format( args.column)) else: db_cursor.execute("ALTER TABLE runs ADD COLUMN {0} TEXT".format( args.column)) db.commit() db_cursor.close() db.close()
def update_sim(name_command_line_tool="sim_db", name_command="update_sim", argv=None): args = command_line_arguments_parser(name_command_line_tool, name_command).parse_args(argv) if args.id == None and args.where == "id > -1": print("Nothing was updated. --id 'ID' or --where 'CONDITION' must be " \ + "passed to the program.") exit(0) db = helpers.connect_sim_db(args.db_path) db_cursor = db.cursor() column_names, column_types = helpers.get_db_column_names_and_types( db_cursor) type_dict = dict(zip(column_names, column_types)) condition = args.where if args.id: condition = condition + " AND id = {0}".format(args.id) for column, value in zip(args.columns, args.values): if type_dict[column] == 'TEXT': value = "'{0}'".format(value) db_cursor.execute("UPDATE runs SET {0} = {1} WHERE {2}" \ .format(column, value, condition)) db.commit() db_cursor.close() db.close()
def get(name_command_line_tool="sim_db", name_command="get", argv=None): args = command_line_arguments_parser(name_command_line_tool, name_command).parse_args(argv) db = helpers.connect_sim_db() db_cursor = db.cursor() if args.id != None: db_cursor.execute("SELECT {0} FROM runs WHERE id={1}".format( args.column, args.id)) value = db_cursor.fetchone()[0] else: db_cursor.execute("SELECT {0} FROM runs".format(args.column)) if args.n == None: args.n = 1 value = db_cursor.fetchall() if len(value) > args.n: value = value[len(value) - args.n][0] db.commit() db_cursor.close() db.close() if value == None: print("Specified simulation does not have a value in the '{0}' column." .format(args.column)) exit() value.replace(" ", "\ ") return value
def delete_sim(name_command_line_tool="sim_db", name_command="delete_sim", argv=None): db = helpers.connect_sim_db() db_cursor = db.cursor() args = command_line_arguments_parser(name_command_line_tool, name_command).parse_args(argv) if args.all: args.where = "id > -1" answer = 'n' if len(args.id) == 0 and args.where == None: print("--id 'ID' or --where 'CONDITION' must be passed to the " "program.") elif len(args.id) > 0 and not args.no_checks: print("Do you really want to delete simulations with following ID's:") for delete_id in args.id: print(delete_id) answer = helpers.user_input("? (y/n)") elif args.where != None and not args.no_checks: print("Do you really want to delete simulations with following " "condition:") print(args.where) answer = helpers.user_input("? (y/n)") if (answer == 'y' or answer == 'Y' or answer == 'yes' or answer == 'Yes' or args.no_checks): if len(args.id) > 0: if args.no_checks: delete_results_dir_params = ['--no_checks', '--id'] else: delete_results_dir_params = ['--id'] for delete_id in args.id: delete_results_dir_params.append(str(delete_id)) delete_results_dir.delete_results_dir( argv=delete_results_dir_params) for delete_id in args.id: db_cursor.execute( "DELETE FROM runs WHERE id = {0}".format(delete_id)) elif args.where: if args.no_checks: delete_results_dir.delete_results_dir( argv=["--no_checks", "--where", args.where]) else: delete_results_dir.delete_results_dir( argv=["--where", args.where]) db_cursor.execute("DELETE FROM runs WHERE {0}".format(args.where)) else: print("No simulations were deleted.") db.commit() db_cursor.close() db.close()
def delete_empty_columns(name_command_line_tool="sim_db", name_command="delete_empty_columns", argv=None): command_line_arguments_parser(name_command_line_tool, name_command).parse_args(argv) db = helpers.connect_sim_db() db_cursor = db.cursor() column_names, column_types = helpers.get_db_column_names_and_types( db_cursor) new_table_dict = OrderedDict() for column_name, column_type in zip(column_names, column_types): db_cursor.execute("SELECT {0} FROM runs;".format(column_name)) values = db_cursor.fetchall() is_empty = True for value in values: if value != (None, ): is_empty = False break if not is_empty or (column_name in helpers.default_db_columns): new_table_dict[column_name] = column_type new_columns_and_types = "" new_columns = "" for column_name in new_table_dict: new_columns_and_types += column_name + " " + new_table_dict[ column_name] + ", " new_columns += column_name + ", " new_columns_and_types = new_columns_and_types[:-2] new_columns = new_columns[:-2] assert new_columns_and_types[0:2] == 'id', ( "Name of first column in database is not 'id'.") new_columns_and_types = new_columns_and_types[0:10] + " PRIMARY KEY" \ +new_columns_and_types[10:] # Correct id type db_cursor.execute("CREATE TABLE IF NOT EXISTS new_runs ({0});".format( new_columns_and_types)) db_cursor.execute( "INSERT INTO new_runs SELECT {0} FROM runs;".format(new_columns)) db_cursor.execute("DROP TABLE runs;") db_cursor.execute("ALTER TABLE new_runs RENAME TO runs;") db.commit() db_cursor.close() db.close()
def test_add_column_and_delete_empty_columns(capsys): common_test_helpers.skip_if_outside_sim_db() with capsys.disabled(): print("\nTest add_column and delete_empty_columns...") command_line_tool("sim_db", "add_column --column new_column --type 'TEXT'".split()) db = helpers.connect_sim_db() db_cursor = db.cursor() column_names, column_types = helpers.get_db_column_names_and_types( db_cursor) assert "new_column" in column_names command_line_tool("sim_db", ["delete_empty_columns"]) column_names, column_types = helpers.get_db_column_names_and_types( db_cursor) assert "new_column" not in column_names
def duplicate_sim(name_command_line_tool="sim_db", name_command="duplicate_sim", argv=None): args = command_line_arguments_parser(name_command_line_tool, name_command).parse_args(argv) db = helpers.connect_sim_db() db_cursor = db.cursor() db_cursor.execute("CREATE TEMPORARY TABLE tmp AS SELECT * FROM runs WHERE " "id = {0};".format(args.id)) db_cursor.execute("UPDATE tmp SET id = NULL;") db_cursor.execute("UPDATE tmp SET status = 'new';") db_cursor.execute("INSERT INTO runs SELECT * FROM tmp;") db_id = db_cursor.lastrowid db_cursor.execute("DROP TABLE tmp") db.commit() db_cursor.close() db.close() return db_id
def add_empty_sim(store_metadata=False): """Add an empty entry into the database and SimDB connected to it. :param store_metadata: If False, no metadata is added to the database. Typically used when postprocessing (visualizing) data from a simulation. :type store_metadata: bool """ db = helpers.connect_sim_db() db_cursor = db.cursor() default_db_columns = "" for key in helpers.default_db_columns: default_db_columns += key + " " + str( helpers.default_db_columns[key]) + ", " default_db_columns = default_db_columns[:-2] db_cursor.execute( "CREATE TABLE IF NOT EXISTS runs ({0});".format(default_db_columns)) db_cursor.execute("INSERT INTO runs DEFAULT VALUES") db_id = db_cursor.lastrowid db.commit() db_cursor.close() db.close() return SimDB(db_id=db_id, store_metadata=store_metadata)
def add_comment(name_command_line_tool="sim_db", name_command="add_comment", argv=None): args = command_line_arguments_parser(name_command_line_tool, name_command).parse_args(argv) if (args.comment == None and args.filename == None): print( "ERROR: Either '--comment'/'-c' or '--filename'/'-f' need to be provided." ) exit() comment = "" if args.append: db = helpers.connect_sim_db() db_cursor = db.cursor() db_cursor.execute("SELECT comment FROM runs") fetched = db_cursor.fetchone() if len(fetched) > 0: comment = fetched[0] db.commit() db_cursor.close() db.close() if args.comment != None: comment += args.comment if args.filename != None: with open(args.filename) as comment_file: comment_content = comment_file.read() if len(comment_content) > 3000: warning = "WARNING: Comment limited to the last 3000 characters of the file." print(warning) comment_content = warning + '\n' comment_content += comment_content[(len(comment_content) - 3000):] comment += comment_content update_sim.update_sim(argv=[ '--id', str(args.id), '--columns', 'comment', '--values', comment ])
def run_serial_sims(name_command_line_tool="sim_db", name_command="run_serial_sims", argv=None): """Run multiple simulations in series.""" args = command_line_arguments_parser(name_command_line_tool, name_command).parse_args(argv) db = helpers.connect_sim_db() db_cursor = db.cursor() if len(args.id) > 0: ids = args.id elif len(args.where) > 0: db_cursor.execute("SELECT id FROM runs WHERE status = '{0}';" .format(args.where)) ids = db_cursor.fetchall() ids = [i[0] for i in ids] else: db_cursor.execute("SELECT id FROM runs WHERE status = 'new';") ids = db_cursor.fetchall() ids = [i[0] for i in ids] db.commit() db_cursor.close() db.close() print("Running simulations with the following IDs one after another:") print(ids) for id_sim in ids: db = helpers.connect_sim_db() db_cursor = db.cursor() run_command = helpers.get_run_command(db_cursor, id_sim) db.commit() db_cursor.close() db.close() update_sim.update_sim(argv=[ "--id", str(id_sim), "--columns", "cpu_info", "--values", helpers.get_cpu_and_mem_info() ]) update_sim.update_sim(argv=[ "--id", str(id_sim), "--columns", "status", "--values", "running" ]) print("\nRunning simulation with ID: {0}".format(id_sim)) for command in run_command.split(';'): process = subprocess.Popen( command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) if sys.version_info[0] < 3: for line in iter(process.stdout.readline, ''): sys.stdout.write(line.decode('ascii', 'replace')) sys.stdout.flush() for line in iter(process.stderr.readline, ''): sys.stdout.write(line.decode('ascii', 'replace')) sys.stdout.flush() else: for line in iter(process.stdout.readline, b''): sys.stdout.write(line.decode('ascii', 'replace')) sys.stdout.flush() for line in iter(process.stderr.readline, b''): sys.stdout.write(line.decode('ascii', 'replace')) sys.stdout.flush() update_sim.update_sim(argv=[ "--id", str(id_sim), "--columns", "status", "--values", "finished" ])
def add_sim(name_command_line_tool="sim_db", name_command="add", argv=None): db = helpers.connect_sim_db() args = command_line_arguments_parser(name_command_line_tool, name_command).parse_args(argv) sim_params_filename = args.filename if sim_params_filename == None: sim_params_filename = search_for_parameter_file_matching_settings() if sim_params_filename == None: print("No parameter files in the current directory matches the " "ones under 'Parameter filenames'\nin settings.txt.\n" "\nAdd the '--filename' flag to specify the filename of " "the parameter file.") exit(1) elif (sim_params_filename[0:5] == 'root/'): sim_params_filename = os.path.join( os.path.join(helpers.get_dot_sim_db_dir_path(), os.pardir), sim_params_filename[5:]) elif (sim_params_filename[0:6] == '"root/'): sim_params_filename = '"' + os.path.join( os.path.join(helpers.get_dot_sim_db_dir_path(), os.pardir), sim_params_filename[6:]) try: sim_params_file = open(sim_params_filename, 'r') except: print("Could NOT open {0}.".format(sim_params_filename)) exit(1) sim_params_file_lines = sim_params_file.readlines() sim_params_file.close() sim_params_file_lines = add_included_parameter_files(sim_params_file_lines) sim_params_file_lines = replace_aliases(sim_params_file_lines) db_cursor = db.cursor() default_db_columns = "" for key in helpers.default_db_columns: default_db_columns += key + " " + str( helpers.default_db_columns[key]) + ", " default_db_columns = default_db_columns[:-2] db_cursor.execute( "CREATE TABLE IF NOT EXISTS runs ({0});".format(default_db_columns)) column_names, column_types = helpers.get_db_column_names_and_types( db_cursor) initial_parameters = [] last_row_id = None for i, line in enumerate(sim_params_file_lines): if len(line.split(':')) > 1: param_name, param_type, value = split_parameter_line(line, i) initial_parameters.append(param_name) if param_name == 'run_command': value = make_path_relative_to_root(value, sim_params_filename) try: row_index = column_names.index(param_name) except ValueError: row_index = None if row_index is None: add_new_column(db_cursor, i, param_type, param_name, value, column_names, column_types) else: check_type_matches(param_type, column_types[row_index], value, i) value = standardize_value(value, param_type) if len(value) > 0: last_row_id = insert_value(db_cursor, param_name, last_row_id, value) initial_parameters = standardize_value(str(initial_parameters), "string array") last_row_id = insert_value(db_cursor, 'initial_parameters', last_row_id, initial_parameters) db.commit() db_cursor.close() db.close() return last_row_id
def delete_results_dir(name_command_line_tool="sim_db", name_command="delete_results_dir", argv=None): db = helpers.connect_sim_db() db_cursor = db.cursor() args = command_line_arguments_parser(name_command_line_tool, name_command).parse_args(argv) if (len(args.id) == 0 and args.where == None and args.not_in_db_but_in_dir == None): print( "No 'results_dir' was deleted. --id 'ID' or --where 'CONDITION' " + "must be passed to the program.") exit(0) results_dirs = [] for delete_id in args.id: db_cursor.execute( "SELECT results_dir FROM runs WHERE id = {0}".format(delete_id)) results_dir = db_cursor.fetchone() if results_dir != None and results_dir[0] != None: results_dirs.append(results_dir[0]) if args.where: db_cursor.execute("SELECT results_dir FROM runs WHERE {0}".format( args.where)) for selected_output in db_cursor.fetchall(): if selected_output[0] != None: results_dirs.append(selected_output[0]) if args.not_in_db_but_in_dir != None: db_cursor.execute("SELECT results_dir FROM runs") results_dir = db_cursor.fetchone() while results_dir != None and results_dir[0] != None: results_dirs.append(results_dir[0]) results_dir = db_cursor.fetchone() db.commit() db_cursor.close() db.close() if args.not_in_db_but_in_dir != None: if not os.path.isabs(args.not_in_db_but_in_dir): if args.not_in_db_but_in_dir[0] == '.': args.not_in_db_but_in_dir = args.not_in_db_but_in_dir[2:] if args.not_in_db_but_in_dir[-1] == '/': args.not_in_db_but_in_dir = args.not_in_db_but_in_dir[:-1] args.not_in_db_but_in_dir = os.path.join(os.getcwd(), args.not_in_db_but_in_dir) for path in os.listdir(args.not_in_db_but_in_dir): path = os.path.join(args.not_in_db_but_in_dir, path) if os.path.isdir(path) and (path not in results_dirs): print("\nDo you really want to delete:") print(path) answer = helpers.user_input("? (y/n)") if (answer == 'y' or answer == 'Y' or answer == 'yes' or answer == 'Yes' or args.no_checks): shutil.rmtree(path) else: print(path) print("was NOT deleted.") elif len(results_dirs) > 0: answer = 'no' if not args.no_checks: print("Do you really want to delete the following directories and " "everything in them:") for results_dir in results_dirs: print(results_dir) answer = helpers.user_input("? (y/n)") if (answer == 'y' or answer == 'Y' or answer == 'yes' or answer == 'Yes' or args.no_checks): for results_dir in results_dirs: try: shutil.rmtree(results_dir) except Exception as e: if e.errno == errno.ENOENT: print("Results directory NOT found: {0}".format( results_dir)) else: raise e elif not args.no_checks: print("No results deleted.") elif not args.no_checks: print("No 'results_dir' to delete.")
def submit_sim(name_command_line_tool="sim_db", name_command="submit_sim", argv=None): args = command_line_arguments_parser(name_command_line_tool, name_command).parse_args(argv) ids = args.id db = helpers.connect_sim_db() db_cursor = db.cursor() job_script_name = None job_id = None if ids == None: db_cursor.execute("SELECT id FROM runs WHERE status = 'new';") ids = db_cursor.fetchall() ids = [i[0] for i in ids] answer = None while (not args.no_confirmation and answer != 'y' and answer != 'Y' and answer != 'yes' and answer != 'Yes'): answer = helpers.user_input( "Would you like to submit simulations " "with the following ID's: {0}? (y/n) ".format(ids)) if (answer == 'n' or answer == 'N' or answer == 'no' or answer == 'No'): db.commit() db_cursor.close() db.close() return (job_script_name, job_id) elif not args.allow_reruns: for i in ids: db_cursor.execute( "SELECT status FROM runs WHERE id = {0};".format(i)) status = db_cursor.fetchone()[0] if status != "new": print("No simulations was submitted.\nStatus of simulation " "with 'ID' = {0} is {1}.\nEither:\n- Add " "'--allow_reruns' flag to allow it to run.\n- Update " "status to 'new'.".format(args.id, status)) exit() if args.add_unique_results_dir: for i in ids: unique_results_dir = helpers.unique_results_dir(db_cursor, i) os.mkdir(unique_results_dir) update_sim.update_sim(argv=[ "--id", str(i), "--columns", "results_dir", "--values", unique_results_dir ]) for i, id_submit in enumerate(ids): name_job_script = make_job_script(db_cursor, i, args, id_submit) if not args.do_not_submit_job_script: job_id = submit_job_script(name_job_script) db_cursor.execute("UPDATE runs SET status='submitted' WHERE id={0}" \ .format(id_submit)) db_cursor.execute("UPDATE runs SET job_id={0} WHERE id={1}" \ .format(job_id, id_submit)) db.commit() db_cursor.close() db.close() print("Job ID: {0}".format(job_id)) return (name_job_script, job_id)
def print_sim(name_command_line_tool="sim_db", name_command="print_sim", argv=None): if argv == None: argv = ['-p', 'default'] elif '--columns' not in argv and '-c' not in argv: if len(argv) > 0 and argv[0][0] != '-': # To print correct error message even though '-p default' is added. command_line_arguments_parser(name_command_line_tool, name_command).parse_args(argv) argv = ['-p', 'default'] + argv argv_p_replaced = [] i = 0 while i < len(argv): if argv[i] != '-p': argv_p_replaced.append(argv[i]) else: while i + 1 < len(argv) and argv[i + 1][0] != '-': i += 1 print_config_key = argv[i] print_config = get_personalized_print_config(print_config_key) if print_config == None: print( "No personalized print configuration with key string " "'{0}' is found in settings.".format(print_config_key)) exit(1) argv_p_replaced = argv_p_replaced + shlex.split(print_config) i += 1 argv = argv_p_replaced args = command_line_arguments_parser(name_command_line_tool, name_command).parse_args(argv) db = helpers.connect_sim_db() db_cursor = db.cursor() column_names, column_types = helpers.get_db_column_names_and_types( db_cursor) type_dict = dict(zip(column_names, column_types)) selected_output, column_names = select_command(name_command_line_tool, name_command, db_cursor, args, column_names) if args.diff: selected_output, column_names = remove_columns_with_only_same_values( selected_output, column_names) if not args.vertically: print_selected_parameters(selected_output, column_names, args.no_headers, args.max_width, args.first_line) else: print_selected_parameters_vertically(selected_output, column_names, args.no_headers, args.max_width, args.first_line) if args.column_names: print("") print("Column names and types in database:") for col_name in column_names: col_type = type_dict[col_name] print("{0}, {1}".format(col_name, col_type)) db.commit() db_cursor.close() db.close()
def combine_dbs(name_command_line_tool="sim_db", name_command="combine_dbs", argv=None): args = command_line_arguments_parser(name_command_line_tool, name_command).parse_args(argv) db_1 = helpers.connect_sim_db(args.path_db_1) db_1_cursor = db_1.cursor() db_2 = helpers.connect_sim_db(args.path_db_2) db_2_cursor = db_2.cursor() (columns_db_1, types_db_1) = helpers.get_db_column_names_and_types(db_1_cursor) (columns_db_2, types_db_2) = helpers.get_db_column_names_and_types(db_2_cursor) column_type_dict = OrderedDict( [(col, typ) for col, typ in zip(columns_db_1, types_db_1)] + [(col, typ) for col, typ in zip(columns_db_2, types_db_2)]) new_db = helpers.connect_sim_db(args.name_new_db) new_db_cursor = new_db.cursor() new_db_columns_string = "" for col in column_type_dict: new_db_columns_string += col + " " + column_type_dict[col] + ", " new_db_columns_string = new_db_columns_string[:-2] new_db_columns_string = new_db_columns_string.replace( "id INTEGER", "id INTEGER PRIMARY KEY", 1) new_db_cursor.execute( "CREATE TABLE runs ({0});".format(new_db_columns_string)) db_1_cursor.execute("SELECT * FROM runs") for row in db_1_cursor.fetchall(): column_tuple = () value_tuple = () for column, value in zip(columns_db_1, row): column = helpers.if_unicode_convert_to_str(column) value = helpers.if_unicode_convert_to_str(value) if value != None and column != 'id': column_tuple += (column, ) value_tuple += (value, ) new_db_cursor.execute("INSERT INTO runs {0} VALUES {1}".format( column_tuple, value_tuple)) db_2_cursor.execute("SELECT * FROM runs") for row in db_2_cursor.fetchall(): column_tuple = () value_tuple = () for column, value in zip(columns_db_2, row): column = helpers.if_unicode_convert_to_str(column) value = helpers.if_unicode_convert_to_str(value) if value != None and column != 'id': column_tuple += (column, ) value_tuple += (value, ) new_db_cursor.execute("INSERT INTO runs {0} VALUES {1}".format( column_tuple, value_tuple)) db_1.commit() db_2.commit() new_db.commit() db_1_cursor.close() db_2_cursor.close() new_db_cursor.close() db_1.close() db_2.close() new_db.close()
def __init__(self, store_metadata=True, db_id=None, rank=None, only_write_on_rank=0): """Connect to the **sim_db** database. :param store_metadata: If False, no metadata is added to the database. Typically used when postprocessing (visualizing) data from a simulation. :type store_metadata: bool :param db_id: ID number of the simulation parameters in the **sim_db** database. If it is 'None', then it is read from the argument passed to the program after option '--id'. :type db_id: int :param rank: Number identifing the calling process and/or thread. (Typically the MPI or OpenMP rank.) If provided, only the 'rank' matching 'only_write_on_rank' will write to the database to avoid too much concurrent writing to the database. Single process and threaded programs may ignore this, while multithreading/multiprocessing programs need to provide it. :type rank: int :param only_write_on_rank: Number identifing the only process/thread that will write to the database. Only used if 'rank' is provided. :type only_write_on_rank: int """ self.rank = rank self.only_write_on_rank = only_write_on_rank self.start_time = time.time() self.store_metadata = store_metadata self.id, self.path_proj_root = self.__read_from_command_line_arguments( db_id) self.db = helpers.connect_sim_db() self.db_cursor = self.db.cursor() self.column_names = [] self.column_types = [] if (self.store_metadata and (self.rank == None or self.rank == self.only_write_on_rank)): self.write('status', 'running') self.write('time_started', self.__get_date_and_time_as_string()) if (self.store_metadata and self.__is_a_git_project() and (self.rank == None or self.rank == self.only_write_on_rank)): proc = subprocess.Popen( ['cd "{0}"; git rev-parse HEAD'.format(self.path_proj_root)], stdout=subprocess.PIPE, stderr=open(os.devnull, 'w'), shell=True) (out, err) = proc.communicate() self.write(column="git_hash", value=out.decode('ascii', 'replace')) proc = subprocess.Popen([ 'cd "{0}"; git log -n 1 --format=%B HEAD'.format( self.path_proj_root) ], stdout=subprocess.PIPE, stderr=open(os.devnull, 'w'), shell=True) (out, err) = proc.communicate() self.write(column="commit_message", value=out.decode('ascii', 'replace')) proc = subprocess.Popen( ['cd "{0}"; git diff HEAD --stat'.format(self.path_proj_root)], stdout=subprocess.PIPE, stderr=open(os.devnull, 'w'), shell=True) (out, err) = proc.communicate() self.write(column="git_diff_stat", value=out.decode('ascii', 'replace')) proc = subprocess.Popen( ['cd "{0}"; git diff HEAD'.format(self.path_proj_root)], stdout=subprocess.PIPE, stderr=open(os.devnull, 'w'), shell=True) (out, err) = proc.communicate() out = out.decode('ascii', 'replace') if len(out) > 3000: warning = "WARNING: Diff limited to first 3000 characters.\n" out = warning + '\n' + out[0:3000] + '\n\n' + warning self.write(column="git_diff", value=out)
def run_sim(name_command_line_tool="sim_db", name_command="run_sim", argv=None): """Run simulation with parameters with ID passed or the highest ID.""" args = command_line_arguments_parser(name_command_line_tool, name_command).parse_args(argv) db = helpers.connect_sim_db() db_cursor = db.cursor() if args.id == None: db_cursor.execute("SELECT id FROM runs WHERE status = 'new';") ids = db_cursor.fetchall() ids = [i[0] for i in ids] args.id = max(ids) print("Start simulation with ID {0}.".format(args.id)) elif not args.allow_reruns: db_cursor.execute("SELECT status FROM runs WHERE id = {0};".format( args.id)) status = db_cursor.fetchone() if status == None: print("There exists no entry in the database with id = {0} and a " "'run_command'.".format(args.id)) exit(1) status = status[0] if status != "new": print( "Status of simulation with 'ID' = {0} is {1}.\nEither:\n" "- Add '--allow_reruns' flag to allow it to run.\n- Update " "status to 'new'.\n- Duplicate it, delete the old and run it " "with: 'ddr' / 'duplicate_delete_and_run'".format( args.id, status)) exit(1) if args.add_unique_results_dir: unique_results_dir = helpers.unique_results_dir(db_cursor, args.id) print(unique_results_dir) os.mkdir(unique_results_dir) update_sim.update_sim(argv=[ "--id", str(args.id), "--columns", "results_dir", "--values", unique_results_dir ]) run_command = helpers.get_run_command(db_cursor, args.id, args.n) db.commit() db_cursor.close() db.close() update_sim.update_sim(argv=[ "--id", str(args.id), "--columns", "cpu_info", "--values", helpers.get_cpu_and_mem_info() ]) update_sim.update_sim(argv=[ "--id", str(args.id), "--columns", "status", "--values", "running" ]) for command in run_command.split(';'): process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) if sys.version_info[0] < 3: for line in iter(process.stdout.readline, ''): sys.stdout.write(line.decode('ascii', 'replace')) sys.stdout.flush() for line in iter(process.stderr.readline, ''): sys.stdout.write(line.decode('ascii', 'replace')) sys.stdout.flush() else: for line in iter(process.stdout.readline, b''): sys.stdout.write(line.decode('ascii', 'replace')) sys.stdout.flush() for line in iter(process.stderr.readline, b''): sys.stdout.write(line.decode('ascii', 'replace')) sys.stdout.flush() update_sim.update_sim(argv=[ "--id", str(args.id), "--columns", "status", "--values", "finished" ])
def add_range_sim(name_command_line_tool="sim_db", name_command="add_range_sim", argv=None): args = command_line_arguments_parser(name_command_line_tool, name_command).parse_args(argv) # Check command line arguments n_cols = len(args.columns) if ((len(args.lin_steps) != n_cols or len(args.exp_steps) != 0) and (len(args.lin_steps) != 0 or len(args.exp_steps) != n_cols) and (len(args.lin_steps) != n_cols or len(args.exp_steps) != n_cols)): print("ERROR: Either '--lin_steps', '--exp_steps' or both have to be provided " "and they have to be same length as '--columns'.") exit() elif ((len(args.end_steps) != n_cols or len(args.n_steps) != 0) and (len(args.end_steps) != 0 or len(args.n_steps) != n_cols) and (len(args.end_steps) != n_cols or len(args.n_steps) != n_cols)): print("ERROR: Either '--end_steps', '--n_steps' or both have to be provided " "and they have to be same length as '--columns'.") exit() ids_added = [] if args.filename == None: ids_added.append(add_sim.add_sim()) else: ids_added.append(add_sim.add_sim(argv=['--filename', args.filename])) # Get start value of each column that varies. db = helpers.connect_sim_db() db_cursor = db.cursor() start_values = [] for column in args.columns: db_cursor.execute("SELECT {0} FROM runs WHERE id = {1}".format( column, ids_added[0])) start_values.append(db_cursor.fetchone()[0]) db.commit() db_cursor.close() db.close() # Make table of the cartisian product. if len(args.lin_steps) == 0: args.lin_steps = [0 for i in range(n_cols)] if len(args.exp_steps) == 0: args.exp_steps = [1 for i in range(n_cols)] column_range = [[] for i in range(n_cols)] for i in range(n_cols): column_range[i].append(start_values[i]) if len(args.end_steps) == 0: for j in range(args.n_steps[i]): column_range[i].append(column_range[i][-1] * args.exp_steps[i] + args.lin_steps[i]) elif len(args.lin_steps) == 0: j = 0 while (start_values[i] != args.end_steps[i] and (start_values[i] < args.end_steps[i]) == ( column_range[i][j] < args.end_steps[i])): column_range[i].append(column_range[i][-1] * args.exp_steps[i] + args.lin_steps[i]) j += 1 else: j = 0 while (j < args.n_steps[i] or (start_values[i] != args.end_steps[i] and (start_values[i] < args.end_steps[i]) == (column_range[i][j] < args.end_steps[i]))): column_range[i].append(column_range[i][-1] * args.exp_steps[i] + args.lin_steps[i]) j += 1 cartisian_product = [[] for i in range(n_cols)] for i in range(n_cols - 1, -1, -1): cartisian_product = add_new_column_in_cartisian_product( cartisian_product, column_range[i], i) # Add all simulations. for i in range(1, len(cartisian_product[0])): if args.filename == None: ids_added.append(add_sim.add_sim()) else: ids_added.append( add_sim.add_sim(argv=['--filename', args.filename])) update_params = ['--id', str(ids_added[-1]), '--columns'] update_params.extend(args.columns) update_params.append('--values') for j in range(n_cols): update_params.append(str(cartisian_product[j][i])) update_sim.update_sim(argv=update_params) return ids_added
def extract_params(name_command_line_tool="sim_db", name_command="extract_params", argv=None): args = command_line_arguments_parser(name_command_line_tool, name_command).parse_args(argv) is_printing_parameters = True if args.default_file: param_files = helpers.Settings().read('parameter_files') if len(param_files) == 0: print("ERROR: No '--filename' provided and no 'Parameters files' " "in settings.txt.") exit() else: filename = param_files[0] if os.path.exists(filename): answer = helpers.user_input( "Would you like to overwrite '{0}'? (y/n)".format( filename)) if (answer != 'y' and answer != 'Y' and answer != 'yes' and answer != 'Yes'): exit() print("Extracts parameters to '{0}'.".format(filename)) is_printing_parameters = False elif args.filename != None: filename = args.filename is_printing_parameters = False if not is_printing_parameters: params_file = open(filename, 'w') db = helpers.connect_sim_db() db_cursor = db.cursor() db_cursor.execute("SELECT * FROM runs WHERE id={0}".format(args.id)) extracted_row = db_cursor.fetchall() column_names, column_types = helpers.get_db_column_names_and_types( db_cursor) db_cursor.execute("SELECT initial_parameters FROM runs WHERE id={0}" .format(args.id)) initial_parameters = db_cursor.fetchone()[0] initial_parameters, correct_type = helpers.convert_text_to_correct_type( initial_parameters, 'string array') for col_name, col_type, value in zip(column_names, column_types, extracted_row[0]): if (value != None and (col_name in initial_parameters or (args.also_results and col_name not in no_extract_columns) or args.all)): line = col_name param_type = get_param_type_as_string(col_type, value) line += " ({0}): ".format(param_type) if param_type[-3:] == 'ray': value = '[' + value.split('[')[1] line += str(value).replace(':', ';') + '\n' if is_printing_parameters: print(line) else: params_file.write(line) if not is_printing_parameters: params_file.close() db.commit() db_cursor.close() db.close()