def mongo_check(config_path, config_file): """Function: mongo_check Description: Check the contents of the output file based on the items in the search_list variable and check to see if file is in JSON format. Arguments: (input) config_path -> Path name to configuration file. (input) config_file -> Name of configuration file, without .py ext. (output) status -> True|False - Status of checks. """ status = True cfg = gen_libs.load_module(config_file, config_path) coll = mongo_libs.crt_coll_inst(cfg, cfg.db, cfg.coll) coll.connect() status = coll.coll_cnt() == 1 mongo = mongo_class.DB(cfg.name, cfg.user, cfg.passwd, host=cfg.host, port=cfg.port, db=cfg.db, auth=cfg.auth, conf_file=cfg.conf_file) mongo.db_connect(cfg.db) mongo.db_cmd("dropDatabase") cmds_gen.disconnect([coll, mongo]) return status
def run_program(args_array, func_dict, **kwargs): """Function: run_program Description: Creates class instance(s) and controls flow of the program. Arguments: (input) args_array -> Array of command line options and values. (input) func_dict -> Dictionary list of functions and options. """ args_array = dict(args_array) func_dict = dict(func_dict) slaves = create_instances(args_array) if slaves and gtid_enabled(slaves): # Call function(s) - intersection of command line and function dict. for item in set(args_array.keys()) & set(func_dict.keys()): err_flag, err_msg = func_dict[item](slaves, args_array) if err_flag: print(err_msg) break cmds_gen.disconnect(slaves) else: print("Error: Empty Slave array or Slave(s) not GTID enabled.")
def run_program(args_array, func_dict, opt_arg_list, **kwargs): """Function: run_program Description: Creates class instance(s) and controls flow of the program. Arguments: (input) args_array -> Array of command line options and values. (input) func_dict -> Dictionary list of functions and options. (input) opt_arg_list -> Arguments to be added to command line. """ args_array = dict(args_array) func_dict = dict(func_dict) opt_arg_list = list(opt_arg_list) server = mysql_libs.create_instance(args_array["-c"], args_array["-d"], mysql_class.Server) server.connect(silent=True) if not server.conn_msg: server.set_srv_binlog_crc() # Call function(s) - intersection of command line and function dict. for item in set(args_array.keys()) & set(func_dict.keys()): # Call the function requested. func_dict[item](server, args_array, opt_arg_list) cmds_gen.disconnect(server) else: print("run_program: Error encountered on master(%s): %s" % (server.name, server.conn_msg))
def setUp(self): """Function: setUp Description: Initialization for unit testing. Arguments: """ self.base_dir = "test/integration/server_usage" self.test_path = os.path.join(os.getcwd(), self.base_dir) self.config_path = os.path.join(self.test_path, "config") self.cfg = gen_libs.load_module("configuration", self.config_path) self.args_array = {"-c": "configuration", "-d": self.config_path} svr = mongo_class.Server(self.cfg.name, self.cfg.user, self.cfg.passwd, host=self.cfg.host, port=self.cfg.port, auth=self.cfg.auth, conf_file=self.cfg.conf_file) svr.connect() if self.cfg.db in svr.fetch_dbs(): print("ERROR: Test environment not clean - database: %s exists" % (self.cfg.db)) cmds_gen.disconnect([svr]) self.skipTest("Pre-conditions not met.") cmds_gen.disconnect([svr])
def run_program(args_array, func_dict, ord_prec_list, **kwargs): """Function: run_program Description: Creates class instance(s) and controls flow of the program. Arguments: (input) args_array -> Dict of command line options and values. (input) func_dict -> Dictionary list of functions and options. (input) ord_prec_list -> Order of precedence of the arguments. """ args_array = dict(args_array) func_dict = dict(func_dict) ord_prec_list = list(ord_prec_list) server = mysql_libs.create_instance(args_array["-c"], args_array["-d"], mysql_class.Server) server.connect() # Execute functions based on order of precedence. for item in ord_prec_list: if item in args_array: func_dict[item](args_array, server) cmds_gen.disconnect([server])
def truncate_coll(repclu, args_array, **kwargs): """Function: truncate_coll Description: Truncate a collection in a Mongo database. Arguments: (input) repclu -> Replication set/cluster instance. (input) args_array -> Array of command line options and values. """ args_array = dict(args_array) coll = mongo_class.RepSetColl(repclu.name, repclu.user, repclu.passwd, host=repclu.host, port=repclu.port, auth=repclu.auth, repset=repclu.repset, repset_hosts=repclu.repset_hosts, db=args_array.get("-b"), coll=args_array.get("-t"), db_auth=args_array.get("-a", None)) coll.connect() # Require override option. coll.coll_del_many({}, True) cmds_gen.disconnect([coll])
def get_repset_name(svr_cfg, **kwargs): """Function: get_repset_name Description: Fetch the Replication Set Name from the condfiguration file or from the Mongo database. Arguments: (input) svr_cfg -> Server configuration module. (output) rep_set -> Replication set name. """ try: rep_set = svr_cfg.repset except AttributeError: coll = mongo_class.Coll(svr_cfg.name, svr_cfg.user, svr_cfg.passwd, host=svr_cfg.host, port=svr_cfg.port, db="local", coll="system.replset", auth=svr_cfg.auth, conf_file=svr_cfg.conf_file) coll.connect() # Are there any records. if coll.coll_cnt() != 0: rep_set = coll.coll_find1().get("_id") else: rep_set = None cmds_gen.disconnect([coll]) return rep_set
def chk_rep_cfg(source, clone, args_array, req_rep_cfg, opt_arg_list, **kwargs): """Function: chk_rep_cfg Description: Replication configuration check on the source & clone servers. Arguments: (input) source -> Source server instance. (input) clone -> Destination server instance. (input) args_array -> Array of command line options and values. (input) req_rep_cfg -> Required replication config settings. (input) opt_arg_list -> List of options to add to dump cmd line. (output) opt_arg_list -> List of options to add to dump cmd line. """ args_array = dict(args_array) req_rep_cfg = dict(req_rep_cfg) opt_arg_list = list(opt_arg_list) if "-n" not in args_array: source.upd_mst_rep_stat() clone.upd_slv_rep_stat() # Both servers must meet replication requirements. if not cfg_chk(source.fetch_mst_rep_cfg, req_rep_cfg["master"]) \ or not cfg_chk(clone.fetch_slv_rep_cfg, req_rep_cfg["slave"]): # Create list to act as a failure of the requirements. opt_arg_list = list() cmds_gen.disconnect(source, clone) print("Error: Master and/or Slave rep config did not pass.") else: if clone.gtid_mode: # Exclude "change master to" option from dump file. opt_arg_list.append("--master-data=2") else: # Include "change master to" option in dump file. opt_arg_list.append("--master-data=1") else: # Exclude "change master to" option from dump file. opt_arg_list.append("--master-data=2") return opt_arg_list
def load_log(server, args_array, opt_arg_list, **kwargs): """Function: load_log Description: Get the binary logs from the source database, then fetch the revelant binary log entries and load them into the target database before closing all connections. Arguments: (input) server -> Server instance. (input) args_array -> Array of command line options and values. (input) opt_arg_list -> Arguments to be added to command line. """ subp = gen_libs.get_inst(subprocess) args_array = dict(args_array) opt_arg_list = list(opt_arg_list) status, binlog_list = process_logs_list(server, args_array) if status[0]: target = mysql_libs.create_instance(args_array["-e"], args_array["-d"], mysql_class.Server) target.connect(silent=True) if not target.conn_msg: cmd = mysql_libs.crt_cmd( target, arg_parser.arg_set_path(args_array, "-p") + "mysql") # Fetch binary logs and restore to target database proc1 = fetch_binlog(server, args_array.get("-s"), args_array.get("-t"), binlog_list, opt_arg_list, arg_parser.arg_set_path(args_array, "-p")) proc2 = subp.Popen(cmd, stdin=proc1) proc2.wait() cmds_gen.disconnect(target) else: print("load_log: Error encountered on slave(%s): %s" % (target.name, target.conn_msg)) else: print("load_log: Error encountered in process_logs_list: %s" % (status[1]))
def chk_rep(clone, args_array, **kwargs): """Function: chk_rep Description: Create master and slave instances and check the status of the replication system between the two servers. Arguments: (input) clone -> Destination server instance. (input) args_array -> Array of command line options and values. """ args_array = dict(args_array) if "-n" not in args_array: cfg = gen_libs.load_module(args_array["-c"], args_array["-d"]) master = mysql_class.MasterRep(cfg.name, cfg.sid, cfg.user, cfg.japd, os_type=getattr(machine, cfg.serv_os)(), host=cfg.host, port=cfg.port, defaults_file=cfg.cfg_file, extra_def_file=cfg.__dict__.get( "extra_def_file", None), rep_user=cfg.rep_user, rep_japd=cfg.rep_japd) master.connect() mysql_libs.change_master_to(master, clone) slave = mysql_libs.create_instance(args_array["-t"], args_array["-d"], mysql_class.SlaveRep) slave.connect() slave.start_slave() # Waiting for slave to start. time.sleep(5) master.upd_mst_status() slave.upd_slv_status() chk_slv_err([slave]) chk_slv_thr([slave]) chk_mst_log(master, [slave]) cmds_gen.disconnect(master, slave)
def delete_docs(repclu, args_array, **kwargs): """Function: delete_docs Description: Deletion of a document in a Mongo database. Allows for one or more key|value(s) pairs to be used as the search criteria for the deletion command. For each key, must have a corresponding set of value(s). Arguments: (input) repclu -> Replication set/cluster instance. (input) args_array -> Array of command line options and values. """ args_array = dict(args_array) coll = mongo_class.RepSetColl(repclu.name, repclu.user, repclu.passwd, host=repclu.host, port=repclu.port, auth=repclu.auth, repset=repclu.repset, repset_hosts=repclu.repset_hosts, db=args_array.get("-b"), coll=args_array.get("-t"), db_auth=args_array.get("-a", None)) coll.connect() if args_array.get("-f", None): for fname in args_array["-f"]: lines = gen_libs.file_2_list(fname) # Process each line as a delete. for qry in lines: coll.coll_del_many(gen_libs.str_2_type(qry)) # Assume -kN and -lN options. else: status, qry = process_args(args_array) if not status: coll.coll_del_many(qry) cmds_gen.disconnect([coll])
def tearDown(self): """Function: tearDown Description: Clean up of integration testing. Arguments: """ mongo = mongo_class.DB(self.cfg.name, self.cfg.user, self.cfg.passwd, host=self.cfg.host, port=self.cfg.port, db=self.cfg.db, auth=self.cfg.auth, conf_file=self.cfg.conf_file) mongo.db_connect(self.cfg.db) mongo.db_cmd("dropDatabase") cmds_gen.disconnect([mongo])
def run_program(args_array, func_dict, **kwargs): """Function: run_program Description: Creates class instance(s) and controls flow of the program. Arguments: (input) args_array -> Array of command line options and values. (input) func_dict -> Dictionary list of functions and options. """ args_array = dict(args_array) func_dict = dict(func_dict) master = None if "-c" in args_array: mst_cfg = gen_libs.load_module(args_array["-c"], args_array["-d"]) master = mysql_class.MasterRep( mst_cfg.name, mst_cfg.sid, mst_cfg.user, mst_cfg.passwd, machine=getattr(machine, mst_cfg.serv_os)(), host=mst_cfg.host, port=mst_cfg.port, defaults_file=mst_cfg.cfg_file) master.connect() slaves = [] if "-s" in args_array: slv_cfg = cmds_gen.create_cfg_array(args_array["-s"], cfg_path=args_array["-d"]) slaves = mysql_libs.create_slv_array(slv_cfg) call_run_chk(args_array, func_dict, master, slaves) if master: cmds_gen.disconnect(master, slaves) else: cmds_gen.disconnect(slaves)
def run_program(args_array, req_rep_cfg, opt_arg_list, **kwargs): """Function: run_program Description: Creates class instance(s) and controls flow of the program. Also determines whether the cloning operation is for replication or as a stand-along server. Arguments: (input) args_array -> Dict of command line options and values. (input) req_rep_cfg -> Required replication config settings. (input) opt_arg_list -> List of options to add to dump cmd line. (input) **kwargs: opt_dump_list -> Dictionary of additional options. """ args_array = dict(args_array) req_rep_cfg = dict(req_rep_cfg) opt_arg_list = list(opt_arg_list) source = mysql_libs.create_instance(args_array["-c"], args_array["-d"], mysql_class.Server) clone = mysql_libs.create_instance(args_array["-t"], args_array["-d"], mysql_class.Server) source.connect() source.set_srv_gtid() clone.connect() clone.set_srv_gtid() status, status_msg = mysql_libs.is_cfg_valid([source, clone]) # Master cannot be set to loopback IP if setting up replication. if source.host in ["127.0." + "0.1", "localhost"] \ and "-n" not in args_array: status = False status_msg.append("Master host entry has incorrect entry.") status_msg.append("Master host: %s" % (source.host)) if status: # Do not proceed if GTID modes don't match and rep is being configured. if source.gtid_mode != clone.gtid_mode and "-n" not in args_array: cmds_gen.disconnect(source, clone) print( "Error: Source (%s) and Clone (%s) GTID modes do not match." % (source.gtid_mode, clone.gtid_mode)) else: stop_clr_rep(clone, args_array) # Add to argument list array based on rep config. opt_arg_list = chk_rep_cfg(source, clone, args_array, req_rep_cfg, opt_arg_list) # If empty list, then failure in requirements check. if opt_arg_list: print("Starting dump-load process...") dump_load_dbs(source, clone, args_array, req_rep_cfg, opt_arg_list, **kwargs) print("Finished dump-load process...") chk_rep(clone, args_array) cmds_gen.disconnect(source, clone) else: cmds_gen.disconnect(source, clone) print("Error: Detected problem in the configuration file.") for msg in status_msg: print(msg)