def mongo_check(config_path, config_file):
"""Function: mongo_check
Description: Check the contents of the output file based on the items in
the search_list variable and check to see if file is in JSON format.
Arguments:
(input) config_path -> Path name to configuration file.
(input) config_file -> Name of configuration file, without .py ext.
(output) status -> True|False - Status of checks.
"""
status = True
cfg = gen_libs.load_module(config_file, config_path)
coll = mongo_libs.crt_coll_inst(cfg, cfg.db, cfg.coll)
coll.connect()
status = coll.coll_cnt() == 1
mongo = mongo_class.DB(cfg.name,
cfg.user,
cfg.passwd,
host=cfg.host,
port=cfg.port,
db=cfg.db,
auth=cfg.auth,
conf_file=cfg.conf_file)
mongo.db_connect(cfg.db)
mongo.db_cmd("dropDatabase")
cmds_gen.disconnect([coll, mongo])
return status
def run_program(args_array, func_dict, **kwargs):
"""Function: run_program
Description: Creates class instance(s) and controls flow of the program.
Arguments:
(input) args_array -> Array of command line options and values.
(input) func_dict -> Dictionary list of functions and options.
"""
args_array = dict(args_array)
func_dict = dict(func_dict)
slaves = create_instances(args_array)
if slaves and gtid_enabled(slaves):
# Call function(s) - intersection of command line and function dict.
for item in set(args_array.keys()) & set(func_dict.keys()):
err_flag, err_msg = func_dict[item](slaves, args_array)
if err_flag:
print(err_msg)
break
cmds_gen.disconnect(slaves)
else:
print("Error: Empty Slave array or Slave(s) not GTID enabled.")
def run_program(args_array, func_dict, opt_arg_list, **kwargs):
"""Function: run_program
Description: Creates class instance(s) and controls flow of the program.
Arguments:
(input) args_array -> Array of command line options and values.
(input) func_dict -> Dictionary list of functions and options.
(input) opt_arg_list -> Arguments to be added to command line.
"""
args_array = dict(args_array)
func_dict = dict(func_dict)
opt_arg_list = list(opt_arg_list)
server = mysql_libs.create_instance(args_array["-c"], args_array["-d"],
mysql_class.Server)
server.connect(silent=True)
if not server.conn_msg:
server.set_srv_binlog_crc()
# Call function(s) - intersection of command line and function dict.
for item in set(args_array.keys()) & set(func_dict.keys()):
# Call the function requested.
func_dict[item](server, args_array, opt_arg_list)
cmds_gen.disconnect(server)
else:
print("run_program: Error encountered on master(%s): %s" %
(server.name, server.conn_msg))
def setUp(self):
"""Function: setUp
Description: Initialization for unit testing.
Arguments:
"""
self.base_dir = "test/integration/server_usage"
self.test_path = os.path.join(os.getcwd(), self.base_dir)
self.config_path = os.path.join(self.test_path, "config")
self.cfg = gen_libs.load_module("configuration", self.config_path)
self.args_array = {"-c": "configuration", "-d": self.config_path}
svr = mongo_class.Server(self.cfg.name,
self.cfg.user,
self.cfg.passwd,
host=self.cfg.host,
port=self.cfg.port,
auth=self.cfg.auth,
conf_file=self.cfg.conf_file)
svr.connect()
if self.cfg.db in svr.fetch_dbs():
print("ERROR: Test environment not clean - database: %s exists" %
(self.cfg.db))
cmds_gen.disconnect([svr])
self.skipTest("Pre-conditions not met.")
cmds_gen.disconnect([svr])
def run_program(args_array, func_dict, ord_prec_list, **kwargs):
"""Function: run_program
Description: Creates class instance(s) and controls flow of the program.
Arguments:
(input) args_array -> Dict of command line options and values.
(input) func_dict -> Dictionary list of functions and options.
(input) ord_prec_list -> Order of precedence of the arguments.
"""
args_array = dict(args_array)
func_dict = dict(func_dict)
ord_prec_list = list(ord_prec_list)
server = mysql_libs.create_instance(args_array["-c"], args_array["-d"],
mysql_class.Server)
server.connect()
# Execute functions based on order of precedence.
for item in ord_prec_list:
if item in args_array:
func_dict[item](args_array, server)
cmds_gen.disconnect([server])
def truncate_coll(repclu, args_array, **kwargs):
"""Function: truncate_coll
Description: Truncate a collection in a Mongo database.
Arguments:
(input) repclu -> Replication set/cluster instance.
(input) args_array -> Array of command line options and values.
"""
args_array = dict(args_array)
coll = mongo_class.RepSetColl(repclu.name, repclu.user, repclu.passwd,
host=repclu.host, port=repclu.port,
auth=repclu.auth, repset=repclu.repset,
repset_hosts=repclu.repset_hosts,
db=args_array.get("-b"),
coll=args_array.get("-t"),
db_auth=args_array.get("-a", None))
coll.connect()
# Require override option.
coll.coll_del_many({}, True)
cmds_gen.disconnect([coll])
def get_repset_name(svr_cfg, **kwargs):
"""Function: get_repset_name
Description: Fetch the Replication Set Name from the condfiguration file
or from the Mongo database.
Arguments:
(input) svr_cfg -> Server configuration module.
(output) rep_set -> Replication set name.
"""
try:
rep_set = svr_cfg.repset
except AttributeError:
coll = mongo_class.Coll(svr_cfg.name, svr_cfg.user, svr_cfg.passwd,
host=svr_cfg.host, port=svr_cfg.port,
db="local", coll="system.replset",
auth=svr_cfg.auth, conf_file=svr_cfg.conf_file)
coll.connect()
# Are there any records.
if coll.coll_cnt() != 0:
rep_set = coll.coll_find1().get("_id")
else:
rep_set = None
cmds_gen.disconnect([coll])
return rep_set
def chk_rep_cfg(source, clone, args_array, req_rep_cfg, opt_arg_list,
**kwargs):
"""Function: chk_rep_cfg
Description: Replication configuration check on the source & clone
servers.
Arguments:
(input) source -> Source server instance.
(input) clone -> Destination server instance.
(input) args_array -> Array of command line options and values.
(input) req_rep_cfg -> Required replication config settings.
(input) opt_arg_list -> List of options to add to dump cmd line.
(output) opt_arg_list -> List of options to add to dump cmd line.
"""
args_array = dict(args_array)
req_rep_cfg = dict(req_rep_cfg)
opt_arg_list = list(opt_arg_list)
if "-n" not in args_array:
source.upd_mst_rep_stat()
clone.upd_slv_rep_stat()
# Both servers must meet replication requirements.
if not cfg_chk(source.fetch_mst_rep_cfg, req_rep_cfg["master"]) \
or not cfg_chk(clone.fetch_slv_rep_cfg, req_rep_cfg["slave"]):
# Create list to act as a failure of the requirements.
opt_arg_list = list()
cmds_gen.disconnect(source, clone)
print("Error: Master and/or Slave rep config did not pass.")
else:
if clone.gtid_mode:
# Exclude "change master to" option from dump file.
opt_arg_list.append("--master-data=2")
else:
# Include "change master to" option in dump file.
opt_arg_list.append("--master-data=1")
else:
# Exclude "change master to" option from dump file.
opt_arg_list.append("--master-data=2")
return opt_arg_list
def load_log(server, args_array, opt_arg_list, **kwargs):
"""Function: load_log
Description: Get the binary logs from the source database, then fetch the
revelant binary log entries and load them into the target
database before closing all connections.
Arguments:
(input) server -> Server instance.
(input) args_array -> Array of command line options and values.
(input) opt_arg_list -> Arguments to be added to command line.
"""
subp = gen_libs.get_inst(subprocess)
args_array = dict(args_array)
opt_arg_list = list(opt_arg_list)
status, binlog_list = process_logs_list(server, args_array)
if status[0]:
target = mysql_libs.create_instance(args_array["-e"], args_array["-d"],
mysql_class.Server)
target.connect(silent=True)
if not target.conn_msg:
cmd = mysql_libs.crt_cmd(
target,
arg_parser.arg_set_path(args_array, "-p") + "mysql")
# Fetch binary logs and restore to target database
proc1 = fetch_binlog(server, args_array.get("-s"),
args_array.get("-t"), binlog_list,
opt_arg_list,
arg_parser.arg_set_path(args_array, "-p"))
proc2 = subp.Popen(cmd, stdin=proc1)
proc2.wait()
cmds_gen.disconnect(target)
else:
print("load_log: Error encountered on slave(%s): %s" %
(target.name, target.conn_msg))
else:
print("load_log: Error encountered in process_logs_list: %s" %
(status[1]))
def chk_rep(clone, args_array, **kwargs):
"""Function: chk_rep
Description: Create master and slave instances and check the status of the
replication system between the two servers.
Arguments:
(input) clone -> Destination server instance.
(input) args_array -> Array of command line options and values.
"""
args_array = dict(args_array)
if "-n" not in args_array:
cfg = gen_libs.load_module(args_array["-c"], args_array["-d"])
master = mysql_class.MasterRep(cfg.name,
cfg.sid,
cfg.user,
cfg.japd,
os_type=getattr(machine, cfg.serv_os)(),
host=cfg.host,
port=cfg.port,
defaults_file=cfg.cfg_file,
extra_def_file=cfg.__dict__.get(
"extra_def_file", None),
rep_user=cfg.rep_user,
rep_japd=cfg.rep_japd)
master.connect()
mysql_libs.change_master_to(master, clone)
slave = mysql_libs.create_instance(args_array["-t"], args_array["-d"],
mysql_class.SlaveRep)
slave.connect()
slave.start_slave()
# Waiting for slave to start.
time.sleep(5)
master.upd_mst_status()
slave.upd_slv_status()
chk_slv_err([slave])
chk_slv_thr([slave])
chk_mst_log(master, [slave])
cmds_gen.disconnect(master, slave)
def delete_docs(repclu, args_array, **kwargs):
"""Function: delete_docs
Description: Deletion of a document in a Mongo database. Allows for one
or more key|value(s) pairs to be used as the search criteria for
the deletion command. For each key, must have a corresponding
set of value(s).
Arguments:
(input) repclu -> Replication set/cluster instance.
(input) args_array -> Array of command line options and values.
"""
args_array = dict(args_array)
coll = mongo_class.RepSetColl(repclu.name, repclu.user, repclu.passwd,
host=repclu.host, port=repclu.port,
auth=repclu.auth, repset=repclu.repset,
repset_hosts=repclu.repset_hosts,
db=args_array.get("-b"),
coll=args_array.get("-t"),
db_auth=args_array.get("-a", None))
coll.connect()
if args_array.get("-f", None):
for fname in args_array["-f"]:
lines = gen_libs.file_2_list(fname)
# Process each line as a delete.
for qry in lines:
coll.coll_del_many(gen_libs.str_2_type(qry))
# Assume -kN and -lN options.
else:
status, qry = process_args(args_array)
if not status:
coll.coll_del_many(qry)
cmds_gen.disconnect([coll])
def tearDown(self):
"""Function: tearDown
Description: Clean up of integration testing.
Arguments:
"""
mongo = mongo_class.DB(self.cfg.name,
self.cfg.user,
self.cfg.passwd,
host=self.cfg.host,
port=self.cfg.port,
db=self.cfg.db,
auth=self.cfg.auth,
conf_file=self.cfg.conf_file)
mongo.db_connect(self.cfg.db)
mongo.db_cmd("dropDatabase")
cmds_gen.disconnect([mongo])
def run_program(args_array, func_dict, **kwargs):
"""Function: run_program
Description: Creates class instance(s) and controls flow of the program.
Arguments:
(input) args_array -> Array of command line options and values.
(input) func_dict -> Dictionary list of functions and options.
"""
args_array = dict(args_array)
func_dict = dict(func_dict)
master = None
if "-c" in args_array:
mst_cfg = gen_libs.load_module(args_array["-c"], args_array["-d"])
master = mysql_class.MasterRep(
mst_cfg.name, mst_cfg.sid, mst_cfg.user, mst_cfg.passwd,
machine=getattr(machine, mst_cfg.serv_os)(), host=mst_cfg.host,
port=mst_cfg.port, defaults_file=mst_cfg.cfg_file)
master.connect()
slaves = []
if "-s" in args_array:
slv_cfg = cmds_gen.create_cfg_array(args_array["-s"],
cfg_path=args_array["-d"])
slaves = mysql_libs.create_slv_array(slv_cfg)
call_run_chk(args_array, func_dict, master, slaves)
if master:
cmds_gen.disconnect(master, slaves)
else:
cmds_gen.disconnect(slaves)
def run_program(args_array, req_rep_cfg, opt_arg_list, **kwargs):
"""Function: run_program
Description: Creates class instance(s) and controls flow of the program.
Also determines whether the cloning operation is for replication or as
a stand-along server.
Arguments:
(input) args_array -> Dict of command line options and values.
(input) req_rep_cfg -> Required replication config settings.
(input) opt_arg_list -> List of options to add to dump cmd line.
(input) **kwargs:
opt_dump_list -> Dictionary of additional options.
"""
args_array = dict(args_array)
req_rep_cfg = dict(req_rep_cfg)
opt_arg_list = list(opt_arg_list)
source = mysql_libs.create_instance(args_array["-c"], args_array["-d"],
mysql_class.Server)
clone = mysql_libs.create_instance(args_array["-t"], args_array["-d"],
mysql_class.Server)
source.connect()
source.set_srv_gtid()
clone.connect()
clone.set_srv_gtid()
status, status_msg = mysql_libs.is_cfg_valid([source, clone])
# Master cannot be set to loopback IP if setting up replication.
if source.host in ["127.0." + "0.1", "localhost"] \
and "-n" not in args_array:
status = False
status_msg.append("Master host entry has incorrect entry.")
status_msg.append("Master host: %s" % (source.host))
if status:
# Do not proceed if GTID modes don't match and rep is being configured.
if source.gtid_mode != clone.gtid_mode and "-n" not in args_array:
cmds_gen.disconnect(source, clone)
print(
"Error: Source (%s) and Clone (%s) GTID modes do not match." %
(source.gtid_mode, clone.gtid_mode))
else:
stop_clr_rep(clone, args_array)
# Add to argument list array based on rep config.
opt_arg_list = chk_rep_cfg(source, clone, args_array, req_rep_cfg,
opt_arg_list)
# If empty list, then failure in requirements check.
if opt_arg_list:
print("Starting dump-load process...")
dump_load_dbs(source, clone, args_array, req_rep_cfg,
opt_arg_list, **kwargs)
print("Finished dump-load process...")
chk_rep(clone, args_array)
cmds_gen.disconnect(source, clone)
else:
cmds_gen.disconnect(source, clone)
print("Error: Detected problem in the configuration file.")
for msg in status_msg:
print(msg)
