def _call_func(args_array, func_dict, repinst, **kwargs):
"""Function: _call_func
Description: Private function for run_program. Call each function
selected.
Arguments:
(input) args_array -> Dict of command line options and values.
(input) func_dict -> Dictionary list of functions and options.
(input) repset -> Replication set instance.
"""
args_array = dict(args_array)
func_dict = dict(func_dict)
mail = None
if args_array.get("-e", None):
mail = gen_class.setup_mail(args_array.get("-e"),
subj=args_array.get("-s", None))
# Call function: Intersection of command line & function dict.
for item in set(args_array.keys()) & set(func_dict.keys()):
status3 = func_dict[item](repinst, args_array, mail=mail)
if not status3[0]:
print("Error detected: %s" % (status3[1]))
def run_program(args_array, opt_arg_list, opt_dump_list, **kwargs):
"""Function: run_program
Description: Creates class instance(s) and controls flow of the program.
Arguments:
(input) args_array -> Array of command line options and values.
(input) opt_arg_list -> List of commands to add to cmd line.
(input) opt_dump_list -> Dictionary of additional options.
"""
args_array = dict(args_array)
opt_dump_list = dict(opt_dump_list)
opt_arg_list = list(opt_arg_list)
mail = None
server = mysql_libs.create_instance(args_array["-c"], args_array["-d"],
mysql_class.Server)
server.connect(silent=True)
if server.conn_msg:
print("run_program: Error encountered on server(%s): %s" %
(server.name, server.conn_msg))
else:
server.set_srv_gtid()
dump_cmd = crt_dump_cmd(server, args_array, opt_arg_list,
opt_dump_list)
db_list = set_db_list(server, args_array, **kwargs)
# Remove the -r option if database is not GTID enabled.
if "-r" in args_array and not server.gtid_mode \
and opt_dump_list["-r"] in dump_cmd:
dump_cmd.remove(opt_dump_list["-r"])
compress = args_array.get("-z", False)
dmp_path = None
if "-o" in args_array:
dmp_path = args_array["-o"] + "/"
if args_array.get("-e", False):
dtg = datetime.datetime.strftime(datetime.datetime.now(),
"%Y%m%d_%H%M%S")
subj = args_array.get("-t",
[server.name, ": mysql_db_dump: ", dtg])
mail = gen_class.setup_mail(args_array.get("-e"), subj=subj)
err_sup = args_array.get("-w", False)
dump_db(dump_cmd,
db_list,
compress,
dmp_path,
err_sup=err_sup,
mail=mail,
use_mailx=args_array.get("-u", False))
mysql_libs.disconnect(server)
def run_program(args_array, func_dict, **kwargs):
"""Function: run_program
Description: Creates class instance(s) and controls flow of the program.
Arguments:
(input) args_array -> Dict of command line options and values.
(input) func_dict -> Dictionary list of functions and options.
(input) **kwargs:
opt_arg -> Dictionary of additional options to add.
arg_req_dict -> contains link between config and required option.
"""
args_array = dict(args_array)
func_dict = dict(func_dict)
arg_req_dict = dict(kwargs.get("arg_req_dict", {}))
mail = None
server = mongo_libs.create_instance(args_array["-c"], args_array["-d"],
mongo_class.Server)
status = server.connect()
if status[0]:
req_arg = get_req_options(server, arg_req_dict)
if args_array.get("-e", False):
dtg = datetime.datetime.strftime(datetime.datetime.now(),
"%Y%m%d_%H%M%S")
subj = args_array.get("-s",
[server.name, ": mongo_db_dump: ", dtg])
mail = gen_class.setup_mail(args_array.get("-e"), subj=subj)
# Intersect args_array and func_dict to decide which functions to call.
for item in set(args_array.keys()) & set(func_dict.keys()):
err_flag, err_msg = func_dict[item](server, args_array, mail=mail,
req_arg=req_arg, **kwargs)
if err_flag:
print(err_msg)
break
mongo_libs.disconnect([server])
else:
print("Connection failure: %s" % (status[1]))
def run_program(args_array, func_dict, **kwargs):
"""Function: run_program
Description: Creates class instance(s) and controls flow of the program.
Arguments:
(input) args_array -> Dict of command line options and values.
(input) func_dict -> Dictionary list of functions and options.
"""
args_array = dict(args_array)
func_dict = dict(func_dict)
server = mongo_libs.create_instance(args_array["-c"], args_array["-d"],
mongo_class.Server)
state = server.connect()
if state[0]:
outfile = args_array.get("-o", None)
db_tbl = args_array.get("-i", None)
repcfg = None
mail = None
if args_array.get("-m", None):
repcfg = gen_libs.load_module(args_array["-m"], args_array["-d"])
if args_array.get("-e", None):
mail = gen_class.setup_mail(args_array.get("-e"),
subj=args_array.get("-s", None))
# Call function(s) - intersection of command line and function dict.
for item in set(args_array.keys()) & set(func_dict.keys()):
err_flag, err_msg = func_dict[item](
server, args_array, ofile=outfile, db_tbl=db_tbl,
class_cfg=repcfg, mail=mail, **kwargs)
if err_flag:
print("Error: %s" % (err_msg))
break
mongo_libs.disconnect([server])
else:
print("Connection failure: %s" % (state[1]))
def setUp(self):
"""Function: setUp
Description: Initialization for unit testing.
Arguments:
"""
self.master = Server()
self.slave = Server()
self.mail = gen_class.setup_mail("email_addr", subj="subjectline")
self.tbl = "tbl1"
self.no_std = True
self.results = "\tChecking: {0} {1}".format(self.tbl.ljust(40),
"Synced")
self.results2 = "\tChecking: {0} {1}".format(
self.tbl.ljust(40), "Error: Checksums do not match")
def run_program(args_array, func_dict, **kwargs):
"""Function: run_program
Description: Creates class instance(s) and controls flow of the program.
Arguments:
(input) args_array -> Dictionary of command line options.
(input) func_dict -> Dictionary of functions.
(input) **kwargs:
sys_dbs -> List of system databases.
multi_val -> List of options that may have multiple values.
"""
args_array = dict(args_array)
func_dict = dict(func_dict)
server = mysql_libs.create_instance(args_array["-c"], args_array["-d"],
mysql_class.Server)
server.connect(silent=True)
if server.conn_msg:
print("run_program: Error encountered on server(%s): %s" %
(server.name, server.conn_msg))
else:
outfile = args_array.get("-o", None)
db_tbl = args_array.get("-i", None)
mongo = None
mail = None
if args_array.get("-m", None):
mongo = gen_libs.load_module(args_array["-m"], args_array["-d"])
if args_array.get("-e", None):
mail = gen_class.setup_mail(args_array.get("-e"),
subj=args_array.get("-s", None))
# Intersect args_array & func_dict to determine which functions to call
for item in set(args_array.keys()) & set(func_dict.keys()):
func_dict[item](server, args_array, ofile=outfile, db_tbl=db_tbl,
class_cfg=mongo, mail=mail, **kwargs)
mysql_libs.disconnect(server)
def setUp(self):
"""Function: setUp
Description: Initialization for unit testing.
Arguments:
"""
self.mail = gen_class.setup_mail("email_addr", subj="subject_line")
self.server = Server()
self.server2 = Server2()
self.server3 = Server3()
self.args_array = {"-o": "DirectoryPath"}
self.msg = "Error/Warning detected in database dump."
self.msg1 = "Warning: Database still locked after dump."
self.msg1a = self.msg + self.msg1
self.msg2 = "Error: Unable to lock the database for dump to occur."
self.msg2a = self.msg + self.msg1
self.msg3 = ""
self.msg3a = ""
def setUp(self):
"""Function: setUp
Description: Initialization for unit testing.
Arguments:
"""
self.data = {}
self.data2 = {"key1": "value1"}
self.data3 = {"key1": "value1", "key2": "value2"}
self.data4 = {"key1": {"key1a": "value1a"}}
self.data5 = {"key1": {"key1a": "value1a"}, "key2": "value2"}
self.data6 = {
"key1": "value1",
"key2": {
"key2A": "value2A",
"key2B": "value2B"
},
"key3": {
"key3A": {
"key3AA": "value3AA"
}
}
}
self.mail = gen_class.setup_mail("email_addr", subj="subject_line")
self.results = ""
self.results2 = "key1: value1\n"
self.results3 = "key2: value2\nkey1: value1\n"
self.results4 = "key1:\n key1a: value1a\n"
self.results5 = "key2: value2\nkey1:\n key1a: value1a\n"
self.sub = "key1: value1\n"
self.sub2 = "key3:\n key3A:\n key3AA: value3AA\n"
self.sub3 = "key2:\n key2B: value2B\n key2A: value2A\n"
self.results6 = self.sub2 + self.sub3 + self.sub
def mongo_stat(server, args_array, **kwargs):
"""Function: mongo_stat
Description: Creates and executes the mongostat utility program.
Arguments:
(input) server -> Database server instance.
(input) args_array -> Array of command line options and values.
(input) **kwargs:
req_arg -> List of options to add to cmd line.
opt_arg -> Dictionary of additional options to add.
ofile -> file name - Name of output file.
db_tbl database:table_name -> Mongo database and table name.
class_cfg -> Mongo server configuration.
"""
global SUBJ_LINE
subinst = gen_libs.get_inst(subprocess)
mail = None
mail_body = []
mode = "w"
mode2 = "wb"
indent = 4
args_array = dict(args_array)
outfile = kwargs.get("ofile", None)
no_std = args_array.get("-z", False)
cmd = mongo_libs.create_cmd(server, args_array, "mongostat", "-p",
**kwargs)
if args_array.get("-a", False):
mode = "a"
mode2 = "ab"
if args_array.get("-f", False):
indent = None
if "-b" in args_array:
cmd.append(args_array["-b"])
if args_array.get("-t", None):
mail = gen_class.setup_mail(args_array.get("-t"),
subj=args_array.get("-s", SUBJ_LINE))
if "-j" in args_array:
for row in get_data(cmd).rstrip().split("\n"):
# Evaluate "row" to dict format.
_, value = ast.literal_eval(row).popitem()
rep_set = value["set"]
rep_state = value["repl"]
time = value["time"]
value = gen_libs.rm_key(value, "time")
value = gen_libs.rm_key(value, "set")
value = gen_libs.rm_key(value, "repl")
data = {
"Server": server.name,
"AsOf": gen_libs.get_date() + " " + time,
"RepSet": rep_set,
"RepState": rep_state,
"PerfStats": value
}
mail_body.append(data)
_process_json(data, outfile, indent, no_std, mode, **kwargs)
# Append to file after first loop.
mode = "a"
if mail:
for line in mail_body:
mail.add_2_msg(json.dumps(line, indent=indent))
mail.send_mail(use_mailx=args_array.get("-u", False))
elif outfile:
with open(outfile, mode2) as f_name:
proc1 = subinst.Popen(cmd, stdout=f_name)
proc1.wait()
else:
proc1 = subinst.Popen(cmd)
proc1.wait()
def process_yum(args_array, yum, dict_key, func_name, **kwargs):
"""Function: process_yum
Description: Create and populate dictionary form based on the dict_key and
func_name. Send dictionary to output.
Arguments:
(input) args_array -> Array of command line options and values.
(input) yum -> Yum class instance.
(input) dict_key -> Dictionary key value.
(input) func_name -> Name of class method to call.
(input) **kwargs:
class_cfg -> Mongo server configuration.
(output) status -> Tuple on connection status.
status[0] - True|False - Mongo connection successful.
status[1] - Error message if Mongo connection failed.
"""
status = (True, None)
indent = 4
mode = "w"
args_array = dict(args_array)
os_distro = yum.get_distro()
data = {
"Server":
yum.get_hostname(),
"OsRelease":
os_distro[0] + " " + os_distro[1],
"AsOf":
datetime.datetime.strftime(datetime.datetime.now(),
"%Y-%m-%d %H:%M:%S"),
dict_key:
func_name()
}
ofile = args_array.get("-o", False)
db_tbl = args_array.get("-i", False)
class_cfg = kwargs.get("class_cfg", False)
if args_array.get("-a", False):
mode = "a"
if args_array.get("-f", False):
indent = None
if db_tbl and class_cfg:
dbn, tbl = db_tbl.split(":")
status = mongo_libs.ins_doc(class_cfg, dbn, tbl, data)
if not status[0]:
status = (status[0], "Mongo_Insert: " + status[1])
data = json.dumps(data, indent=indent)
if ofile:
gen_libs.write_file(ofile, mode, data)
if not args_array.get("-z", False):
gen_libs.display_data(data)
if args_array.get("-e", False):
mail = gen_class.setup_mail(args_array.get("-e"),
subj=args_array.get("-s", None))
mail.add_2_msg(data)
use_mailx = args_array.get("-u", False)
mail.send_mail(use_mailx=use_mailx)
return status
def mysql_stat(server, args_array, **kwargs):
"""Function: mysql_stat
Description: Setup for processing the mysql statistics and loop on getting
the MySQL statistics based on count and interval options.
Arguments:
(input) server -> Database server instance.
(input) args_array -> Array of command line options and values.
(input) **kwargs:
class_cfg -> Mongo server configuration.
"""
global SUBJ_LINE
args_array = dict(args_array)
ofile = args_array.get("-o", False)
db_tbl = args_array.get("-i", False)
json_fmt = args_array.get("-j", False)
no_std = args_array.get("-z", False)
mode = "w"
indent = 4
mail = None
if args_array.get("-a", False):
mode = "a"
if args_array.get("-f", False):
indent = None
if args_array.get("-t", None):
mail = gen_class.setup_mail(args_array.get("-t"),
subj=args_array.get("-s", SUBJ_LINE))
# List of performance statistics to be checked.
perf_list = [
"indb_buf_data", "indb_buf_tot", "indb_buf_data_pct", "indb_buf_drty",
"max_use_conn", "uptime_flush", "binlog_disk", "binlog_use",
"binlog_tot", "indb_buf_wait", "indb_log_wait", "indb_lock_avg",
"indb_lock_max", "indb_buf_read", "indb_buf_reqt", "indb_buf_read_pct",
"indb_buf_ahd", "indb_buf_evt", "indb_buf_evt_pct", "indb_buf_free",
"crt_tmp_tbls", "cur_conn", "uptime", "indb_buf", "indb_log_buf",
"max_conn"
]
# Loop iteration based on the -n option.
for item in range(0, int(args_array["-n"])):
mysql_stat_run(server,
perf_list,
db_tbl=db_tbl,
ofile=ofile,
json_fmt=json_fmt,
no_std=no_std,
mode=mode,
indent=indent,
mail=mail,
**kwargs)
# Append to file after first loop.
mode = "a"
# Do not sleep on the last loop.
if item != int(args_array["-n"]) - 1:
time.sleep(float(args_array["-b"]))
if mail:
mail.send_mail(use_mailx=args_array.get("-u", False))
def call_run_chk(args_array, func_dict, master, slaves, **kwargs):
"""Function: call_run_chk
Description: Calls the specified option function calls based on what
instances are available. Both the master and slaves
instances are passed to the functions, the functions will
determine what instances to be used.
Arguments:
(input) args_array -> Array of command line options and values.
(input) func_dict -> Dictionary list of functions and options.
(input) master -> Master instance.
(input) slaves -> Slave instances.
"""
args_array = dict(args_array)
func_dict = dict(func_dict)
slaves = list(slaves)
json_fmt = args_array.get("-j", False)
outfile = args_array.get("-o", None)
db_tbl = args_array.get("-b", None)
sup_std = args_array.get("-z", False)
mongo_cfg = None
mail = None
mode = "w"
indent = 4
if args_array.get("-a", False):
mode = "a"
if args_array.get("-f", False):
indent = None
if args_array.get("-m", None):
mongo_cfg = gen_libs.load_module(args_array["-m"], args_array["-d"])
if args_array.get("-t", None):
mail = gen_class.setup_mail(args_array.get("-t"),
subj=args_array.get("-u", None))
if "-A" in args_array:
for opt in func_dict["-A"]:
func_dict[opt](
master, slaves, json_fmt=json_fmt, ofile=outfile,
db_tbl=db_tbl, class_cfg=mongo_cfg, mail=mail, sup_std=sup_std,
mode=mode, indent=indent)
# The option is in func_dict but not under the ALL option and is not
# the ALL option itself.
for item in (opt for opt in args_array if opt in func_dict and
opt not in func_dict["-A"] and opt != "-A"):
func_dict[item](
master, slaves, json_fmt=json_fmt, ofile=outfile,
db_tbl=db_tbl, class_cfg=mongo_cfg, mail=mail,
sup_std=sup_std, mode=mode, indent=indent)
else:
# Intersect args_array & func_dict to find which functions to call.
for opt in set(args_array.keys()) & set(func_dict.keys()):
func_dict[opt](
master, slaves, json_fmt=json_fmt, ofile=outfile,
db_tbl=db_tbl, class_cfg=mongo_cfg, mail=mail, sup_std=sup_std,
mode=mode, indent=indent)
def run_program(args_array, sys_ign_db, **kwargs):
"""Function: run_program
Description: Creates class instance(s) and controls flow of the program.
Arguments:
(input) args_array -> Array of command line options and values.
(input) sys_ign_db -> List of system databases to ignore.
"""
global SUBJ_LINE
args_array = dict(args_array)
sys_ign_db = list(sys_ign_db)
mail = None
use_mailx = False
no_std = args_array.get("-z", False)
master = mysql_libs.create_instance(args_array["-c"], args_array["-d"],
mysql_class.MasterRep)
master.connect(silent=True)
slave = mysql_libs.create_instance(args_array["-r"], args_array["-d"],
mysql_class.SlaveRep)
slave.connect(silent=True)
if master.conn_msg or slave.conn_msg:
print("run_program: Error encountered with connection of master/slave")
print("\tMaster: %s" % (master.conn_msg))
print("\tSlave: %s" % (slave.conn_msg))
else:
if args_array.get("-e", None):
mail = gen_class.setup_mail(args_array.get("-e"),
subj=args_array.get("-s", SUBJ_LINE))
use_mailx = args_array.get("-u", False)
# Determine datatype of server_id and convert appropriately.
# Required for mysql.connector v1.1.6 as this version assigns the
# id to a different datatype then later mysql.connector versions.
slv_list = gen_libs.dict_2_list(master.show_slv_hosts(), "Server_id")
slv_id = str(slave.server_id) \
if isinstance(slv_list[0], str) else slave.server_id
# Is slave in replication with master
if slv_id in slv_list:
# Check specified tables in database
if "-t" in args_array:
setup_cmp(master,
slave,
sys_ign_db,
args_array["-B"],
args_array["-t"],
mail=mail,
no_std=no_std,
use_mailx=use_mailx,
**kwargs)
# Check single database
elif "-B" in args_array:
setup_cmp(master,
slave,
sys_ign_db,
args_array["-B"],
"",
mail=mail,
no_std=no_std,
use_mailx=use_mailx,
**kwargs)
# Check all tables in all databases
else:
setup_cmp(master,
slave,
sys_ign_db,
"",
"",
mail=mail,
no_std=no_std,
use_mailx=use_mailx,
**kwargs)
mysql_libs.disconnect(master, slave)
else:
mysql_libs.disconnect(master, slave)
print("Error: Replica is not in replication with Master.")