def post_process(proc_data, args_array, cfg, **kwargs):
"""Function: post_process
Description: Send the data to the requested output and format if
requested.
Arguments:
(input) proc_data -> Dictionary of process data.
(input) args_array -> Dictionary of command line options and values.
(input) cfg -> Configuration module settings.
"""
proc_data = dict(proc_data)
args_array = dict(args_array)
if "-n" not in args_array:
if "-f" in args_array:
gen_libs.display_data(proc_data)
else:
print(proc_data)
if "-m" in args_array:
mongo_libs.ins_doc(cfg, cfg.db, cfg.coll, proc_data)
def _process_queue(queue, body, r_key, cfg, f_name, log, **kwargs):
"""Function: _process_queue
Description: Private function to process message queue.
Arguments:
(input) queue -> RabbitMQ queue.
(input) body -> Message body.
(input) r_key -> Routing key.
(input) cfg -> Configuration settings module for the program.
(input) f_name -> PDF file name.
(input) log -> Log class instance.
(output) status -> True|False - successfully extraction of data.
"""
global DTG_FORMAT
status = True
log.log_info("_process_queue: Extracting and processing metadata.")
dtg = datetime.datetime.strftime(datetime.datetime.now(), DTG_FORMAT)
metadata = {
"FileName": os.path.basename(f_name),
"Directory": queue["directory"],
"DateTime": dtg
}
# Use the PyPDF2 module to extract data.
status_pypdf2, final_data = get_pypdf2_data(f_name, cfg, log)
if status_pypdf2:
log.log_info("_process_queue: Adding metadata from pypdf2.")
metadata = create_metadata(metadata, final_data)
# Use the textract module to extract data.
status_textract, final_data = get_textract_data(f_name, cfg, log)
if status_textract:
log.log_info("_process_queue: Adding metadata from textract.")
metadata = create_metadata(metadata, final_data)
# Use the pdfminer module to extract data.
status_pdfminer, final_data = get_pdfminer_data(f_name, cfg, log)
if status_pdfminer:
log.log_info("_process_queue: Adding metadata from pdfminer.")
metadata = create_metadata(metadata, final_data)
if status_pypdf2 or status_textract or status_pdfminer:
log.log_info("_process_queue: Insert metadata into MongoDB.")
mongo_libs.ins_doc(cfg.mongo, cfg.mongo.dbs, cfg.mongo.tbl, metadata)
log.log_info("_process_queue: Moving PDF to: %s" %
(queue["directory"]))
gen_libs.mv_file2(f_name, queue["directory"], os.path.basename(f_name))
else:
log.log_err("_process_queue: All extractions methods failed.")
status = False
return status
def _process_json(data, outfile, indent, no_std, mode, **kwargs):
"""Function: _process_json
Description: Private function for mongo_stat to process JSON data.
Arguments:
(input) data -> Dictionary of Mongo performance stat.
(input) outfile -> Name of output file..
(input) indent -> Indentation setting for JSON format.
(input) no_std -> Suppress standard out.
(input) mode -> File write mode (append|write).
(input) **kwargs:
db_tbl -> Mongo database and table name.
class_cfg -> Mongo server configuration.
"""
data = dict(data)
if kwargs.get("db_tbl", False) and kwargs.get("class_cfg", False):
dbn, tbl = kwargs.get("db_tbl").split(":")
status = mongo_libs.ins_doc(kwargs.get("class_cfg"), dbn, tbl, data)
if not status[0]:
print("Insert error: %s" % (status[1]))
if outfile:
gen_libs.write_file(outfile, mode, json.dumps(data, indent=indent))
if not no_std:
gen_libs.print_data(json.dumps(data, indent=indent))
def _process_json(outdata, **kwargs):
"""Function: _process_json
Description: Private function for chk_mem_rep_lag(). Process JSON data.
Arguments:
(input) outdata -> JSON document from chk_mem_rep_lag function.
(input) **kwargs:
ofile -> file name - Name of output file.
db_tbl -> database:collection - Name of db and collection.
class_cfg -> Server class configuration settings.
mail -> Mail instance.
args_array -> Array of command line options and values.
(output) status -> Tuple on connection status.
status[0] - True|False - Connection successful.
status[1] - Error message if connection failed.
"""
status = (True, None)
mode = "w"
indent = 4
mongo_cfg = kwargs.get("class_cfg", None)
db_tbl = kwargs.get("db_tbl", None)
ofile = kwargs.get("ofile", None)
mail = kwargs.get("mail", None)
args_array = dict(kwargs.get("args_array", {}))
if args_array.get("-a", False):
mode = "a"
if args_array.get("-f", False):
indent = None
jdata = json.dumps(outdata, indent=indent)
if mongo_cfg and db_tbl:
dbs, tbl = db_tbl.split(":")
status = mongo_libs.ins_doc(mongo_cfg, dbs, tbl, outdata)
if not status[0]:
status = (status[0], "_process_json: " + status[1])
if ofile:
gen_libs.write_file(ofile, mode, jdata)
if mail:
mail.add_2_msg(jdata)
mail.send_mail()
if not args_array.get("-z", False):
gen_libs.display_data(jdata)
return status
def _process_json(outdata, **kwargs):
"""Function: _process_json
Description: Private function for chk_slv_time(). Process JSON data.
Arguments:
(input) outdata -> JSON document of Check Slave Time output.
(input) **kwargs:
ofile -> file name - Name of output file.
db_tbl -> database:collection - Name of db and collection.
class_cfg -> Server class configuration settings.
mail -> Mail instance.
sup_std -> Suppress standard out.
mode -> File write mode.
indent -> Indentation level for JSON document.
"""
indent = kwargs.get("indent", None)
jdata = json.dumps(outdata, indent=indent)
mongo_cfg = kwargs.get("class_cfg", None)
db_tbl = kwargs.get("db_tbl", None)
ofile = kwargs.get("ofile", None)
mail = kwargs.get("mail", None)
mode = kwargs.get("mode", "w")
if mongo_cfg and db_tbl:
dbn, tbl = db_tbl.split(":")
mongo_libs.ins_doc(mongo_cfg, dbn, tbl, outdata)
if ofile:
gen_libs.write_file(ofile, mode, jdata)
if mail:
mail.add_2_msg(jdata)
mail.send_mail()
if not kwargs.get("sup_std", False):
gen_libs.print_data(jdata)
def _process_json(args_array, outdata, mode, **kwargs):
"""Function: _process_json
Description: Private function for status to process json format data.
Arguments:
(input) args_array -> Dictionary of command line options.
(input) outdata -> Dictionary of performance data.
(input) mode -> File write mode.
(input) **kwargs:
ofile -> file name - Name of output file.
db_tbl database:table_name -> Mongo database and table name.
class_cfg -> Mongo server configuration.
mail -> Mail instance.
"""
indent = None if args_array.get("-f", False) else 4
jdata = json.dumps(outdata, indent=indent)
mongo_cfg = kwargs.get("class_cfg", None)
db_tbl = kwargs.get("db_tbl", None)
ofile = kwargs.get("ofile", None)
mail = kwargs.get("mail", None)
if mongo_cfg and db_tbl:
dbs, tbl = db_tbl.split(":")
conn_stats = mongo_libs.ins_doc(mongo_cfg, dbs, tbl, outdata)
if not conn_stats[0]:
print("Error: status.mongo_insert: %s" % (conn_stats[1]))
if ofile:
gen_libs.write_file(ofile, mode, jdata)
if mail:
mail.add_2_msg(jdata)
mail.send_mail(use_mailx=args_array.get("-u", False))
if not args_array.get("-z", False):
gen_libs.print_data(jdata)
def _process_std(outdata, **kwargs):
"""Function: _process_std
Description: Private function for chk_mem_rep_lag(). Process standard out
formatted data.
Arguments:
(input) outdata -> JSON document from chk_mem_rep_lag function.
(input) **kwargs:
suf -> Primary|Freshest Secondary who has latest date time.
args_array -> Array of command line options and values.
(output) status -> Tuple on connection status.
status[0] - True|False - Connection successful.
status[1] - Error message if connection failed.
"""
status = (True, None)
mode = "w"
mongo_cfg = kwargs.get("class_cfg", None)
db_tbl = kwargs.get("db_tbl", None)
ofile = kwargs.get("ofile", None)
mail = kwargs.get("mail", None)
args_array = dict(kwargs.get("args_array", {}))
body = []
if args_array.get("-a", False):
mode = "a"
body.append("\nReplication lag for Replica set: %s." % (outdata["RepSet"]))
for item in outdata["Slaves"]:
body.append("\nSource: {0}".format(item["Name"]))
body.append("\tsynced to: {0}".format(item["SyncTo"]))
body.append("\t{0} secs ({1} hrs) behind the {2}".format(
item["LagTime"], (item["LagTime"] / 36) / 100, kwargs["suf"]))
if mongo_cfg and db_tbl:
dbs, tbl = db_tbl.split(":")
status = mongo_libs.ins_doc(mongo_cfg, dbs, tbl, outdata)
if not status[0]:
status = (status[0], "_process_std: " + status[1])
if ofile:
f_hldr = gen_libs.openfile(ofile, mode)
for line in body:
gen_libs.write_file2(f_hldr, line)
if mail:
for line in body:
mail.add_2_msg(line)
mail.send_mail()
if not args_array.get("-z", False):
for item in body:
print(item)
return status
def process_yum(args_array, yum, dict_key, func_name, **kwargs):
"""Function: process_yum
Description: Create and populate dictionary form based on the dict_key and
func_name. Send dictionary to output.
Arguments:
(input) args_array -> Array of command line options and values.
(input) yum -> Yum class instance.
(input) dict_key -> Dictionary key value.
(input) func_name -> Name of class method to call.
(input) **kwargs:
class_cfg -> Mongo server configuration.
(output) status -> Tuple on connection status.
status[0] - True|False - Mongo connection successful.
status[1] - Error message if Mongo connection failed.
"""
status = (True, None)
indent = 4
mode = "w"
args_array = dict(args_array)
os_distro = yum.get_distro()
data = {
"Server":
yum.get_hostname(),
"OsRelease":
os_distro[0] + " " + os_distro[1],
"AsOf":
datetime.datetime.strftime(datetime.datetime.now(),
"%Y-%m-%d %H:%M:%S"),
dict_key:
func_name()
}
ofile = args_array.get("-o", False)
db_tbl = args_array.get("-i", False)
class_cfg = kwargs.get("class_cfg", False)
if args_array.get("-a", False):
mode = "a"
if args_array.get("-f", False):
indent = None
if db_tbl and class_cfg:
dbn, tbl = db_tbl.split(":")
status = mongo_libs.ins_doc(class_cfg, dbn, tbl, data)
if not status[0]:
status = (status[0], "Mongo_Insert: " + status[1])
data = json.dumps(data, indent=indent)
if ofile:
gen_libs.write_file(ofile, mode, data)
if not args_array.get("-z", False):
gen_libs.display_data(data)
if args_array.get("-e", False):
mail = gen_class.setup_mail(args_array.get("-e"),
subj=args_array.get("-s", None))
mail.add_2_msg(data)
use_mailx = args_array.get("-u", False)
mail.send_mail(use_mailx=use_mailx)
return status
def mysql_stat_run(server, perf_list=None, **kwargs):
"""Function: mysql_stat_run
Description: Updated the class' server and performance statistics. Send
to output in a number of different formats (e.g. standard, or JSON) and
to a number of locations (e.g. standard out, database, and/or file).
Arguments:
(input) server -> Database server instance.
(input) perf_list -> List of performance statistics.
(input) **kwargs:
class_cfg -> Mongo server configuration.
indent -> Indentation level for JSON document.
mode -> File write mode.
db_tbl -> database:collection - Name of db and collection.
ofile -> file name - Name of output file.
no_std -> Suppress standard out.
json_fmt -> True|False - convert output to JSON format.
mail -> Mail class instance.
"""
json_fmt = kwargs.get("json_fmt", False)
indent = kwargs.get("indent", 4)
mongo_cfg = kwargs.get("class_cfg", None)
db_tbl = kwargs.get("db_tbl", None)
ofile = kwargs.get("ofile", None)
mode = kwargs.get("mode", "w")
no_std = kwargs.get("no_std", False)
mail = kwargs.get("mail", None)
if perf_list is None:
perf_list = []
else:
perf_list = list(perf_list)
server.upd_srv_stat()
server.upd_srv_perf()
data = {
"Server": server.name,
"AsOf": gen_libs.get_date() + " " + gen_libs.get_time(),
"PerfStats": {}
}
for item in perf_list:
data["PerfStats"].update({item: getattr(server, item)})
if mongo_cfg and db_tbl:
dbn, tbl = db_tbl.split(":")
status = mongo_libs.ins_doc(mongo_cfg, dbn, tbl, data)
if not status[0]:
print("Insert error: %s" % (status[1]))
if json_fmt:
jdata = json.dumps(data, indent=indent)
_process_json(jdata, ofile, mail, mode, no_std)
else:
err_flag, err_msg = gen_libs.print_dict(data,
ofile=ofile,
no_std=no_std,
mode=mode)
if err_flag:
print(err_msg)
if mail:
convert_dict(data, mail)
def status(server, args_array, **kwargs):
"""Function: status
Description: Retrieves a number of database status variables and sends
them out either in standard out (print) or to a JSON format which
is printed and/or insert into the database.
Arguments:
(input) server -> Database server instance.
(input) args_array -> Array of command line options and values.
(input) **kwargs:
ofile -> file name - Name of output file.
db_tbl database:table_name -> Mongo database and table name.
class_cfg -> Mongo Rep Set server configuration.
mail -> Mail instance.
(output) err_flag -> True|False - If an error has occurred.
(output) err_msg -> Error message.
"""
err_flag = False
err_msg = None
mode = "w"
indent = 4
args_array = dict(args_array)
server.upd_srv_stat()
outdata = {"Application": "MongoDB",
"Server": server.name,
"AsOf": datetime.datetime.strftime(datetime.datetime.now(),
"%Y-%m-%d %H:%M:%S")}
outdata.update({"Memory": {"CurrentUsage": server.cur_mem,
"MaxUsage": server.max_mem,
"PercentUsed": server.prct_mem},
"UpTime": server.days_up,
"Connections": {"CurrentConnected": server.cur_conn,
"MaxConnections": server.max_conn,
"PercentUsed": server.prct_conn}})
ofile = kwargs.get("ofile", None)
mail = kwargs.get("mail", None)
mongo_cfg = kwargs.get("class_cfg", None)
db_tbl = kwargs.get("db_tbl", None)
if args_array.get("-a", False):
mode = "a"
if args_array.get("-g", False):
indent = None
if "-j" in args_array:
outdata = json.dumps(outdata, indent=indent)
if mongo_cfg and db_tbl:
dbn, tbl = db_tbl.split(":")
if isinstance(outdata, dict):
state = mongo_libs.ins_doc(mongo_cfg, dbn, tbl, outdata)
else:
state = mongo_libs.ins_doc(mongo_cfg, dbn, tbl,
ast.literal_eval(outdata))
if not state[0]:
err_flag = True
err_msg = "Inserting into Mongo database: %s" % state[1]
if ofile and "-j" in args_array:
gen_libs.write_file(ofile, mode, outdata)
elif ofile:
gen_libs.display_data(outdata, f_hdlr=gen_libs.openfile(ofile, mode))
if mail and isinstance(outdata, dict):
mail.add_2_msg(json.dumps(outdata, indent=indent))
mail.send_mail()
elif mail:
mail.add_2_msg(outdata)
mail.send_mail()
if not args_array.get("-z", False):
gen_libs.display_data(outdata)
return err_flag, err_msg
