def non_proc_msg(rmq, log, cfg, data, subj, r_key, **kwargs):
"""Function: non_proc_msg
Description: Process non-processed messages.
Arguments:
(input) rmq -> RabbitMQ class instance.
(input) log -> Log class instance.
(input) cfg -> Configuration settings module for the program.
(input) data -> Body of message that was not processed.
(input) subj -> Email subject line.
(input) r_key -> Routing key for message.
"""
global DTG_FORMAT
log.log_info("non_proc_msg: Processing failed message: Routing Key: %s" %
(r_key))
frm_line = getpass.getuser() + "@" + socket.gethostname()
rdtg = datetime.datetime.now()
msecs = str(rdtg.microsecond / 100)
dtg = datetime.datetime.strftime(rdtg, DTG_FORMAT) + "." + msecs
f_name = rmq.exchange + "_" + r_key + "_" + dtg + ".txt"
f_path = os.path.join(cfg.message_dir, f_name)
subj = "rmq_metadata: " + subj
line1 = "RabbitMQ message was not processed due to: %s" % (subj)
line2 = "Exchange: %s, Routing Key: %s" % (rmq.exchange, r_key)
line3 = "The body of the message is encoded data."
line4 = "Body of message saved to: %s" % (f_path)
if cfg.to_line:
log.log_info("Sending email to: %s..." % (cfg.to_line))
email = gen_class.Mail(cfg.to_line, subj, frm_line)
email.add_2_msg(line1)
email.add_2_msg(line2)
email.add_2_msg(line3)
email.add_2_msg(line4)
email.send_mail()
else:
log.log_warn("No email being sent as TO line is empty.")
log.log_err(line1)
log.log_err(line2)
log.log_err(line3)
log.log_err(line4)
gen_libs.write_file(f_path, data=data, mode="w")
def update_marker(args_array, line, **kwargs):
"""Function: update_marker
Description: Writes the last line of the log to the marker file, if the
marker option is selected and not the no_update option.
Arguments:
(input) args_array -> Dictionary of command line options and values.
(input) line -> Last line of log.
"""
args_array = dict(args_array)
if "-m" in args_array and "-n" not in args_array:
gen_libs.write_file(args_array["-m"], mode="w", data=line)
def process_msg(rmq, log, cfg, method, body, **kwargs):
"""Function: process_msg
Description: Process message from RabbitMQ queue.
Arguments:
(input) rmq -> RabbitMQ class instance.
(input) log -> Log class instance.
(input) cfg -> Configuration settings module for the program.
(input) method -> Delivery properties.
(input) body -> Message body.
"""
global DTG_FORMAT
r_key = method.routing_key
queue = None
log.log_info("process_msg: Processing message body from Routing Key: %s" %
(r_key))
for item in cfg.queue_list:
if item["routing_key"] == r_key:
queue = item
if queue["archive"] and cfg.archive_dir:
rdtg = datetime.datetime.now()
msecs = str(rdtg.microsecond / 100)
dtg = datetime.datetime.strftime(rdtg, DTG_FORMAT) + \
"." + msecs
f_name = rmq.exchange + "_" + queue["routing_key"] + "_" + \
dtg + ".body"
f_path = os.path.join(cfg.archive_dir, f_name)
log.log_info("process_msg: Archiving message to: %s" %
(f_path))
gen_libs.write_file(f_path, data=body, mode="w")
break
if queue:
_convert_data(rmq, log, cfg, queue, body, r_key)
else:
non_proc_msg(rmq, log, cfg, body, "No queue detected", r_key)
def _process_queue(queue, data, r_key, x_name, **kwargs):
"""Function: _process_queue
Description: Private function to process message queue.
Arguments:
(input) queue -> RabbitMQ queue.
(input) data -> Converted message body.
(input) r_key -> Routing key.
(input) x_name -> Exchange name.
"""
k_name = ""
ext = ""
indent = 4
dtg = ""
if queue["key"] and queue["key"] in data and queue["stype"] == "dict":
k_name = str(data[queue["key"]].split(".")[0])
if queue["ext"]:
ext = "." + queue["ext"]
if queue["flatten"]:
indent = None
if queue["dtg"]:
dtg = datetime.datetime.strftime(datetime.datetime.now(),
"%Y%m%d_%H%M%S")
elif queue["date"]:
dtg = datetime.datetime.strftime(datetime.datetime.now(), "%Y%m%d")
if isinstance(data, dict):
data = json.dumps(data, indent=indent)
f_name = queue["prename"] + k_name + queue["postname"] + dtg
if not f_name:
f_name = "Default_" + x_name + "_" + r_key
f_name = os.path.join(queue["directory"], f_name + ext)
gen_libs.write_file(fname=f_name, mode=queue["mode"], data=data)
def non_proc_msg(rmq, log, cfg, data, subj, r_key, **kwargs):
"""Function: non_proc_msg
Description: Process non-processed messages.
Arguments:
(input) rmq -> RabbitMQ class instance.
(input) log -> Log class instance.
(input) cfg -> Configuration settings module for the program.
(input) data -> Body of message that was not processed.
(input) subj -> Email subject line.
(input) r_key -> Routing key for message.
"""
log.log_info(
"non_proc_msg: Processing failed message from Routing Key: %s" %
(r_key))
frm_line = getpass.getuser() + "@" + socket.gethostname()
rdtg = datetime.datetime.now()
msecs = str(rdtg.microsecond / 1000)
dtg = datetime.datetime.strftime(rdtg, "%Y-%m-%d_%H:%M:%S") + "." + msecs
f_name = rmq.exchange + "_" + r_key + "_" + dtg + ".txt"
f_path = os.path.join(cfg.message_dir, f_name)
subj = "rmq_2_sysmon: " + subj
if cfg.to_line:
log.log_info("Sending email to: %s..." % (cfg.to_line))
email = gen_class.Mail(cfg.to_line, subj, frm_line)
email.add_2_msg(data)
email.send_mail()
else:
log.log_warn("No email being sent as TO line is empty.")
log.log_err("RabbitMQ message was not processed due to: %s" % (subj))
log.log_info("Saving message to: %s" % (f_path))
gen_libs.write_file(f_path,
data="Exchange: %s, Routing Key: %s" %
(rmq.exchange, r_key))
gen_libs.write_file(f_path, data=data)
def _process_json(outdata, **kwargs):
"""Function: _process_json
Description: Private function for chk_slv_time(). Process JSON data.
Arguments:
(input) outdata -> JSON document of Check Slave Time output.
(input) **kwargs:
ofile -> file name - Name of output file.
db_tbl -> database:collection - Name of db and collection.
class_cfg -> Server class configuration settings.
mail -> Mail instance.
sup_std -> Suppress standard out.
mode -> File write mode.
indent -> Indentation level for JSON document.
"""
indent = kwargs.get("indent", None)
jdata = json.dumps(outdata, indent=indent)
mongo_cfg = kwargs.get("class_cfg", None)
db_tbl = kwargs.get("db_tbl", None)
ofile = kwargs.get("ofile", None)
mail = kwargs.get("mail", None)
mode = kwargs.get("mode", "w")
if mongo_cfg and db_tbl:
dbn, tbl = db_tbl.split(":")
mongo_libs.ins_doc(mongo_cfg, dbn, tbl, outdata)
if ofile:
gen_libs.write_file(ofile, mode, jdata)
if mail:
mail.add_2_msg(jdata)
mail.send_mail()
if not kwargs.get("sup_std", False):
gen_libs.print_data(jdata)
def _convert_data(rmq, log, cfg, queue, body, r_key, **kwargs):
"""Function: _convert_data
Description: Private function to process message queue.
Arguments:
(input) rmq -> RabbitMQ class instance.
(input) log -> Log class instance.
(input) cfg -> Configuration settings module for the program.
(input) queue -> RabbitMQ queue.
(input) body -> Message body.
(input) r_key -> Routing key.
"""
prename = ""
postname = ""
ext = ""
log.log_info("_convert_data: Converting data in message body.")
rdtg = datetime.datetime.now()
msecs = str(rdtg.microsecond / 100)
dtg = datetime.datetime.strftime(rdtg, "%Y%m%d%H%M%S") + "." + msecs
t_filename = "tmp_" + rmq.exchange + "_" + r_key + "_" + dtg + ".txt"
t_file = os.path.join(cfg.tmp_dir, t_filename)
if queue["prename"]:
prename = queue["prename"] + "_"
if queue["postname"]:
postname = "_" + queue["postname"]
if queue["ext"]:
ext = "." + queue["ext"]
f_filename = prename + rmq.exchange + "_" + r_key + "_" + dtg + \
postname + ext
f_name = os.path.join(cfg.tmp_dir, f_filename)
log.log_info("Starting processing of: %s" % (f_name))
gen_libs.write_file(t_file, data=body, mode="w")
if queue["stype"] == "encoded":
log.log_info("_convert_data: Decoding data in message body.")
base64.decode(open(t_file, 'rb'), open(f_name, 'wb'))
os.remove(t_file)
else:
log.log_info("_convert_data: No encoding setting detected.")
gen_libs.rename_file(t_filename, f_filename, cfg.tmp_dir)
status = _process_queue(queue, body, r_key, cfg, f_name, log)
if status:
log.log_info("Finished processing of: %s" % (f_filename))
else:
log.log_err("All extractions failed on: %s" % (f_filename))
log.log_info("Body of message being saved to a file - see below")
non_proc_msg(rmq, log, cfg, body, "All extractions failed", r_key)
os.remove(f_name)
log.log_info("Cleanup of temporary files completed.")
log.log_info("Finished processing of: %s" % (f_filename))