def post_process(proc_data, args_array, cfg, **kwargs): """Function: post_process Description: Send the data to the requested output and format if requested. Arguments: (input) proc_data -> Dictionary of process data. (input) args_array -> Dictionary of command line options and values. (input) cfg -> Configuration module settings. """ proc_data = dict(proc_data) args_array = dict(args_array) if "-n" not in args_array: if "-f" in args_array: gen_libs.display_data(proc_data) else: print(proc_data) if "-m" in args_array: mongo_libs.ins_doc(cfg, cfg.db, cfg.coll, proc_data)
def _process_queue(queue, body, r_key, cfg, f_name, log, **kwargs): """Function: _process_queue Description: Private function to process message queue. Arguments: (input) queue -> RabbitMQ queue. (input) body -> Message body. (input) r_key -> Routing key. (input) cfg -> Configuration settings module for the program. (input) f_name -> PDF file name. (input) log -> Log class instance. (output) status -> True|False - successfully extraction of data. """ global DTG_FORMAT status = True log.log_info("_process_queue: Extracting and processing metadata.") dtg = datetime.datetime.strftime(datetime.datetime.now(), DTG_FORMAT) metadata = { "FileName": os.path.basename(f_name), "Directory": queue["directory"], "DateTime": dtg } # Use the PyPDF2 module to extract data. status_pypdf2, final_data = get_pypdf2_data(f_name, cfg, log) if status_pypdf2: log.log_info("_process_queue: Adding metadata from pypdf2.") metadata = create_metadata(metadata, final_data) # Use the textract module to extract data. status_textract, final_data = get_textract_data(f_name, cfg, log) if status_textract: log.log_info("_process_queue: Adding metadata from textract.") metadata = create_metadata(metadata, final_data) # Use the pdfminer module to extract data. status_pdfminer, final_data = get_pdfminer_data(f_name, cfg, log) if status_pdfminer: log.log_info("_process_queue: Adding metadata from pdfminer.") metadata = create_metadata(metadata, final_data) if status_pypdf2 or status_textract or status_pdfminer: log.log_info("_process_queue: Insert metadata into MongoDB.") mongo_libs.ins_doc(cfg.mongo, cfg.mongo.dbs, cfg.mongo.tbl, metadata) log.log_info("_process_queue: Moving PDF to: %s" % (queue["directory"])) gen_libs.mv_file2(f_name, queue["directory"], os.path.basename(f_name)) else: log.log_err("_process_queue: All extractions methods failed.") status = False return status
def _process_json(data, outfile, indent, no_std, mode, **kwargs): """Function: _process_json Description: Private function for mongo_stat to process JSON data. Arguments: (input) data -> Dictionary of Mongo performance stat. (input) outfile -> Name of output file.. (input) indent -> Indentation setting for JSON format. (input) no_std -> Suppress standard out. (input) mode -> File write mode (append|write). (input) **kwargs: db_tbl -> Mongo database and table name. class_cfg -> Mongo server configuration. """ data = dict(data) if kwargs.get("db_tbl", False) and kwargs.get("class_cfg", False): dbn, tbl = kwargs.get("db_tbl").split(":") status = mongo_libs.ins_doc(kwargs.get("class_cfg"), dbn, tbl, data) if not status[0]: print("Insert error: %s" % (status[1])) if outfile: gen_libs.write_file(outfile, mode, json.dumps(data, indent=indent)) if not no_std: gen_libs.print_data(json.dumps(data, indent=indent))
def _process_json(outdata, **kwargs): """Function: _process_json Description: Private function for chk_mem_rep_lag(). Process JSON data. Arguments: (input) outdata -> JSON document from chk_mem_rep_lag function. (input) **kwargs: ofile -> file name - Name of output file. db_tbl -> database:collection - Name of db and collection. class_cfg -> Server class configuration settings. mail -> Mail instance. args_array -> Array of command line options and values. (output) status -> Tuple on connection status. status[0] - True|False - Connection successful. status[1] - Error message if connection failed. """ status = (True, None) mode = "w" indent = 4 mongo_cfg = kwargs.get("class_cfg", None) db_tbl = kwargs.get("db_tbl", None) ofile = kwargs.get("ofile", None) mail = kwargs.get("mail", None) args_array = dict(kwargs.get("args_array", {})) if args_array.get("-a", False): mode = "a" if args_array.get("-f", False): indent = None jdata = json.dumps(outdata, indent=indent) if mongo_cfg and db_tbl: dbs, tbl = db_tbl.split(":") status = mongo_libs.ins_doc(mongo_cfg, dbs, tbl, outdata) if not status[0]: status = (status[0], "_process_json: " + status[1]) if ofile: gen_libs.write_file(ofile, mode, jdata) if mail: mail.add_2_msg(jdata) mail.send_mail() if not args_array.get("-z", False): gen_libs.display_data(jdata) return status
def _process_json(outdata, **kwargs): """Function: _process_json Description: Private function for chk_slv_time(). Process JSON data. Arguments: (input) outdata -> JSON document of Check Slave Time output. (input) **kwargs: ofile -> file name - Name of output file. db_tbl -> database:collection - Name of db and collection. class_cfg -> Server class configuration settings. mail -> Mail instance. sup_std -> Suppress standard out. mode -> File write mode. indent -> Indentation level for JSON document. """ indent = kwargs.get("indent", None) jdata = json.dumps(outdata, indent=indent) mongo_cfg = kwargs.get("class_cfg", None) db_tbl = kwargs.get("db_tbl", None) ofile = kwargs.get("ofile", None) mail = kwargs.get("mail", None) mode = kwargs.get("mode", "w") if mongo_cfg and db_tbl: dbn, tbl = db_tbl.split(":") mongo_libs.ins_doc(mongo_cfg, dbn, tbl, outdata) if ofile: gen_libs.write_file(ofile, mode, jdata) if mail: mail.add_2_msg(jdata) mail.send_mail() if not kwargs.get("sup_std", False): gen_libs.print_data(jdata)
def _process_json(args_array, outdata, mode, **kwargs): """Function: _process_json Description: Private function for status to process json format data. Arguments: (input) args_array -> Dictionary of command line options. (input) outdata -> Dictionary of performance data. (input) mode -> File write mode. (input) **kwargs: ofile -> file name - Name of output file. db_tbl database:table_name -> Mongo database and table name. class_cfg -> Mongo server configuration. mail -> Mail instance. """ indent = None if args_array.get("-f", False) else 4 jdata = json.dumps(outdata, indent=indent) mongo_cfg = kwargs.get("class_cfg", None) db_tbl = kwargs.get("db_tbl", None) ofile = kwargs.get("ofile", None) mail = kwargs.get("mail", None) if mongo_cfg and db_tbl: dbs, tbl = db_tbl.split(":") conn_stats = mongo_libs.ins_doc(mongo_cfg, dbs, tbl, outdata) if not conn_stats[0]: print("Error: status.mongo_insert: %s" % (conn_stats[1])) if ofile: gen_libs.write_file(ofile, mode, jdata) if mail: mail.add_2_msg(jdata) mail.send_mail(use_mailx=args_array.get("-u", False)) if not args_array.get("-z", False): gen_libs.print_data(jdata)
def _process_std(outdata, **kwargs): """Function: _process_std Description: Private function for chk_mem_rep_lag(). Process standard out formatted data. Arguments: (input) outdata -> JSON document from chk_mem_rep_lag function. (input) **kwargs: suf -> Primary|Freshest Secondary who has latest date time. args_array -> Array of command line options and values. (output) status -> Tuple on connection status. status[0] - True|False - Connection successful. status[1] - Error message if connection failed. """ status = (True, None) mode = "w" mongo_cfg = kwargs.get("class_cfg", None) db_tbl = kwargs.get("db_tbl", None) ofile = kwargs.get("ofile", None) mail = kwargs.get("mail", None) args_array = dict(kwargs.get("args_array", {})) body = [] if args_array.get("-a", False): mode = "a" body.append("\nReplication lag for Replica set: %s." % (outdata["RepSet"])) for item in outdata["Slaves"]: body.append("\nSource: {0}".format(item["Name"])) body.append("\tsynced to: {0}".format(item["SyncTo"])) body.append("\t{0} secs ({1} hrs) behind the {2}".format( item["LagTime"], (item["LagTime"] / 36) / 100, kwargs["suf"])) if mongo_cfg and db_tbl: dbs, tbl = db_tbl.split(":") status = mongo_libs.ins_doc(mongo_cfg, dbs, tbl, outdata) if not status[0]: status = (status[0], "_process_std: " + status[1]) if ofile: f_hldr = gen_libs.openfile(ofile, mode) for line in body: gen_libs.write_file2(f_hldr, line) if mail: for line in body: mail.add_2_msg(line) mail.send_mail() if not args_array.get("-z", False): for item in body: print(item) return status
def process_yum(args_array, yum, dict_key, func_name, **kwargs): """Function: process_yum Description: Create and populate dictionary form based on the dict_key and func_name. Send dictionary to output. Arguments: (input) args_array -> Array of command line options and values. (input) yum -> Yum class instance. (input) dict_key -> Dictionary key value. (input) func_name -> Name of class method to call. (input) **kwargs: class_cfg -> Mongo server configuration. (output) status -> Tuple on connection status. status[0] - True|False - Mongo connection successful. status[1] - Error message if Mongo connection failed. """ status = (True, None) indent = 4 mode = "w" args_array = dict(args_array) os_distro = yum.get_distro() data = { "Server": yum.get_hostname(), "OsRelease": os_distro[0] + " " + os_distro[1], "AsOf": datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d %H:%M:%S"), dict_key: func_name() } ofile = args_array.get("-o", False) db_tbl = args_array.get("-i", False) class_cfg = kwargs.get("class_cfg", False) if args_array.get("-a", False): mode = "a" if args_array.get("-f", False): indent = None if db_tbl and class_cfg: dbn, tbl = db_tbl.split(":") status = mongo_libs.ins_doc(class_cfg, dbn, tbl, data) if not status[0]: status = (status[0], "Mongo_Insert: " + status[1]) data = json.dumps(data, indent=indent) if ofile: gen_libs.write_file(ofile, mode, data) if not args_array.get("-z", False): gen_libs.display_data(data) if args_array.get("-e", False): mail = gen_class.setup_mail(args_array.get("-e"), subj=args_array.get("-s", None)) mail.add_2_msg(data) use_mailx = args_array.get("-u", False) mail.send_mail(use_mailx=use_mailx) return status
def mysql_stat_run(server, perf_list=None, **kwargs): """Function: mysql_stat_run Description: Updated the class' server and performance statistics. Send to output in a number of different formats (e.g. standard, or JSON) and to a number of locations (e.g. standard out, database, and/or file). Arguments: (input) server -> Database server instance. (input) perf_list -> List of performance statistics. (input) **kwargs: class_cfg -> Mongo server configuration. indent -> Indentation level for JSON document. mode -> File write mode. db_tbl -> database:collection - Name of db and collection. ofile -> file name - Name of output file. no_std -> Suppress standard out. json_fmt -> True|False - convert output to JSON format. mail -> Mail class instance. """ json_fmt = kwargs.get("json_fmt", False) indent = kwargs.get("indent", 4) mongo_cfg = kwargs.get("class_cfg", None) db_tbl = kwargs.get("db_tbl", None) ofile = kwargs.get("ofile", None) mode = kwargs.get("mode", "w") no_std = kwargs.get("no_std", False) mail = kwargs.get("mail", None) if perf_list is None: perf_list = [] else: perf_list = list(perf_list) server.upd_srv_stat() server.upd_srv_perf() data = { "Server": server.name, "AsOf": gen_libs.get_date() + " " + gen_libs.get_time(), "PerfStats": {} } for item in perf_list: data["PerfStats"].update({item: getattr(server, item)}) if mongo_cfg and db_tbl: dbn, tbl = db_tbl.split(":") status = mongo_libs.ins_doc(mongo_cfg, dbn, tbl, data) if not status[0]: print("Insert error: %s" % (status[1])) if json_fmt: jdata = json.dumps(data, indent=indent) _process_json(jdata, ofile, mail, mode, no_std) else: err_flag, err_msg = gen_libs.print_dict(data, ofile=ofile, no_std=no_std, mode=mode) if err_flag: print(err_msg) if mail: convert_dict(data, mail)
def status(server, args_array, **kwargs): """Function: status Description: Retrieves a number of database status variables and sends them out either in standard out (print) or to a JSON format which is printed and/or insert into the database. Arguments: (input) server -> Database server instance. (input) args_array -> Array of command line options and values. (input) **kwargs: ofile -> file name - Name of output file. db_tbl database:table_name -> Mongo database and table name. class_cfg -> Mongo Rep Set server configuration. mail -> Mail instance. (output) err_flag -> True|False - If an error has occurred. (output) err_msg -> Error message. """ err_flag = False err_msg = None mode = "w" indent = 4 args_array = dict(args_array) server.upd_srv_stat() outdata = {"Application": "MongoDB", "Server": server.name, "AsOf": datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d %H:%M:%S")} outdata.update({"Memory": {"CurrentUsage": server.cur_mem, "MaxUsage": server.max_mem, "PercentUsed": server.prct_mem}, "UpTime": server.days_up, "Connections": {"CurrentConnected": server.cur_conn, "MaxConnections": server.max_conn, "PercentUsed": server.prct_conn}}) ofile = kwargs.get("ofile", None) mail = kwargs.get("mail", None) mongo_cfg = kwargs.get("class_cfg", None) db_tbl = kwargs.get("db_tbl", None) if args_array.get("-a", False): mode = "a" if args_array.get("-g", False): indent = None if "-j" in args_array: outdata = json.dumps(outdata, indent=indent) if mongo_cfg and db_tbl: dbn, tbl = db_tbl.split(":") if isinstance(outdata, dict): state = mongo_libs.ins_doc(mongo_cfg, dbn, tbl, outdata) else: state = mongo_libs.ins_doc(mongo_cfg, dbn, tbl, ast.literal_eval(outdata)) if not state[0]: err_flag = True err_msg = "Inserting into Mongo database: %s" % state[1] if ofile and "-j" in args_array: gen_libs.write_file(ofile, mode, outdata) elif ofile: gen_libs.display_data(outdata, f_hdlr=gen_libs.openfile(ofile, mode)) if mail and isinstance(outdata, dict): mail.add_2_msg(json.dumps(outdata, indent=indent)) mail.send_mail() elif mail: mail.add_2_msg(outdata) mail.send_mail() if not args_array.get("-z", False): gen_libs.display_data(outdata) return err_flag, err_msg