def archive_collections(CONFIG, age=90): logger = logging.getLogger("collection_archive") # Parse my General Configuration if isinstance(CONFIG, dict): config_items = CONFIG elif isinstance(CONFIG, str): config_items = manoward.get_manoward(explicit_config=CONFIG) else: raise TypeError("No Configuration Given.") db_conn = manoward.get_conn( config_items, prefix="store_", tojq=".database", ac_def=True) cur = db_conn.cursor() archive_ts = int(time.time()) logger.debug("Archive ts: {}".format(archive_ts)) populate_archive_sql = '''REPLACE INTO collection_archive SELECT * FROM collection WHERE last_update < FROM_UNIXTIME(%s) - interval %s DAY ; ''' remove_overachieving_sql = '''DELETE FROM collection WHERE last_update < FROM_UNIXTIME(%s) - interval %s DAY ; ''' archive_args = [archive_ts, age] copy_action = manoward.run_query(cur, populate_archive_sql, args=archive_args, require_results=False, do_abort=False) if copy_action["has_error"] is True: logger.error("{}Had an Error When Running Archive. Ignoring Delete{}".format( Fore.RED, Style.RESET_ALL)) else: # Run Delete logger.info("Archive Worked Swimmingly. Let's Go Ahead and Delete.") delete_action = manoward.run_query(cur, remove_overachieving_sql, args=archive_args, require_results=False, do_abort=False) if delete_action["has_error"] is True: logger.error("{}Error when deleting the Excess.{}".format( Fore.RED, Style.RESET_ALL)) else: logger.info("{}Collection Table Archived {}".format( Fore.GREEN, Style.RESET_ALL))
def get_current_table_data(db_config, table): db_conn = manoward.get_conn(db_config, prefix="analyze_", tojq=".database", ac_def=True) cur = db_conn.cursor() grab_current_table_list = list() grab_current_table_list.append( "SELECT {0}_id, {0}_text, fk_audits_id, {0}_passed, {0}_failed, {0}_exempt " .format(table)) grab_current_table_list.append("FROM audits_by_{0}".format(table)) grab_current_table_list.append( "WHERE {0}_last_audit >= now() - INTERVAL %s SECOND".format(table)) grab_current_table_list.append( "group by {0}_text, fk_audits_id".format(table)) grab_ctl_args = [db_config["collate"]["freshseconds"]] grab_current_table_query = " ".join(grab_current_table_list) # print(grab_current_table_query) try: cur.execute(grab_current_table_query, grab_ctl_args) except Exception as grab_ctl_query_error: logger.error( "Unable to grab current table list table {}".format(table)) else: if cur.rowcount: query_results_list = cur.fetchall() else: query_results_list = list() finally: cur.close() return query_results_list
def grab_all_sapi(config_items): logger = logging.getLogger("sapicheck") all_hosts = list() db_conn = manoward.get_conn( config_items, prefix="store_", tojq=".database", ac_def=True) db_cur = db_conn.cursor(pymysql.cursors.DictCursor) all_sapi_hosts_sql = "select hostname from sapiActiveHosts where last_updated >= (now() - INTERVAL 3 DAY) " try: db_cur.execute(all_sapi_hosts_sql) results = db_cur.fetchall() except Exception as sapi_query_error: logger.warning("{}{}Unable to Grab all SAPI Hosts.{}".format(Back.WHITE, Fore.RED, Style.RESET_ALL)) logger.debug("Error : {}".format(sapi_query_error)) else: for host in results: # Pop host onto array all_hosts.append(host["hostname"]) # Retun the hosts I've found, if non will return an empty list. finally: db_cur.close() db_conn.close() return all_hosts
def before_request(): try: g.db = manoward.get_conn(config_items, prefix="api_", tojq=".database", ac_def=True) g.logger = logger g.debug = FDEBUG except Exception as connection_error: logger.debug("Connection to DB Error Abandoing Connection Error.") return str(connection_error) # Endpoint Authorization List of Endosements and Restrictions that Define what you may and may not access # For endpoints with fine grained controls g.session_endorsements = list() g.session_restrictions = list() # Open a New Cursor for this Request # Get Username username = "******" try: auth_header = request.headers.get("Authorization") uname_pass_64 = auth_header.split()[1] decoded_uname_pass = base64.b64decode(uname_pass_64).decode( "utf-8") username = decoded_uname_pass.split(":")[0] except Exception as no_auth_error: logger.debug( "No Authentication token given, either local access or IP whitelist : {}" .format(no_auth_error)) username = "******" g.session_endorsements.append(("conntype", "whitelist")) g.session_restrictions.append(("conntype", "whitelist")) logger.warning("Local Flask or Whitelist IP detected.") else: # Parsing was successful add ldap endorsment g.session_endorsements.append(("conntype", "ldap")) g.session_restrictions.append(("conntype", "ldap")) finally: logger.debug("User {} Connected a Session.".format(username)) g.USERNAME = username # Robot Authentication try: robot_header = ast.literal_eval( request.headers.get("robotauth", "False")) logger.debug("Robot Header : {}".format(robot_header)) if robot_header is True: # I need to do robot auth username, apikey = auth_header.split(':') g.USERNAME = username auth_cursor = g.db.cursor(pymysql.cursors.DictCursor) # Integrating Token Types. For now new tokens types will be processed # here. In the future this "endorsements" section will be replaced # with logic to look at our new centralized authorization system. anyvalid = False for tokentype in ["robot", "sapi", "ipintel"]: key_valid = validate_key(username=username, giventoken=apikey, dbcur=auth_cursor, tokentype=tokentype) print(tokentype, key_valid) if key_valid is True: # Add Robots Endorsement and Restriction g.session_endorsements.append(("conntype", tokentype)) g.session_restrictions.append(("conntype", tokentype)) anyvalid = True # No break loop break if anyvalid is False: # No Valid Token was found logger.warning("Robot Token Out of Date.") abort(403) auth_cursor.close() else: # This isn't a robot call pass except AttributeError as attribute_error: logger.error( "Attribute Error parsing Robot Items. Killing. {}".format( attribute_error)) abort(403) except SyntaxError as syntax_error: logger.error( "Syntax Error parsing Robot Items. Killing. {}".format( syntax_error)) abort(500) finally: pass # Default Endorsement g.session_endorsements.append(("username", "{}".format(g.USERNAME))) # DEFAULT Fresh. Use this instead of "Fresh" Values to allow for query caching. NOW = int(time.time()) seconds_after_midnight = NOW % 86400 MIDNIGHT = NOW - seconds_after_midnight oldest = MIDNIGHT - (86400 * 2) g.NOW = NOW g.MIDNIGHT = MIDNIGHT g.twoDayTimestamp = oldest g.cur = g.db.cursor(pymysql.cursors.DictCursor) g.HTTPENDPOINT = config_items["webserver"]["accesslink"] g.config_items = config_items logger.debug("Current Session Endorsements : {}".format( g.session_endorsements))
def grab_single_collated(db_config, result_enum, type_to_grab): logger = logging.getLogger("grab_single_collated") db_conn = manoward.get_conn(db_config, prefix="analyze_", tojq=".database", ac_def=True) cur = db_conn.cursor() # Check Result Enum if result_enum not in ["pass", "fail", "notafflicted"]: raise Exception( "Result Enum not in pass/fail/notafflicted. Instead it's : ", str(results_enum)) if type_to_grab not in ["acoll", "pop", "srvtype"]: raise Exception( "Type to Grab Unknown. Not in acoll, pop, srvtype instead it's : ", str(type_to_grab)) if type_to_grab == "acoll": grouper = "audits.audit_name" table = "audits_by_acoll" elif type_to_grab == "pop": grouper = "hosts.pop" table = "audits_by_pop" elif type_to_grab == "srvtype": grouper = "hosts.srvtype" table = "audits_by_srvtype" grab_single_collated_query_list = list() grab_single_collated_query_list.append( "SELECT " + grouper + ", fk_audits_id, count(DISTINCT fk_host_id) ") grab_single_collated_query_list.append("FROM audits_by_host ") grab_single_collated_query_list.append( "join hosts on fk_host_id = hosts.host_id ") grab_single_collated_query_list.append( "join audits on fk_audits_id = audits.audit_id ") grab_single_collated_query_list.append("WHERE") grab_single_collated_query_list.append("audit_result = %s ") grab_single_collated_query_list.append( "and last_audit >= FROM_UNIXTIME(%s) ") grab_single_collated_query_list.append("group by " + grouper + ", fk_audits_id") grab_scq_args = [result_enum, twodaytimestamp] grab_single_collated_query = " ".join(grab_single_collated_query_list) # print(grab_single_collated_query) try: gsqq_debug = cur.mogrify(grab_single_collated_query, grab_scq_args) logger.debug("GSQQ Debug : {}".format(gsqq_debug)) cur.execute(grab_single_collated_query, grab_scq_args) except Exception as gscq_query_error: logger.error( "Unable to Grab GSCQ Error for Group : {} on type {}".format( grouper, type_to_grab)) raise gscq_query_error else: if cur.rowcount: query_results_list = cur.fetchall() else: # No Results # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! query_results_list = list() finally: cur.close() return query_results_list
def compare_update_insert_table(collected, currently_on_disk, table, db_config): logger = logging.getLogger("compare_update_insert_table") updates = 0 inserts = 0 db_conn = manoward.get_conn(db_config, prefix="analyze_", tojq=".database", ac_def=True) COLLATE_TIME = int(time()) cur = db_conn.cursor() collected_index = list() # Hydrate Collecteded Items for index in collected.keys(): for this_audit_id in collected[index].keys(): collected_index.append((index, this_audit_id)) # Hydrate this if "pass" not in collected[index][this_audit_id].keys(): collected[index][this_audit_id]["pass"] = 0 if "fail" not in collected[index][this_audit_id].keys(): collected[index][this_audit_id]["fail"] = 0 if "notafflicted" not in collected[index][this_audit_id].keys( ): collected[index][this_audit_id]["notafflicted"] = 0 #print("Hydrated Collections", collected) #print("Collected Index", collected_index) # Compare My Stuff (Get a list of update hosts and insert hosts #print("Current Amounts", currently_on_disk) # IDs to Update Used in SQL update_ids = [ item[0] for item in currently_on_disk if item[1] in collected.keys() and item[2] in collected[item[1]]. keys() and collected[item[1]][item[2]]["pass"] == item[3] and collected[item[1]][item[2]]["fail"] == item[4] and collected[item[1]][item[2]]["notafflicted"] == item[5] ] # Items to Insert # Part of What's Needed current_text_location = dict() for i in range(0, len(currently_on_disk)): #print(currently_on_disk[i][1], currently_on_disk[i][2]) if currently_on_disk[i][1] not in current_text_location.keys(): current_text_location[currently_on_disk[i][1]] = dict() if currently_on_disk[i][2] not in current_text_location[ currently_on_disk[i][1]].keys(): current_text_location[currently_on_disk[i][1]][ currently_on_disk[i][2]] = i #print("On Disk Lookups", current_text_location) insert_list = list() insert_list = [ [ item[0], item[1], COLLATE_TIME, COLLATE_TIME, collected[item[0]][item[1]]["pass"], collected[item[0]][item[1]]["fail"], collected[item[0]][item[1]]["notafflicted"] ] for item in collected_index if item[0] not in current_text_location.keys() or item[1] not in current_text_location[item[0]].keys() or (collected[item[0]][item[1]]["pass"], collected[item[0]][item[1]]["fail"], collected[item[0]][item[1]]["notafflicted"]) != (currently_on_disk[current_text_location[item[0]][item[1]]][3], currently_on_disk[current_text_location[item[0]][item[1]]][4], currently_on_disk[current_text_location[item[0]][item[1]]][5]) ] logger.debug("Update IDs : {}".format(update_ids)) logger.debug("Insert List : {}".format(insert_list)) try: if len(update_ids) > 0: #update_ids_string = ",".join(map(str, update_ids)) # Update ID's will now be used as a query paramertization list update_ids_parameters = [" %s " for x in update_ids] update_ids_string = ",".join(map(str, update_ids_parameters)) update_query_parameters = [str(COLLATE_TIME)] update_query_parameters.extend(update_ids) # Query has been parameterized update_query = "UPDATE " update_query = update_query + "audits_by_" + table + " SET " + table +\ "_last_audit = FROM_UNIXTIME( %s ) where " +\ table +\ "_id in ( " + update_ids_string + " ) " try: update_query_debug = cur.mogrify(update_query, update_query_parameters) logger.info( "Running Update Query for table : {}".format(table)) logger.debug("{}".format(table, update_query_debug)) cur.execute(update_query, update_query_parameters) except Exception as update_query_error: logger.error("Error updating hosts for {} : {}".format( table, update_query_error)) else: updates += len(update_ids) except Exception as update_error: logger.error("Error doing Updates. {}".format(update_error)) # Inserts try: if len(insert_list) > 0: # Only do this is there's stuff. insert_query = [] # This query is properly paramaterized and the table value is properly # hardcoded earlier in the program. I'm noseccing it. insert_query.append( "INSERT into audits_by_{0} ( {0}_text, ".format( table)) # nosec insert_query.append( "fk_audits_id, {0}_initial_audit, {0}_last_audit, {0}_passed, {0}_failed, {0}_exempt ) " .format(table)) insert_query.append( "VALUES( %s, %s, FROM_UNIXTIME(%s), FROM_UNIXTIME(%s), %s, %s, %s ) " ) insert_query_string = " ".join(insert_query) try: cur.executemany(insert_query_string, insert_list) except Exception as insert_query_error: logger.error("Error doing Inserts for {} : {}".format( table, insert_error)) else: inserts += len(insert_list) except Exception as insert_error: logger.error("Error doing Inserts : {}".format(insert_error)) return updates, inserts
def storage(config_items, hostdata, sapi=False): ''' Does a Storage of an Object. ''' logger = logging.getLogger("storage.py") STORAGE_TIME = int(time()) storage_stats = dict() MAX = config_items["storage"]["collectionmaxchars"] storage_stats = dict() storage_stats["storage_timestamp"] = STORAGE_TIME db_conn = manoward.get_conn(config_items, prefix="store_", tojq=".database", ac_def=False) try: try: host_id = insert_update_host(hostdata, db_conn) except Exception as insert_update_host_error: logger.error("{}Unable to Update Host with Error : {}{}".format( Fore.RED, insert_update_host_error, Style.RESET_ALL)) raise insert_update_host_error else: logger.info(host_id) # Unique Data FTM hostname = hostdata["collection_hostname"] storage_stats["collection_timestamp"] = hostdata[ "collection_timestamp"] try: storage_stats["inserts"], storage_stats["updates"], storage_stats[ "errors"] = insert_update_collections(db_conn, host_id, hostdata, MAX) except Exception as insert_update_collections_error: logger.error( "{}Unable to Update Collections associated with {}{}".format( Fore.RED, hostname, Style.RESET_ALL)) logger.debug("Error : {}".format(insert_update_collections_error)) raise insert_update_collections_error except Exception as dbconnection_error: logger.error("{}Error Updating Host Collecitons {}{}".format( Fore.RED, dbconnection_error, Style.RESET_ALL)) storage_stats["insert_update"] = 0 storage_stats["errors"] = 1 else: logger.info("{}Updating Collection Success{}{}".format( Fore.GREEN, storage_stats, Style.RESET_ALL)) # Updating Collection has been a success Let's check if this is a sapi host. if sapi is True: # I am so update sapi table storage_stats["sapi_data"] = store_as_SAPI_host(host_id=host_id, db_conn=db_conn, hostname=hostname) do_ipintel = config_items["ip_intel"].get("do_intel", False) logger.debug("Doing IP Intel. ({} Statement).".format(do_ipintel)) if do_ipintel is True and "ip_intel" in hostdata.keys(): # Process the IP Intelligence for this host result = manoward.process_ip_intel( config_dict=config_items, multireport=hostdata["ip_intel"], host_id=host_id) bad_results = [res for res in result if res not in (200, 202)] if len(bad_results) == 0: logger.info("{}IP Intel : {} for host {}{}".format( Fore.GREEN, result, hostname, Style.RESET_ALL)) else: logger.error("{}IP Intel : {} for host {}{}".format( Fore.RED, result, hostname, Style.RESET_ALL)) try: db_conn.commit() db_conn.close() except Exception as e: logger.error("{}Error Closing DB Connection{}".format( Fore.RED, Style.RESET_ALL)) if __name__ == "__main__": print(json.dumps(storage_stats, sort_keys=True, indent=4)) return storage_stats
def analyze(CONFIGDIR, CONFIG, newer=-1): ''' The Main driver for the analyze process. Will load and process analyze items. ''' logger = logging.getLogger("analyze.py:analyze") #ANALYZE_TIME = int(time()) # Parse my General Configuration if isinstance(CONFIG, dict): config_items = CONFIG elif isinstance(CONFIG, str): config_items = manoward.get_manoward(explicit_config=CONFIG) else: raise TypeError("No Configuration Given.") logger.debug("Configuration Items: {}".format(config_items)) if isinstance(CONFIGDIR, dict): logger.debug("CONFIGDIR is given from external process.") audits = CONFIGDIR elif isinstance(CONFIGDIR, list): # Grab all my Audits in CONFIGDIR Stuff auditfiles = audittools.walk_auditd_dir(CONFIGDIR) # Read all my Audits audits = dict() for auditfile in auditfiles: these_audits = audittools.load_auditfile(auditfile) for found_audit_name in these_audits.keys(): if found_audit_name in audits.keys(): logger.warning( "Duplicate definition for {} found. Ignoring definition in file {}" .format(found_audit_name, auditfile)) else: # Add that audit audits[found_audit_name] = these_audits[found_audit_name] def grab_host_list(db_conn, FRESH=172800): # Grab a Host List logger = logging.getLogger("grab_host_list") db_cur = db_conn.cursor(pymysql.cursors.DictCursor) host_query = '''select host_id, pop, srvtype, last_update from hosts where last_update >= now() - INTERVAL %s SECOND''' host_query_args = [FRESH] try: host_list_debug_query = db_cur.mogrify(host_query, host_query_args) logger.debug("hostslist Query : {}".format(host_list_debug_query)) db_cur.execute(host_query, host_query_args) except Exception as hlq_error: logger.error("Unable to Query for Hostslist.") logger.debug("Error : {}".format(hlq_error)) all_hosts = list() amount_of_hosts = 0 host_good = False else: all_hosts = db_cur.fetchall() amount_of_hosts = len(all_hosts) host_good = bool(amount_of_hosts > 0) finally: db_cur.close() return host_good, amount_of_hosts, all_hosts def analyze_one_audit(db_config_items, list_of_hosts, oneAudit, auditName, return_dict, audit_id): # Note that db config items is the same as config_itesm logger = logging.getLogger("analyze_one_audit") logger.debug("Attempting to Analyze Audit {}/{}".format( auditName, audit_id)) try: # I multithread like a boss now. :) JK But I need to give each audit it's own conn to the DB: db_conn = manoward.get_conn(db_config_items, prefix="analyze_", tojq=".database", ac_def=True) # host_buckets = dict() host_comparison = dict() # New Results Variables for Future Reference pop_results = dict() srvtype_results = dict() audit_results = dict() # Add Insert_Update Counters audit_host_inserts = 0 audit_host_updates = 0 # Debug Host No More # Create Bucket Objects from Config for bucket in oneAudit["filters"]: host_buckets[bucket] = [] host_comparison[bucket] = [] # Bucket Hosts Left (All the hosts before I start silly items_left_to_bucket = list_of_hosts logger.debug( "{} Preparing to Analyze {} Hosts with {} Buckets".format( auditName, len(items_left_to_bucket), len(oneAudit["filters"].keys()))) for bucket in oneAudit["filters"]: logger.debug("{} Processing Bucket {}".format( auditName, bucket)) this_mtype = oneAudit["filters"][bucket]["filter-match"] this_ctype = oneAudit["filters"][bucket][ "filter-collection-type"] this_csubtype = oneAudit["filters"][bucket][ "filter-collection-subtype"] this_mvalue = oneAudit["filters"][bucket]["filter-match-value"] #print(this_mtype, this_ctype, this_csubtype, this_mvalue) try: bucket_results = generic_large_compare( db_conn, items_left_to_bucket, this_mtype, this_ctype, this_csubtype, this_mvalue, FRESH, exemptfail=True, audit_name=auditName) except Exception as glc_bucket_results_error: logger.error( "Error on Generic Large Compare on bucket {} : audit {}" .format(bucket, auditName)) logger.warning( "Maybe no Hosts for Bucket {} on audit {}".format( bucket, auditName)) logger.debug("Error : {}".format(glc_bucket_results_error)) else: # Grab just the items that passed if bucket == "bionic-bucket": logger.info( "{} Bionic Bucket Filter Results {}".format( auditName, bucket_results)) for result in bucket_results: if "pfe" in result.keys(): if result["pfe"] == "pass": # Remove pfe & pfevalue from this host so it can be checked again try: del result["pfe"] del result["pfevalue"] # Add my stripped result to the host bucket. # print(result) host_buckets[bucket].append(result) except Exception as e: print("Error adding host to host buckets", e) # Make an index of just the host ids this_bucket_ids = [ gotem["host_id"] for gotem in host_buckets[bucket] ] # Grab just the items that haven't been bucketd yet (so I don't have to compare everything, everytime) items_left_to_bucket = [ host_id for host_id in list_of_hosts if host_id not in this_bucket_ids ] # Host Bucketing for comparison in host_buckets.keys(): # print(comparison) if len(host_buckets[comparison]) > 0: try: this_mtype = oneAudit["comparisons"][comparison][ "comparison-match"] this_ctype = oneAudit["comparisons"][comparison][ "comparison-collection-type"] this_csubtype = oneAudit["comparisons"][comparison][ "comparison-collection-subtype"] this_mvalue = oneAudit["comparisons"][comparison][ "comparison-match-value"] #print(this_mtype, this_ctype, this_csubtype, this_mvalue) except Exception as comparison_error: logger.error( "Error grabbing comparisons for audit {} : {}". format(auditName, comparison_error)) else: # Check What Type logger.debug( "{} Running Comparison on Bucket {}/{}".format( auditName, comparison, len(host_buckets[comparison]))) if this_mtype in [ "subnonhere", "suballhere", "subknowall" ]: # Add Massive Subtype try: comparison_results = subtype_large_compare( db_conn, host_buckets[comparison], this_mtype, this_ctype, this_csubtype, this_mvalue, FRESH) except Exception as subtype_large_compare_error: logger.error( "{} Error on Subtype Large Compare on Comparison for bucket {}" .format(auditName, comparison)) logger.debug("Error : {}".format( subtype_large_compare_error)) else: host_comparison[ comparison] = comparison_results else: # Generic Comparison try: comparison_results = generic_large_compare( db_conn, host_buckets[comparison], this_mtype, this_ctype, this_csubtype, this_mvalue, FRESH) # print(comparison_results) except Exception as generic_large_compare_error: logger.error( "{} Error on Generic Large Compare on Comparison for bucket {}" .format(auditName, comparison)) logger.debug("Error : {}".format( generic_large_compare_error)) else: host_comparison[ comparison] = comparison_results else: # Possible Future Nothing to Compare for {} bucket on audit blah message pass #bucket in host_bucket #print(auditName, " Results : ", host_comparison) logger.debug(host_comparison) logger.info(host_comparison) massinserts = 0 massupdates = 0 massinserts, massupdates = generic_large_analysis_store( db_conn, audit_id, host_comparison, FRESH) # Return Dict is a manager.dict() so the "above" process knows what changes here. return_dict["host_inserts"] = massinserts return_dict["host_updates"] = massupdates except Exception as analyze_error: logger.error("Error doing analyze for {} : {}".format( auditName, analyze_error)) logger.debug(analyze_error) sys.exit(1) else: sys.exit(0) def dequeue_hosts(db_config_items, list_of_hosts): logger = logging.getLogger("analyze:dequeue_hosts") while True: # Pull Stats Stuff # Pull Enqueued Host global audit_host_inserts global audit_host_updates #global srvtype_results #global pop_results #global audit_results # print(audit_results) try: oneAudit, auditName = audit_queue.get() except Exception as audit_get_error: logger.error("Failure to Pull Items off of Queue.") logger.debug("Error : {}".format(audit_get_error)) audit_queue.task_done() # Abnormal Return return try: manager = multiprocessing.Manager() except Exception as multiprocess_error: logger.error( "Failure to Create Manager for audit {} with error {}". format(auditName, multiprocess_error)) audit_queue.task_done() # Abnormal Return return else: return_dict = manager.dict() # Insert Update the Audit in the Database try: audit_id = insert_update_audit(db_config_items, oneAudit) except Exception as update_audit_db_error: logger.error( "Failure to Create Audit {} in DB with error {}".format( auditName, update_audit_db_error)) audit_queue.task_done() return else: oneAudit["audit_id"] = audit_id logger.debug( "Stored a Record about audit {}/{} in the database.". format(auditName, audit_id)) #print("Pulled Host ", this_one_host_array) # Process One Host Pass it the host_array and the config_array try: #analyze_one_audit(db_config_items, list_of_hosts, oneAudit, auditName, return_dict, audit_id) # analyze_audit_process is a new instance for every new thread we make. try: analyze_audit_process = multiprocessing.Process( target=analyze_one_audit, args=(db_config_items, list_of_hosts, oneAudit, auditName, return_dict, audit_id)) analyze_audit_process.name = auditName analyze_audit_process.daemon = True analyze_audit_process.start() except Exception as analyze_pid_error: logger.error("Error with Analyze Audit {} : {}".format( auditName, analyze_pid_error)) analyze_audit_process.terminate() else: while multiprocessing.Process.is_alive( analyze_audit_process) == True: logger.debug("Waiting for: {} {}".format( auditName, multiprocessing.Process.is_alive( analyze_audit_process))) # Waith 45 Seconds before Asking again sleep(45) analyze_audit_process.join() except Exception as audit_analyisis_error: logger.error("Failure to Analyze Audit {} : {}".format( auditName, audit_analyisis_error)) # I/U Stats only Thing Left logger.debug(return_dict) try: with audit_host_counts_lock: logger.info("{} I:{} U:{}".format( auditName, return_dict["host_inserts"], return_dict["host_updates"])) # This is a Global audit_host_inserts += return_dict["host_inserts"] audit_host_updates += return_dict["host_updates"] except Exception as metrics_error: # print(return_dict) logger.error( "Failure on Audit when Recording Metrics {} : {}".format( auditName, metrics_error)) audit_queue.task_done() return def analyze_all_audits(db_config_items, list_of_hosts, FRESH, MAXTHREADS): # Audits are a global variable logger = logging.getLogger("analyze_all_audits") logger.debug("Apparantly I'm not using fresh {}".format(FRESH)) # Copy Time results_host = deepcopy(list_of_hosts) # Create My ThreadPool for x in range(MAXTHREADS): # This is the magic. It calls dequeu hostsk t = threading.Thread(target=dequeue_hosts, args=(db_config_items, list_of_hosts)) # Make Threads Die if Parent is Killed t.daemon = True # Start my Threads t.start() # Counter for Time Spent start = time() for audit in audits: # Populate Audit Queue logger.info("About to Queue audit {}".format(audit)) # try: this_queue_item = [audits[audit], audit] audit_queue.put(this_queue_item) # Sleep to allow for better placement sleep(1) # If your running verbosely Print out this stuff Else not while audit_queue.unfinished_tasks > 0: nowtime = time() - start logger.debug("---------------------------------------") logger.debug("AuditsLeft : {}".format( audit_queue.unfinished_tasks)) logger.debug("QSize : {}".format(audit_queue.qsize())) logger.debug("Thread : {}".format(threading.active_count())) logger.debug("QStuff : {}".format(audit_queue.empty())) logger.debug("Time : {}".format(nowtime)) logger.debug("---------------------------------------") # Give me an Update every 30 seconds sleep(15) # When I'm Not Verbose Just wait and don't say shit. # Otherwise when I see a small number of unfinished tasks Let's move back an djoin. audit_queue.join() jobtime = time() - start return audit_host_inserts, audit_host_updates, jobtime def insert_update_audit(db_config_items, audit): #logger = logging.getLogger("analyze:insert_update_audit") # Literals this_audit_name = audit["vuln-name"] this_audit_short = audit["vuln-short-description"] this_audit_long_description = re.sub( r'[\'|\;]', r'', audit["vuln-long-description"][:511]) this_audit_primary_link = audit["vuln-primary-link"] this_audit_secondary_links = audit["vuln-additional-links"] this_audit_filters = audit["filters"] this_audit_comparison = audit["comparisons"] this_audit_filename = audit["filename"] this_audit_priority = audit.get("vuln-priority", 5) db_conn = manoward.get_conn(db_config_items, prefix="analyze_", tojq=".database", ac_def=True) cur = db_conn.cursor() # Always Match by audit_name grab_audit = "SELECT audit_id from audits where audit_name = %s ; " grab_audit_paramaters = [str(this_audit_name)] cur.execute(grab_audit, grab_audit_paramaters) if cur.rowcount: # There's Data this_audit_id = cur.fetchone()[0] have_audit_id = True else: have_audit_id = False replace_query_args = list() # Always Specified Columns columns = """audit_name, audit_short_description, audit_long_description, audit_primary_link, audit_secondary_links, audit_filters, audit_comparison, audit_priority, filename """ dynamic_column_items = list() dynamic_column_args = list() # print(this_audit_secondary_links) for secondary_link in this_audit_secondary_links: # print(secondary_link) dynamic_column_items.append(" %s , %s ") # Add to Args List dynamic_column_args.append(str(secondary_link)) dynamic_column_args.append( str(this_audit_secondary_links[secondary_link])) # Make this a list of double %s 's # Put it in the thing dynamic_column = " , ".join(dynamic_column_items) dynamic_column = "COLUMN_CREATE(" + dynamic_column + ")" # Values String this_audit_values = " %s , %s , %s , %s , " + dynamic_column + ", %s " + \ " , %s , %s , %s " this_audit_value_paramaters = [ str(this_audit_name), str(this_audit_short)[:63], str(this_audit_long_description), str(this_audit_primary_link) ] this_audit_value_paramaters.extend(dynamic_column_args) temp_list = [ str(this_audit_filters).replace('\'', '"')[:511], str(this_audit_comparison).replace('\'', '"')[:511], str(this_audit_priority), str(this_audit_filename) ] this_audit_value_paramaters.extend(temp_list) query_head = "REPLACE into audits ( " query_mid = " ) VALUES ( " query_tail = " ) " if have_audit_id: # Have Audit ID so add it, otherwise these won't be added columns = columns + " , audit_id " this_audit_values = this_audit_values + ", %s " this_audit_value_paramaters.append(str(this_audit_id)) query_string = query_head + columns + query_mid + this_audit_values + query_tail # This is a replace and will update the audit no matter. debug_sql = cur.mogrify(query_string, this_audit_value_paramaters) cur.execute(query_string, this_audit_value_paramaters) this_row = cur.lastrowid cur.close() return this_row # Globals global pop_results global srvtype_results global audit_results # Inserts Updates global audit_host_inserts global audit_host_updates # Results dictionaries pop_results = dict() #pop_results_lock = threading.Lock() srvtype_results = dict() #srvtype_results_lock = threading.Lock() audit_results = dict() #audit_results_lock = threading.Lock() audit_host_inserts = 0 audit_host_updates = 0 audit_host_counts_lock = threading.Lock() # COnfig ITems #MAX = config_items["storage"]["collectionmaxchars"] FRESH = config_items["analyze"]["freshseconds"] MAXTHREADS = int(config_items["analyze"]["maxthreads"]) # Create A Queue audit_queue = Queue() # try: try: db_conn = manoward.get_conn(config_items, prefix="analyze_", tojq=".database", ac_def=True) dbmessage = "Connected" except Exception as db_conn_error: dbmessage = "Unable to Connect" logger.debug("DB Connection Error : {}".format(db_conn_error)) finally: # Start my analyze_stats with data analyze_stats = {"db-status": dbmessage} # Grab Hosts List (Still Single Threaded) host_good, analyze_stats["FreshHosts"], host_list = grab_host_list( db_conn, FRESH) if host_good: logger.info("Successfully Collected {} Hosts as 'Live'".format( len(host_list))) analyze_stats["HostCollectionStatus"] = "Success" analyze_stats["audit_inserts"], analyze_stats[ "audit_updates"], analyze_stats["jobtime"] = analyze_all_audits( config_items, host_list, FRESH, MAXTHREADS) analyze_stats["threads"] = str(MAXTHREADS) analyze_stats["totalaudits"] = len(audits) else: analyze_stats["HostCollectionStatus"] = "Failed" return analyze_stats
def insert_update_audit(db_config_items, audit): #logger = logging.getLogger("analyze:insert_update_audit") # Literals this_audit_name = audit["vuln-name"] this_audit_short = audit["vuln-short-description"] this_audit_long_description = re.sub( r'[\'|\;]', r'', audit["vuln-long-description"][:511]) this_audit_primary_link = audit["vuln-primary-link"] this_audit_secondary_links = audit["vuln-additional-links"] this_audit_filters = audit["filters"] this_audit_comparison = audit["comparisons"] this_audit_filename = audit["filename"] this_audit_priority = audit.get("vuln-priority", 5) db_conn = manoward.get_conn(db_config_items, prefix="analyze_", tojq=".database", ac_def=True) cur = db_conn.cursor() # Always Match by audit_name grab_audit = "SELECT audit_id from audits where audit_name = %s ; " grab_audit_paramaters = [str(this_audit_name)] cur.execute(grab_audit, grab_audit_paramaters) if cur.rowcount: # There's Data this_audit_id = cur.fetchone()[0] have_audit_id = True else: have_audit_id = False replace_query_args = list() # Always Specified Columns columns = """audit_name, audit_short_description, audit_long_description, audit_primary_link, audit_secondary_links, audit_filters, audit_comparison, audit_priority, filename """ dynamic_column_items = list() dynamic_column_args = list() # print(this_audit_secondary_links) for secondary_link in this_audit_secondary_links: # print(secondary_link) dynamic_column_items.append(" %s , %s ") # Add to Args List dynamic_column_args.append(str(secondary_link)) dynamic_column_args.append( str(this_audit_secondary_links[secondary_link])) # Make this a list of double %s 's # Put it in the thing dynamic_column = " , ".join(dynamic_column_items) dynamic_column = "COLUMN_CREATE(" + dynamic_column + ")" # Values String this_audit_values = " %s , %s , %s , %s , " + dynamic_column + ", %s " + \ " , %s , %s , %s " this_audit_value_paramaters = [ str(this_audit_name), str(this_audit_short)[:63], str(this_audit_long_description), str(this_audit_primary_link) ] this_audit_value_paramaters.extend(dynamic_column_args) temp_list = [ str(this_audit_filters).replace('\'', '"')[:511], str(this_audit_comparison).replace('\'', '"')[:511], str(this_audit_priority), str(this_audit_filename) ] this_audit_value_paramaters.extend(temp_list) query_head = "REPLACE into audits ( " query_mid = " ) VALUES ( " query_tail = " ) " if have_audit_id: # Have Audit ID so add it, otherwise these won't be added columns = columns + " , audit_id " this_audit_values = this_audit_values + ", %s " this_audit_value_paramaters.append(str(this_audit_id)) query_string = query_head + columns + query_mid + this_audit_values + query_tail # This is a replace and will update the audit no matter. debug_sql = cur.mogrify(query_string, this_audit_value_paramaters) cur.execute(query_string, this_audit_value_paramaters) this_row = cur.lastrowid cur.close() return this_row
def analyze_one_audit(db_config_items, list_of_hosts, oneAudit, auditName, return_dict, audit_id): # Note that db config items is the same as config_itesm logger = logging.getLogger("analyze_one_audit") logger.debug("Attempting to Analyze Audit {}/{}".format( auditName, audit_id)) try: # I multithread like a boss now. :) JK But I need to give each audit it's own conn to the DB: db_conn = manoward.get_conn(db_config_items, prefix="analyze_", tojq=".database", ac_def=True) # host_buckets = dict() host_comparison = dict() # New Results Variables for Future Reference pop_results = dict() srvtype_results = dict() audit_results = dict() # Add Insert_Update Counters audit_host_inserts = 0 audit_host_updates = 0 # Debug Host No More # Create Bucket Objects from Config for bucket in oneAudit["filters"]: host_buckets[bucket] = [] host_comparison[bucket] = [] # Bucket Hosts Left (All the hosts before I start silly items_left_to_bucket = list_of_hosts logger.debug( "{} Preparing to Analyze {} Hosts with {} Buckets".format( auditName, len(items_left_to_bucket), len(oneAudit["filters"].keys()))) for bucket in oneAudit["filters"]: logger.debug("{} Processing Bucket {}".format( auditName, bucket)) this_mtype = oneAudit["filters"][bucket]["filter-match"] this_ctype = oneAudit["filters"][bucket][ "filter-collection-type"] this_csubtype = oneAudit["filters"][bucket][ "filter-collection-subtype"] this_mvalue = oneAudit["filters"][bucket]["filter-match-value"] #print(this_mtype, this_ctype, this_csubtype, this_mvalue) try: bucket_results = generic_large_compare( db_conn, items_left_to_bucket, this_mtype, this_ctype, this_csubtype, this_mvalue, FRESH, exemptfail=True, audit_name=auditName) except Exception as glc_bucket_results_error: logger.error( "Error on Generic Large Compare on bucket {} : audit {}" .format(bucket, auditName)) logger.warning( "Maybe no Hosts for Bucket {} on audit {}".format( bucket, auditName)) logger.debug("Error : {}".format(glc_bucket_results_error)) else: # Grab just the items that passed if bucket == "bionic-bucket": logger.info( "{} Bionic Bucket Filter Results {}".format( auditName, bucket_results)) for result in bucket_results: if "pfe" in result.keys(): if result["pfe"] == "pass": # Remove pfe & pfevalue from this host so it can be checked again try: del result["pfe"] del result["pfevalue"] # Add my stripped result to the host bucket. # print(result) host_buckets[bucket].append(result) except Exception as e: print("Error adding host to host buckets", e) # Make an index of just the host ids this_bucket_ids = [ gotem["host_id"] for gotem in host_buckets[bucket] ] # Grab just the items that haven't been bucketd yet (so I don't have to compare everything, everytime) items_left_to_bucket = [ host_id for host_id in list_of_hosts if host_id not in this_bucket_ids ] # Host Bucketing for comparison in host_buckets.keys(): # print(comparison) if len(host_buckets[comparison]) > 0: try: this_mtype = oneAudit["comparisons"][comparison][ "comparison-match"] this_ctype = oneAudit["comparisons"][comparison][ "comparison-collection-type"] this_csubtype = oneAudit["comparisons"][comparison][ "comparison-collection-subtype"] this_mvalue = oneAudit["comparisons"][comparison][ "comparison-match-value"] #print(this_mtype, this_ctype, this_csubtype, this_mvalue) except Exception as comparison_error: logger.error( "Error grabbing comparisons for audit {} : {}". format(auditName, comparison_error)) else: # Check What Type logger.debug( "{} Running Comparison on Bucket {}/{}".format( auditName, comparison, len(host_buckets[comparison]))) if this_mtype in [ "subnonhere", "suballhere", "subknowall" ]: # Add Massive Subtype try: comparison_results = subtype_large_compare( db_conn, host_buckets[comparison], this_mtype, this_ctype, this_csubtype, this_mvalue, FRESH) except Exception as subtype_large_compare_error: logger.error( "{} Error on Subtype Large Compare on Comparison for bucket {}" .format(auditName, comparison)) logger.debug("Error : {}".format( subtype_large_compare_error)) else: host_comparison[ comparison] = comparison_results else: # Generic Comparison try: comparison_results = generic_large_compare( db_conn, host_buckets[comparison], this_mtype, this_ctype, this_csubtype, this_mvalue, FRESH) # print(comparison_results) except Exception as generic_large_compare_error: logger.error( "{} Error on Generic Large Compare on Comparison for bucket {}" .format(auditName, comparison)) logger.debug("Error : {}".format( generic_large_compare_error)) else: host_comparison[ comparison] = comparison_results else: # Possible Future Nothing to Compare for {} bucket on audit blah message pass #bucket in host_bucket #print(auditName, " Results : ", host_comparison) logger.debug(host_comparison) logger.info(host_comparison) massinserts = 0 massupdates = 0 massinserts, massupdates = generic_large_analysis_store( db_conn, audit_id, host_comparison, FRESH) # Return Dict is a manager.dict() so the "above" process knows what changes here. return_dict["host_inserts"] = massinserts return_dict["host_updates"] = massupdates except Exception as analyze_error: logger.error("Error doing analyze for {} : {}".format( auditName, analyze_error)) logger.debug(analyze_error) sys.exit(1) else: sys.exit(0)