def main(): err = 0 #Logger settings logging.basicConfig(filename=header.MFC_LOG_FILE, format='%(asctime)s %(message)s',level=logging.DEBUG) mdreq = MdReq() #Get the device map name device_map_name = str(sys.argv[1]) NODE_DEVICE_MAP_FQDN = '/nkn/device_map/config/' + device_map_name + '/device_info/fqdn' NODE_DEVICE_MAP_USERNAME = '******' + device_map_name + '/device_info/username' NODE_DEVICE_MAP_PASSWORD = '******' + device_map_name + '/device_info/password' #Get the fqdn, username and password fqdn = mdreq.query(NODE_DEVICE_MAP_FQDN) username = mdreq.query(NODE_DEVICE_MAP_USERNAME) password = mdreq.queryCleartext(NODE_DEVICE_MAP_PASSWORD) #First connect to the MX router err = gen_utils.Connect_Router(fqdn,username, password, logging); #Check if the connection to the router succeeded if err == 1: logging.error("The command 'clear log-analyzer filter-rules %s' failed. Cannot connect to the device %s", device_map_name,fqdn); else: #Construct the command to delete the pbr from the router set_commands = """ delete policy-options prefix-list redirect-to-proxy set policy-options prefix-list redirect-to-proxy """ #Fire the command to the router err = gen_utils.sendCmd(set_commands, "yes", "","", logging) #Check if the commit succeeded if err == 0: #Create a empty file so that the log-analyzer tool knows the 'clear' command has been fired open(header.CLEAR_FILTER_RULES_FILE, 'w').close() #If the commit is successful remove the persisted file rules file as well pickle_file = '/var/log/'+header.DEST_IP_PICKLE tmp_pickle_file = '/var/log/'+header.TMP_DEST_IP_PICKLE if os.path.isfile(pickle_file): shutil.copyfile(pickle_file, tmp_pickle_file) os.remove(pickle_file) logging.error("The command 'clear log-analyzer filter-rules %s' succeeded", device_map_name); else: logging.error("The command 'clear log-analyzer filter-rules %s' failed. Commit failed in the device %s", device_map_name, fqdn); return err
""" Need to check if there is another instance of this script running. This can be done with either a flock or adding a sentinel at this place... not before and not after. """ pid_file = 'logCleanup.lock' fp = open(pid_file, 'w') try: fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError: # kill if another instance is running logger.warn('attempt to re-enter when already running' \ 'an instance') sys.exit(0) # Read in configured parameters mdreq = MdReq() try: sizeThresh = int(mdreq.query(NODE_LOG_EXPORT_SIZE_THRESH)) #convert the interval from hours to seconds interThresh = int(mdreq.query(NODE_LOG_EXPORT_INTER_THRESH))*60*60 except Exception, e: logger.error('Bad karma!!! - fix my exception handling') onSizeThresh(sizeThresh) onIntervalThresh(interThresh) sys.exit(0)
def main(): global dpiloganalyzer_log global dest_ips global frequency global fire_to_router domain_patterns = "" device_map_name = "" # Basic common initialization init() # Parse the command-line arguments # If the number of arguments is greater than 1 make sure the arguments values are correct if len(sys.argv) > 1: if len(sys.argv) == 3 and ( (str(sys.argv[1]) == "--mode" or str(sys.argv[1]) == "-m") and str(sys.argv[2]) != "mfc" ): print_usage() elif ( len(sys.argv) == 3 and str(sys.argv[1]) != "--config-path" and str(sys.argv[1]) != "-C" and str(sys.argv[1]) != "--mode" and str(sys.argv[1]) != "-m" ): print_usage() elif len(sys.argv) == 5 and ( (str(sys.argv[1]) != "--mode" and str(sys.argv[1]) != "-m") or str(sys.argv[2]) != "mfc" or (str(sys.argv[3]) != "--config-path" and str(sys.argv[3]) != "-C") ): print_usage() elif len(sys.argv) == 2 or len(sys.argv) == 4: print_usage() # If the script is launched in mfc mode then read the MFC accesslog files if len(sys.argv) > 1 and ((str(sys.argv[1]) == "--mode" or str(sys.argv[1]) == "-m") and str(sys.argv[2]) == "mfc"): # Parse the analyzer.conf file and populate the values in a dictionary analyzer_conf_dict = parse_analyzer_conf() mfc_accesslog_analyzer.read_mfc_accesslog(analyzer_conf_dict) else: # Logger settings logformatter = logging.Formatter("%(asctime)s;%(message)s") # Use the custom log rotation handler which gzip's the old rotated log file handler = gen_utils.NewRotatingFileHandler( header.LOG_FILE, maxBytes=header.MAXLOG_SIZE, backupCount=header.LOGBACKP_COUNT ) handler.setFormatter(logformatter) dpiloganalyzer_log = logging.getLogger("__name__") dpiloganalyzer_log.setLevel(logging.DEBUG) # Point to the custom log handler dpiloganalyzer_log.addHandler(handler) # Get the handle to the instance of the class that enables us to query the mgmtd database mdreq = MdReq() # Get the device-map name bound to the log-analyzer dmap = mdreq.queryIterate("/nkn/log_analyzer/config/device_map") for key, val in dmap.iteritems(): device_map_name = val["value"] if device_map_name != "": # Get the mode in which the log-analyzer tool has to be run CMDMODE_NODE = "/nkn/log_analyzer/config/device_map/" + device_map_name + "/command_mode/inline" mode = mdreq.query(CMDMODE_NODE) if mode.lower() == "true": fire_to_router = "yes" else: fire_to_router = "no" FREQUENCY_NODE = header.DEVICE_MAP_COMMON + device_map_name + "/filter_config/frequency" frequency = mdreq.query(FREQUENCY_NODE) # Convert the frequency in seconds as the SIGALRM requires it to be in seconds frequency = int(frequency) * 60 else: dpiloganalyzer_log.error( "The log-analyzer is not associated to a device. The commands will be pushed to a file" ) fire_to_router = "no" # Default to 2hrs frequency to dump into the file frequency = 7200 # Parse the analyzer.conf file and populate the values in a dictionary analyzer_conf_dict = parse_analyzer_conf() signal.alarm(frequency) # Parse Override_rules.xml only if you want to determine the cacheability of url try: if analyzer_conf_dict["fire_pbr_for_cacheable_url"].strip().lower() == "yes": domain_patterns = parse_override_rules_xml() except KeyError as e: dpiloganalyzer_log.info( "The analyzer.conf file is missing the entry %s. Defaulting to 'no' and cacheability check will not be performed", e, ) # Parse the golden-config.txt file and get the list of domain-regex patterns golden_config_dict = parse_golden_config(mdreq, device_map_name) # Check if the user wants the commands to be fired to the router if fire_to_router == "yes": # Get the device details FQDN_NODE = header.DEVICE_MAP_COMMON + device_map_name + "/device_info/fqdn" USERNAME_NODE = header.DEVICE_MAP_COMMON + device_map_name + "/device_info/username" PASSWD_NODE = header.DEVICE_MAP_COMMON + device_map_name + "/device_info/password" device_fqdn = mdreq.query(FQDN_NODE) device_username = mdreq.query(USERNAME_NODE) device_passwd = mdreq.queryCleartext(PASSWD_NODE) # First connect to the MX router err = gen_utils.Connect_Router(device_fqdn, device_username, device_passwd, dpiloganalyzer_log) if err == 1: dpiloganalyzer_log.error( "Unable to connect to the router %s. The filter-rules will be written to a file.", device_fqdn ) fire_to_router = "no" else: if analyzer_conf_dict["fire_one_time_config"].lower() == "yes": # Fire the one time commands into the router fireOneTimeCommands(analyzer_conf_dict) # Fire the mirror configuration commands into the router fireMirrorConfiguration(analyzer_conf_dict) # Load the dest-ip pickle into the dictionary os.chdir("/var/log") if os.path.isfile(header.DEST_IP_PICKLE) and os.stat(header.DEST_IP_PICKLE).st_size > 0: with open(header.DEST_IP_PICKLE, "rb") as f: dest_ips = pickle.load(f) # Read the dpilogs genereated by the dpi-tool while True: read_dpi_logs(domain_patterns, golden_config_dict, analyzer_conf_dict, mdreq, device_map_name) time.sleep(2)
def main(): global dpiloganalyzer_log global dest_ips global frequency global fire_to_router domain_patterns = "" device_map_name = "" #Basic common initialization init() #Parse the command-line arguments #If the number of arguments is greater than 1 make sure the arguments values are correct if len(sys.argv) > 1: if len(sys.argv) == 3 and ( (str(sys.argv[1]) == '--mode' or str(sys.argv[1]) == '-m') and str(sys.argv[2]) != 'mfc'): print_usage() elif len( sys.argv) == 3 and str(sys.argv[1]) != '--config-path' and str( sys.argv[1]) != '-C' and str( sys.argv[1]) != '--mode' and str(sys.argv[1]) != '-m': print_usage() elif len(sys.argv) == 5 and ( (str(sys.argv[1]) != '--mode' and str(sys.argv[1]) != '-m') or str(sys.argv[2]) != 'mfc' or (str(sys.argv[3]) != '--config-path' and str(sys.argv[3]) != '-C')): print_usage() elif len(sys.argv) == 2 or len(sys.argv) == 4: print_usage() #If the script is launched in mfc mode then read the MFC accesslog files if len(sys.argv) > 1 and ( (str(sys.argv[1]) == '--mode' or str(sys.argv[1]) == '-m') and str(sys.argv[2]) == 'mfc'): #Parse the analyzer.conf file and populate the values in a dictionary analyzer_conf_dict = parse_analyzer_conf() mfc_accesslog_analyzer.read_mfc_accesslog(analyzer_conf_dict) else: #Logger settings logformatter = logging.Formatter('%(asctime)s;%(message)s') #Use the custom log rotation handler which gzip's the old rotated log file handler = gen_utils.NewRotatingFileHandler( header.LOG_FILE, maxBytes=header.MAXLOG_SIZE, backupCount=header.LOGBACKP_COUNT) handler.setFormatter(logformatter) dpiloganalyzer_log = logging.getLogger('__name__') dpiloganalyzer_log.setLevel(logging.DEBUG) #Point to the custom log handler dpiloganalyzer_log.addHandler(handler) #Get the handle to the instance of the class that enables us to query the mgmtd database mdreq = MdReq() #Get the device-map name bound to the log-analyzer dmap = mdreq.queryIterate("/nkn/log_analyzer/config/device_map") for key, val in dmap.iteritems(): device_map_name = val['value'] if device_map_name != "": #Get the mode in which the log-analyzer tool has to be run CMDMODE_NODE = '/nkn/log_analyzer/config/device_map/' + device_map_name + '/command_mode/inline' mode = mdreq.query(CMDMODE_NODE) if mode.lower() == 'true': fire_to_router = 'yes' else: fire_to_router = 'no' FREQUENCY_NODE = header.DEVICE_MAP_COMMON + device_map_name + '/filter_config/frequency' frequency = mdreq.query(FREQUENCY_NODE) #Convert the frequency in seconds as the SIGALRM requires it to be in seconds frequency = int(frequency) * 60 else: dpiloganalyzer_log.error( "The log-analyzer is not associated to a device. The commands will be pushed to a file" ) fire_to_router = "no" #Default to 2hrs frequency to dump into the file frequency = 7200 #Parse the analyzer.conf file and populate the values in a dictionary analyzer_conf_dict = parse_analyzer_conf() signal.alarm(frequency) #Parse Override_rules.xml only if you want to determine the cacheability of url try: if analyzer_conf_dict['fire_pbr_for_cacheable_url'].strip().lower( ) == 'yes': domain_patterns = parse_override_rules_xml() except KeyError as e: dpiloganalyzer_log.info( "The analyzer.conf file is missing the entry %s. Defaulting to 'no' and cacheability check will not be performed", e) #Parse the golden-config.txt file and get the list of domain-regex patterns golden_config_dict = parse_golden_config(mdreq, device_map_name) #Check if the user wants the commands to be fired to the router if fire_to_router == 'yes': #Get the device details FQDN_NODE = header.DEVICE_MAP_COMMON + device_map_name + '/device_info/fqdn' USERNAME_NODE = header.DEVICE_MAP_COMMON + device_map_name + '/device_info/username' PASSWD_NODE = header.DEVICE_MAP_COMMON + device_map_name + '/device_info/password' device_fqdn = mdreq.query(FQDN_NODE) device_username = mdreq.query(USERNAME_NODE) device_passwd = mdreq.queryCleartext(PASSWD_NODE) #First connect to the MX router err = gen_utils.Connect_Router(device_fqdn, device_username, device_passwd, dpiloganalyzer_log) if err == 1: dpiloganalyzer_log.error( "Unable to connect to the router %s. The filter-rules will be written to a file.", device_fqdn) fire_to_router = "no" else: if analyzer_conf_dict['fire_one_time_config'].lower() == 'yes': #Fire the one time commands into the router fireOneTimeCommands(analyzer_conf_dict) #Fire the mirror configuration commands into the router fireMirrorConfiguration(analyzer_conf_dict) #Load the dest-ip pickle into the dictionary os.chdir("/var/log") if os.path.isfile(header.DEST_IP_PICKLE) and os.stat( header.DEST_IP_PICKLE).st_size > 0: with open(header.DEST_IP_PICKLE, 'rb') as f: dest_ips = pickle.load(f) #Read the dpilogs genereated by the dpi-tool while True: read_dpi_logs(domain_patterns, golden_config_dict, analyzer_conf_dict, mdreq, device_map_name) time.sleep(2)