def main():
err = 0
#Logger settings
logging.basicConfig(filename=header.MFC_LOG_FILE, format='%(asctime)s %(message)s',level=logging.DEBUG)
mdreq = MdReq()
#Get the device map name
device_map_name = str(sys.argv[1])
NODE_DEVICE_MAP_FQDN = '/nkn/device_map/config/' + device_map_name + '/device_info/fqdn'
NODE_DEVICE_MAP_USERNAME = '******' + device_map_name + '/device_info/username'
NODE_DEVICE_MAP_PASSWORD = '******' + device_map_name + '/device_info/password'
#Get the fqdn, username and password
fqdn = mdreq.query(NODE_DEVICE_MAP_FQDN)
username = mdreq.query(NODE_DEVICE_MAP_USERNAME)
password = mdreq.queryCleartext(NODE_DEVICE_MAP_PASSWORD)
#First connect to the MX router
err = gen_utils.Connect_Router(fqdn,username, password, logging);
#Check if the connection to the router succeeded
if err == 1:
logging.error("The command 'clear log-analyzer filter-rules %s' failed. Cannot connect to the device %s", device_map_name,fqdn);
else:
#Construct the command to delete the pbr from the router
set_commands = """
delete policy-options prefix-list redirect-to-proxy
set policy-options prefix-list redirect-to-proxy
"""
#Fire the command to the router
err = gen_utils.sendCmd(set_commands, "yes", "","", logging)
#Check if the commit succeeded
if err == 0:
#Create a empty file so that the log-analyzer tool knows the 'clear' command has been fired
open(header.CLEAR_FILTER_RULES_FILE, 'w').close()
#If the commit is successful remove the persisted file rules file as well
pickle_file = '/var/log/'+header.DEST_IP_PICKLE
tmp_pickle_file = '/var/log/'+header.TMP_DEST_IP_PICKLE
if os.path.isfile(pickle_file):
shutil.copyfile(pickle_file, tmp_pickle_file)
os.remove(pickle_file)
logging.error("The command 'clear log-analyzer filter-rules %s' succeeded", device_map_name);
else:
logging.error("The command 'clear log-analyzer filter-rules %s' failed. Commit failed in the device %s", device_map_name, fqdn);
return err
""" Need to check if there is another instance of this script
running. This can be done with either a flock or adding a
sentinel at this place... not before and not after.
"""
pid_file = 'logCleanup.lock'
fp = open(pid_file, 'w')
try:
fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB)
except IOError:
# kill if another instance is running
logger.warn('attempt to re-enter when already running' \
'an instance')
sys.exit(0)
# Read in configured parameters
mdreq = MdReq()
try:
sizeThresh = int(mdreq.query(NODE_LOG_EXPORT_SIZE_THRESH))
#convert the interval from hours to seconds
interThresh = int(mdreq.query(NODE_LOG_EXPORT_INTER_THRESH))*60*60
except Exception, e:
logger.error('Bad karma!!! - fix my exception handling')
onSizeThresh(sizeThresh)
onIntervalThresh(interThresh)
sys.exit(0)
def main():
global dpiloganalyzer_log
global dest_ips
global frequency
global fire_to_router
domain_patterns = ""
device_map_name = ""
# Basic common initialization
init()
# Parse the command-line arguments
# If the number of arguments is greater than 1 make sure the arguments values are correct
if len(sys.argv) > 1:
if len(sys.argv) == 3 and (
(str(sys.argv[1]) == "--mode" or str(sys.argv[1]) == "-m") and str(sys.argv[2]) != "mfc"
):
print_usage()
elif (
len(sys.argv) == 3
and str(sys.argv[1]) != "--config-path"
and str(sys.argv[1]) != "-C"
and str(sys.argv[1]) != "--mode"
and str(sys.argv[1]) != "-m"
):
print_usage()
elif len(sys.argv) == 5 and (
(str(sys.argv[1]) != "--mode" and str(sys.argv[1]) != "-m")
or str(sys.argv[2]) != "mfc"
or (str(sys.argv[3]) != "--config-path" and str(sys.argv[3]) != "-C")
):
print_usage()
elif len(sys.argv) == 2 or len(sys.argv) == 4:
print_usage()
# If the script is launched in mfc mode then read the MFC accesslog files
if len(sys.argv) > 1 and ((str(sys.argv[1]) == "--mode" or str(sys.argv[1]) == "-m") and str(sys.argv[2]) == "mfc"):
# Parse the analyzer.conf file and populate the values in a dictionary
analyzer_conf_dict = parse_analyzer_conf()
mfc_accesslog_analyzer.read_mfc_accesslog(analyzer_conf_dict)
else:
# Logger settings
logformatter = logging.Formatter("%(asctime)s;%(message)s")
# Use the custom log rotation handler which gzip's the old rotated log file
handler = gen_utils.NewRotatingFileHandler(
header.LOG_FILE, maxBytes=header.MAXLOG_SIZE, backupCount=header.LOGBACKP_COUNT
)
handler.setFormatter(logformatter)
dpiloganalyzer_log = logging.getLogger("__name__")
dpiloganalyzer_log.setLevel(logging.DEBUG)
# Point to the custom log handler
dpiloganalyzer_log.addHandler(handler)
# Get the handle to the instance of the class that enables us to query the mgmtd database
mdreq = MdReq()
# Get the device-map name bound to the log-analyzer
dmap = mdreq.queryIterate("/nkn/log_analyzer/config/device_map")
for key, val in dmap.iteritems():
device_map_name = val["value"]
if device_map_name != "":
# Get the mode in which the log-analyzer tool has to be run
CMDMODE_NODE = "/nkn/log_analyzer/config/device_map/" + device_map_name + "/command_mode/inline"
mode = mdreq.query(CMDMODE_NODE)
if mode.lower() == "true":
fire_to_router = "yes"
else:
fire_to_router = "no"
FREQUENCY_NODE = header.DEVICE_MAP_COMMON + device_map_name + "/filter_config/frequency"
frequency = mdreq.query(FREQUENCY_NODE)
# Convert the frequency in seconds as the SIGALRM requires it to be in seconds
frequency = int(frequency) * 60
else:
dpiloganalyzer_log.error(
"The log-analyzer is not associated to a device. The commands will be pushed to a file"
)
fire_to_router = "no"
# Default to 2hrs frequency to dump into the file
frequency = 7200
# Parse the analyzer.conf file and populate the values in a dictionary
analyzer_conf_dict = parse_analyzer_conf()
signal.alarm(frequency)
# Parse Override_rules.xml only if you want to determine the cacheability of url
try:
if analyzer_conf_dict["fire_pbr_for_cacheable_url"].strip().lower() == "yes":
domain_patterns = parse_override_rules_xml()
except KeyError as e:
dpiloganalyzer_log.info(
"The analyzer.conf file is missing the entry %s. Defaulting to 'no' and cacheability check will not be performed",
e,
)
# Parse the golden-config.txt file and get the list of domain-regex patterns
golden_config_dict = parse_golden_config(mdreq, device_map_name)
# Check if the user wants the commands to be fired to the router
if fire_to_router == "yes":
# Get the device details
FQDN_NODE = header.DEVICE_MAP_COMMON + device_map_name + "/device_info/fqdn"
USERNAME_NODE = header.DEVICE_MAP_COMMON + device_map_name + "/device_info/username"
PASSWD_NODE = header.DEVICE_MAP_COMMON + device_map_name + "/device_info/password"
device_fqdn = mdreq.query(FQDN_NODE)
device_username = mdreq.query(USERNAME_NODE)
device_passwd = mdreq.queryCleartext(PASSWD_NODE)
# First connect to the MX router
err = gen_utils.Connect_Router(device_fqdn, device_username, device_passwd, dpiloganalyzer_log)
if err == 1:
dpiloganalyzer_log.error(
"Unable to connect to the router %s. The filter-rules will be written to a file.", device_fqdn
)
fire_to_router = "no"
else:
if analyzer_conf_dict["fire_one_time_config"].lower() == "yes":
# Fire the one time commands into the router
fireOneTimeCommands(analyzer_conf_dict)
# Fire the mirror configuration commands into the router
fireMirrorConfiguration(analyzer_conf_dict)
# Load the dest-ip pickle into the dictionary
os.chdir("/var/log")
if os.path.isfile(header.DEST_IP_PICKLE) and os.stat(header.DEST_IP_PICKLE).st_size > 0:
with open(header.DEST_IP_PICKLE, "rb") as f:
dest_ips = pickle.load(f)
# Read the dpilogs genereated by the dpi-tool
while True:
read_dpi_logs(domain_patterns, golden_config_dict, analyzer_conf_dict, mdreq, device_map_name)
time.sleep(2)
def main():
global dpiloganalyzer_log
global dest_ips
global frequency
global fire_to_router
domain_patterns = ""
device_map_name = ""
#Basic common initialization
init()
#Parse the command-line arguments
#If the number of arguments is greater than 1 make sure the arguments values are correct
if len(sys.argv) > 1:
if len(sys.argv) == 3 and (
(str(sys.argv[1]) == '--mode' or str(sys.argv[1]) == '-m')
and str(sys.argv[2]) != 'mfc'):
print_usage()
elif len(
sys.argv) == 3 and str(sys.argv[1]) != '--config-path' and str(
sys.argv[1]) != '-C' and str(
sys.argv[1]) != '--mode' and str(sys.argv[1]) != '-m':
print_usage()
elif len(sys.argv) == 5 and (
(str(sys.argv[1]) != '--mode' and str(sys.argv[1]) != '-m')
or str(sys.argv[2]) != 'mfc' or
(str(sys.argv[3]) != '--config-path'
and str(sys.argv[3]) != '-C')):
print_usage()
elif len(sys.argv) == 2 or len(sys.argv) == 4:
print_usage()
#If the script is launched in mfc mode then read the MFC accesslog files
if len(sys.argv) > 1 and (
(str(sys.argv[1]) == '--mode' or str(sys.argv[1]) == '-m')
and str(sys.argv[2]) == 'mfc'):
#Parse the analyzer.conf file and populate the values in a dictionary
analyzer_conf_dict = parse_analyzer_conf()
mfc_accesslog_analyzer.read_mfc_accesslog(analyzer_conf_dict)
else:
#Logger settings
logformatter = logging.Formatter('%(asctime)s;%(message)s')
#Use the custom log rotation handler which gzip's the old rotated log file
handler = gen_utils.NewRotatingFileHandler(
header.LOG_FILE,
maxBytes=header.MAXLOG_SIZE,
backupCount=header.LOGBACKP_COUNT)
handler.setFormatter(logformatter)
dpiloganalyzer_log = logging.getLogger('__name__')
dpiloganalyzer_log.setLevel(logging.DEBUG)
#Point to the custom log handler
dpiloganalyzer_log.addHandler(handler)
#Get the handle to the instance of the class that enables us to query the mgmtd database
mdreq = MdReq()
#Get the device-map name bound to the log-analyzer
dmap = mdreq.queryIterate("/nkn/log_analyzer/config/device_map")
for key, val in dmap.iteritems():
device_map_name = val['value']
if device_map_name != "":
#Get the mode in which the log-analyzer tool has to be run
CMDMODE_NODE = '/nkn/log_analyzer/config/device_map/' + device_map_name + '/command_mode/inline'
mode = mdreq.query(CMDMODE_NODE)
if mode.lower() == 'true':
fire_to_router = 'yes'
else:
fire_to_router = 'no'
FREQUENCY_NODE = header.DEVICE_MAP_COMMON + device_map_name + '/filter_config/frequency'
frequency = mdreq.query(FREQUENCY_NODE)
#Convert the frequency in seconds as the SIGALRM requires it to be in seconds
frequency = int(frequency) * 60
else:
dpiloganalyzer_log.error(
"The log-analyzer is not associated to a device. The commands will be pushed to a file"
)
fire_to_router = "no"
#Default to 2hrs frequency to dump into the file
frequency = 7200
#Parse the analyzer.conf file and populate the values in a dictionary
analyzer_conf_dict = parse_analyzer_conf()
signal.alarm(frequency)
#Parse Override_rules.xml only if you want to determine the cacheability of url
try:
if analyzer_conf_dict['fire_pbr_for_cacheable_url'].strip().lower(
) == 'yes':
domain_patterns = parse_override_rules_xml()
except KeyError as e:
dpiloganalyzer_log.info(
"The analyzer.conf file is missing the entry %s. Defaulting to 'no' and cacheability check will not be performed",
e)
#Parse the golden-config.txt file and get the list of domain-regex patterns
golden_config_dict = parse_golden_config(mdreq, device_map_name)
#Check if the user wants the commands to be fired to the router
if fire_to_router == 'yes':
#Get the device details
FQDN_NODE = header.DEVICE_MAP_COMMON + device_map_name + '/device_info/fqdn'
USERNAME_NODE = header.DEVICE_MAP_COMMON + device_map_name + '/device_info/username'
PASSWD_NODE = header.DEVICE_MAP_COMMON + device_map_name + '/device_info/password'
device_fqdn = mdreq.query(FQDN_NODE)
device_username = mdreq.query(USERNAME_NODE)
device_passwd = mdreq.queryCleartext(PASSWD_NODE)
#First connect to the MX router
err = gen_utils.Connect_Router(device_fqdn, device_username,
device_passwd, dpiloganalyzer_log)
if err == 1:
dpiloganalyzer_log.error(
"Unable to connect to the router %s. The filter-rules will be written to a file.",
device_fqdn)
fire_to_router = "no"
else:
if analyzer_conf_dict['fire_one_time_config'].lower() == 'yes':
#Fire the one time commands into the router
fireOneTimeCommands(analyzer_conf_dict)
#Fire the mirror configuration commands into the router
fireMirrorConfiguration(analyzer_conf_dict)
#Load the dest-ip pickle into the dictionary
os.chdir("/var/log")
if os.path.isfile(header.DEST_IP_PICKLE) and os.stat(
header.DEST_IP_PICKLE).st_size > 0:
with open(header.DEST_IP_PICKLE, 'rb') as f:
dest_ips = pickle.load(f)
#Read the dpilogs genereated by the dpi-tool
while True:
read_dpi_logs(domain_patterns, golden_config_dict,
analyzer_conf_dict, mdreq, device_map_name)
time.sleep(2)
