def main(call='external', configfile=''):
startTime = time.time()
# if called from terminal
# if not, the parser must be called in this way: parser.main(call='internal',configfile='<route_to_config_file>')
if call is 'external':
args = getArguments()
configfile = args.config
# Get configuration
parserConfig = faac.getConfiguration(configfile)
dataSources = parserConfig['DataSources']
output = parserConfig['Learning_Output']
config = faac.loadConfig(output, dataSources, parserConfig)
# Print configuration summary
configSummary(config)
# Count data entries
stats = create_stats(config)
stats = count_entries(config, stats)
# Parse
output_data = parsing(config, startTime, stats)
# Filter output => Only filter during processing, not here, so we identify features that at relevant during a certain interval
output_data = filter_output(output_data, config['EndLperc'])
# Output results
write_output(config, output_data, stats['total_lines'])
print "Elapsed: %s \n" % (prettyTime(time.time() - startTime))
def main(call='external', configfile=''):
startTime = time.time()
# if called from terminal
# if not, the parser must be called in this way: parser.main(call='internal',configfile='<route_to_config_file>')
if call == 'external':
args = getArguments()
configfile = args.config
global debugmode
debugmode = args.debug # debugmode defined as global as it will be used in many functions
# Get configuration
print("LOADING GENERAL CONFIGURATION FILE...")
parserConfig = faac.getConfiguration(configfile)
config = faac.loadConfig(parserConfig, 'fcparser', debugmode)
# Print configuration summary
configSummary(config)
# Output Weights
outputWeight(config)
# Create stats file and count data entries
stats = create_stats(config)
stats = count_entries(config, stats)
# processing files. Online mode
if parserConfig['Online']:
data = online_parsing(config)
output_data = fuseObs_online(data)
# process offline
else:
if debugmode:
faac.debugProgram('fcparser.init_message', [])
global user_input
user_input = None # Global variable storing user input command
else:
print(
'\033[33m' +
"Note: Malformed logs or inaccurate data source configuration files will result in None variables which will not be counted in any feature."
)
print(
"Run program in debug mode with -d option to check how the records are parsed."
+ '\033[m')
data = offline_parsing(config, startTime, stats)
output_data = fuseObs_offline(data)
#with open(config['OUTDIR']+'fused_dict', 'w') as f: print(output_data, file=f) # this output file does not seem to be relevant for the user
# write in stats file
write_stats(config, stats)
# Output results
write_output(output_data, config)
print("Elapsed: %s \n" % (prettyTime(time.time() - startTime)))
def main(call='external', configfile=''):
global startTime; startTime = time.time()
# If called from terminal. If not, the parser must be called in this way: fcdeparser.main(call='internal',configfile='<route_to_config_file>')
if call == 'external':
args = getArguments()
configfile = args.config
deparsfile = args.input
global debugmode; debugmode = args.debug # debugmode defined as global as it will be used in many functions
# Get configuration
parserConfig = faac.getConfiguration(configfile)
config = faac.loadConfig(parserConfig, 'fcdeparser', debugmode)
deparsInput = getDeparsInput(deparsfile, config)
# Show init message with features and timestamps from deparsInput
initMessage(deparsInput, debugmode)
# Counters for total logs and found logs during deparsing process
count_structured = 0 # structured logs found
count_unstructured = 0 # unstructured logs found
count_tots = 0 # total structured logs
count_totu = 0 # total unstructured logs
# Iterate through features and timestamps
if deparsInput['features']:
for source in config['SOURCES']:
if not debugmode:
print ("---------------------------------------------------------------------------")
print("\nLoading '%s' data source..." %(source))
sourcepath = config['SOURCES'][source]['FILESDEP']
formated_timestamps = format_timestamps(deparsInput['timestamps'], config['TSFORMAT'][source])
# Structured sources
if config['STRUCTURED'][source]:
(cs, ct) = stru_deparsing(config, sourcepath, deparsInput, source, formated_timestamps)
count_structured += cs
count_tots += ct
# Unstructured sources
else:
(cu, ct) = unstr_deparsing(config, sourcepath, deparsInput,source, formated_timestamps)
count_unstructured += cu
count_totu += ct
print ("Elapsed: %s" %(prettyTime(time.time() - startTime)))
if not debugmode:
stats(count_structured, count_tots, count_unstructured, count_totu, config['OUTDIR'], config['OUTSTATS'], startTime)
def main():
startTime = time.time()
# Get configuration
configfile = getArguments()
parserConfig = faac.getConfiguration(configfile.config)
output = parserConfig['Deparsing_output']
threshold = parserConfig['Deparsing_output']['threshold']
dataSources = parserConfig['DataSources']
config = faac.loadConfig(output, dataSources, parserConfig)
deparsInput = getDeparsInput(configfile, config)
# Not Netflow datasources
count_structured = 0
count_unstructured = 0
count_tots = 0
count_totu = 0
count_source = 0
# iterate through features and timestams
if deparsInput['features']:
for source in dataSources:
print source
count_source = 0
sourcepath = config['SOURCES'][source]['FILESDEP']
formated_timestamps = format_timestamps(
deparsInput['timestamps'],
config['SOURCES'][source]['CONFIG']['timestamp_format'])
# Structured sources
if config['STRUCTURED'][source]:
(cs, ct) = stru_deparsing(config, threshold, sourcepath,
deparsInput, source,
formated_timestamps)
count_structured += cs
count_tots += ct
# Unstructured sources
else:
(cu, ct) = unstr_deparsing(config, threshold, sourcepath,
deparsInput, source,
formated_timestamps)
count_unstructured += cu
count_totu += ct
print "\n---------------------------------------------------------------------------\n"
print "Elapsed: %s" % (prettyTime(time.time() - startTime))
print "\n---------------------------------------------------------------------------\n"
stats(count_structured, count_tots, count_unstructured, count_totu,
config['OUTDIR'], config['OUTSTATS'], startTime)
def main(call='external', configfile=''):
startTime = time.time()
# if called from terminal
# if not, the parser must be called in this way: parser.main(call='internal',configfile='<route_to_config_file>')
if call is 'external':
args = getArguments()
configfile = args.config
# Get configuration
parserConfig = faac.getConfiguration(configfile)
online = parserConfig['Online']
dataSources = parserConfig['DataSources']
output = parserConfig['Parsing_Output']
config = faac.loadConfig(output, dataSources, parserConfig)
# Print configuration summary
configSummary(config)
# Output Weights
outputWeight(config)
stats = create_stats(config)
# Count data entries
stats = count_entries(config, stats)
# processing files
if online:
data = online_parsing(config)
output_data = fuseObs_online(data)
# process offline
else:
data = offline_parsing(config, startTime, stats)
output_data = fuseObs_offline(data)
# Output results
write_output(output_data, config)
print "Elapsed: %s \n" % (prettyTime(time.time() - startTime))