def main(call='external', configfile=''): startTime = time.time() # if called from terminal # if not, the parser must be called in this way: parser.main(call='internal',configfile='<route_to_config_file>') if call is 'external': args = getArguments() configfile = args.config # Get configuration parserConfig = faac.getConfiguration(configfile) dataSources = parserConfig['DataSources'] output = parserConfig['Learning_Output'] config = faac.loadConfig(output, dataSources, parserConfig) # Print configuration summary configSummary(config) # Count data entries stats = create_stats(config) stats = count_entries(config, stats) # Parse output_data = parsing(config, startTime, stats) # Filter output => Only filter during processing, not here, so we identify features that at relevant during a certain interval output_data = filter_output(output_data, config['EndLperc']) # Output results write_output(config, output_data, stats['total_lines']) print "Elapsed: %s \n" % (prettyTime(time.time() - startTime))
def main(call='external', configfile=''): startTime = time.time() # if called from terminal # if not, the parser must be called in this way: parser.main(call='internal',configfile='<route_to_config_file>') if call == 'external': args = getArguments() configfile = args.config global debugmode debugmode = args.debug # debugmode defined as global as it will be used in many functions # Get configuration print("LOADING GENERAL CONFIGURATION FILE...") parserConfig = faac.getConfiguration(configfile) config = faac.loadConfig(parserConfig, 'fcparser', debugmode) # Print configuration summary configSummary(config) # Output Weights outputWeight(config) # Create stats file and count data entries stats = create_stats(config) stats = count_entries(config, stats) # processing files. Online mode if parserConfig['Online']: data = online_parsing(config) output_data = fuseObs_online(data) # process offline else: if debugmode: faac.debugProgram('fcparser.init_message', []) global user_input user_input = None # Global variable storing user input command else: print( '\033[33m' + "Note: Malformed logs or inaccurate data source configuration files will result in None variables which will not be counted in any feature." ) print( "Run program in debug mode with -d option to check how the records are parsed." + '\033[m') data = offline_parsing(config, startTime, stats) output_data = fuseObs_offline(data) #with open(config['OUTDIR']+'fused_dict', 'w') as f: print(output_data, file=f) # this output file does not seem to be relevant for the user # write in stats file write_stats(config, stats) # Output results write_output(output_data, config) print("Elapsed: %s \n" % (prettyTime(time.time() - startTime)))
def main(call='external', configfile=''): global startTime; startTime = time.time() # If called from terminal. If not, the parser must be called in this way: fcdeparser.main(call='internal',configfile='<route_to_config_file>') if call == 'external': args = getArguments() configfile = args.config deparsfile = args.input global debugmode; debugmode = args.debug # debugmode defined as global as it will be used in many functions # Get configuration parserConfig = faac.getConfiguration(configfile) config = faac.loadConfig(parserConfig, 'fcdeparser', debugmode) deparsInput = getDeparsInput(deparsfile, config) # Show init message with features and timestamps from deparsInput initMessage(deparsInput, debugmode) # Counters for total logs and found logs during deparsing process count_structured = 0 # structured logs found count_unstructured = 0 # unstructured logs found count_tots = 0 # total structured logs count_totu = 0 # total unstructured logs # Iterate through features and timestamps if deparsInput['features']: for source in config['SOURCES']: if not debugmode: print ("---------------------------------------------------------------------------") print("\nLoading '%s' data source..." %(source)) sourcepath = config['SOURCES'][source]['FILESDEP'] formated_timestamps = format_timestamps(deparsInput['timestamps'], config['TSFORMAT'][source]) # Structured sources if config['STRUCTURED'][source]: (cs, ct) = stru_deparsing(config, sourcepath, deparsInput, source, formated_timestamps) count_structured += cs count_tots += ct # Unstructured sources else: (cu, ct) = unstr_deparsing(config, sourcepath, deparsInput,source, formated_timestamps) count_unstructured += cu count_totu += ct print ("Elapsed: %s" %(prettyTime(time.time() - startTime))) if not debugmode: stats(count_structured, count_tots, count_unstructured, count_totu, config['OUTDIR'], config['OUTSTATS'], startTime)
def main(): startTime = time.time() # Get configuration configfile = getArguments() parserConfig = faac.getConfiguration(configfile.config) output = parserConfig['Deparsing_output'] threshold = parserConfig['Deparsing_output']['threshold'] dataSources = parserConfig['DataSources'] config = faac.loadConfig(output, dataSources, parserConfig) deparsInput = getDeparsInput(configfile, config) # Not Netflow datasources count_structured = 0 count_unstructured = 0 count_tots = 0 count_totu = 0 count_source = 0 # iterate through features and timestams if deparsInput['features']: for source in dataSources: print source count_source = 0 sourcepath = config['SOURCES'][source]['FILESDEP'] formated_timestamps = format_timestamps( deparsInput['timestamps'], config['SOURCES'][source]['CONFIG']['timestamp_format']) # Structured sources if config['STRUCTURED'][source]: (cs, ct) = stru_deparsing(config, threshold, sourcepath, deparsInput, source, formated_timestamps) count_structured += cs count_tots += ct # Unstructured sources else: (cu, ct) = unstr_deparsing(config, threshold, sourcepath, deparsInput, source, formated_timestamps) count_unstructured += cu count_totu += ct print "\n---------------------------------------------------------------------------\n" print "Elapsed: %s" % (prettyTime(time.time() - startTime)) print "\n---------------------------------------------------------------------------\n" stats(count_structured, count_tots, count_unstructured, count_totu, config['OUTDIR'], config['OUTSTATS'], startTime)
def main(call='external', configfile=''): startTime = time.time() # if called from terminal # if not, the parser must be called in this way: parser.main(call='internal',configfile='<route_to_config_file>') if call is 'external': args = getArguments() configfile = args.config # Get configuration parserConfig = faac.getConfiguration(configfile) online = parserConfig['Online'] dataSources = parserConfig['DataSources'] output = parserConfig['Parsing_Output'] config = faac.loadConfig(output, dataSources, parserConfig) # Print configuration summary configSummary(config) # Output Weights outputWeight(config) stats = create_stats(config) # Count data entries stats = count_entries(config, stats) # processing files if online: data = online_parsing(config) output_data = fuseObs_online(data) # process offline else: data = offline_parsing(config, startTime, stats) output_data = fuseObs_offline(data) # Output results write_output(output_data, config) print "Elapsed: %s \n" % (prettyTime(time.time() - startTime))