result.language = languages.__all__[lang] # Read NPSL result.input.update( npsl.parseString( "".join(result.input_file.readlines()) )[0] ) result.input.update( get_default_settings( result.input["settings"] ) ) try: result.language.set_options(result, optargs) except RuntimeError, err: raise ArgumentError(err.message) # Make output folder if it doesn't exist if not os.path.exists( result.output_folder ): os.mkdir( result.output_folder ) return result if __name__ == "__main__": optionparser = makeOptionParser() if len( sys.argv ) <= 1: optionparser.print_help() sys.exit(-1) try: o = Output( process_options( optionparser.parse_args() ) ) o.run() except ArgumentError, err: sys.stderr.write( str( err ) + "\r\n" ) sys.exit(-1)
class Collector(DatagramServer): x = 0 def __init__(self,args): # create logger self.logger = logging.getLogger(__name__) self.logger.setLevel(logging.DEBUG) # create console handler and set level to debug ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) # create formatter formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') # add formatter to ch ch.setFormatter(formatter) # add ch to logger self.logger.addHandler(ch) self.logger.debug( "Starting Collector process in %s"%os.getcwd()) #self.logger.debug( "Gevent Version %s"%gevent.__version__) #TODO: move output file name to config fname = "./NetFlow.%d.bin"%int(time.time()) #WARN: might want to remove this after testing self.out = open(fname,"wb") #create tool instances self.interface = Interface() self.parse = Parse() self.context = Context() self.describe = Describe() self.standardize = Standardize() self.transform = Transform() self.partition = Partition() self.q = Queue() self.inWindow = settings.SETTINGS.get("collector","inWindow") self.score = Score() #TODO: move csv name to config #self.csv = CSV("output.csv") self.output = Output() return super(Collector,self).__init__(args) def done(self): #pass self.out.close() #really important to call del on the csv obj to ensure it closes correctly #del self.csv def handle(self, rawData, address): Collector.x += 1 #print '%s %s: got %r' % (Collector.x, address[0], rawData) self.out.write(rawData) interfacedData = self.interface.run(rawData) #self.logger.debug("Interface: %s"%(repr(interfacedData))) #once the rawData is "interfaced" we are passing it around by reference # interfaced data must be iterable try: for record in interfacedData: self.parse.run(record) #self.logger.debug("Parse: %s"%(repr(record))) self.context.run(record) #self.logger.debug("Context: %s"%(repr(record))) self.describe.run(record) #self.logger.debug("Describe: %s"%(repr(record))) #push the record onto the queue until window if not (self.inWindow): self.q.put(record) #self.logger.debug("adding record to queue %s"%(repr(record))) if (self.q.qsize() == int(settings.SETTINGS.get("collector","describeWindow"))): #self.logger.debug("Describe Window of %s records met, Begin Processing queue"%settings.SETTINGS.get("collector","describeWindow")) self.inWindow = True while not self.q.empty(): item = self.q.get() #self.logger.debug("processing record from queue %s"%(repr(item))) self.standardize.run(item) self.transform.run(item) self.partition.run(item) #self.csv.writeRow(self.csv.format(item)) self.output.run(item) self.q.task_done() else: self.standardize.run(record) #self.logger.debug("Standardize: %s"%(repr(record))) self.transform.run(record) #self.logger.debug("Transform: %s"%(repr(record))) self.partition.run(record) #self.logger.debug("Partition: %s"%(repr(record))) #self.csv.writeRow(self.csv.format(record)) self.output.run(record) #self.score.run(record) except Exception as e: self.logger.error("Interfaced data is not iterable %s"%(str(e)))