class Exploder(object): def __init__ (self, connectionPool, thread_tracker, debug): self.debug = debug self.pool = connectionPool if thread_tracker == None: raise Exception("thread_tracker parameter can not be None") self.thread_tracker = thread_tracker self.registry = Registry(connectionPool, debug) self.num_servers = self.registry.get('hadoop.num_servers') if self.num_servers == None: self.num_servers = 1 self.batch_size = self.registry.get('hbase.batch_size') if self.batch_size == None: self.batch_size = 1000 """ We create one exploder thread per hbase server. Each thread has its own hbase connection. foreach server (1 .. numservers) spawn_exploder_thread(server) """ self.workers = [] for server in range(0, self.num_servers): thr_title = "Exploder daemon %d of %d" % (server, self.num_servers-1) worker_thr = threading.Thread(target=self.run, name=thr_title, args=(server,)) self.workers.append(worker_thr) worker_thr.daemon = True worker_thr.start() while not worker_thr.isAlive(): print "waiting for exploder/worker thread to become alive" time.sleep(1) self.thread_tracker.add(id=worker_thr.ident, user='******', host=socket.gethostname(), state='Running', info=thr_title) def L(self, msg): caller = ".".join([str(__name__), sys._getframe(1).f_code.co_name]) if self.debug != None: print caller + ": " + msg else: syslog.syslog(caller + ": " + msg) def do_some_work(self): self.kickit.release() def getcheckpoint(self, salt): t = self.registry.get('exploder.checkpoint.' + str(salt)) if t != None: return t return 0 def setcheckpoint(self, salt, ts): self.registry.set('exploder.checkpoint.' + str(salt), ts) def run(self, salt): """ run(salt) this routine scans the cif_obj db for rows starting at "salt" + last checkpoint timestamp and ending at row "salt" + now() each row read in is passed to the Indexer for indexing. """ self.L("Exploder thread running for salt: " + str(salt)) with self.pool.connection() as dbh: index_handler = {} # Indexer.Indexer(self.dbh, "botnet", self.num_servers, self.debug) while True: co = dbh.table('cif_objs') startts = self.getcheckpoint(salt) endts = int(time.time()) processed = 0 #self.L("processing: " + str(startts) + " to " + str(endts)) if startts == 0: startts = 1 srowid = struct.pack(">HIIIII", salt, startts-1, 0,0,0,0) erowid = struct.pack(">HIIIII", salt, endts, 0,0,0,0) for key, data in co.scan(row_start=srowid, row_stop=erowid): contains = data.keys()[0] obj_data = data[contains] if contains == "cf:RFC5070_IODEF_v1_pb2": iodef = RFC5070_IODEF_v1_pb2.IODEF_DocumentType() try: iodef.ParseFromString(obj_data) #print "IODEF: ", iodef ii = iodef.Incident[0] table_type = ii.Assessment[0].Impact[0].content.content self.L("\tIndexing: " + table_type) # check to make sure table_name is in index.secondary # index.secondary contains a list of configured/permitted secondary index types if not table_type in index_handler: self.L("index handler for table type %s doesnt exist, creating a new handler thread" % (table_type)) index_handler[table_type] = Indexer.Indexer(self.pool, table_type, self.num_servers, self.batch_size, self.debug) index_handler[table_type].extract(key, iodef) processed = processed + 1 except Exception as e: print "Failed to parse restored object: ", e traceback.print_exc() else: print "Contains an unsupported object type: ", contains time.sleep(5) if processed > 0: self.setcheckpoint(salt, endts)
elif o == "-D": debug = a reg = Registry("localhost", debug) if del_name != None: reg.delete(del_name) kv = reg.get(del_name) if kv != None: print "Failed to delete the key: it seems to still be in the database." elif key_name != None: if key_type != None and key_value != None: key_value = cast(key_type, key_value) reg.set(key_name, key_value) kv = reg.get(key_name) if kv == key_value: print key_name + " has been set to " + str(key_value) else: print "Failed? you gave me: " + str(key_value) + " but the database has " + str(kv) else: kv = reg.get(key_name) print key_name + " = " + str(kv) + " " + str(type(kv)) else: kl = reg.get() print "Available keys: ", kl except getopt.GetoptError as err:
class Exploder(object): def __init__ (self, hbhost, debug): self.debug = debug self.dbh = happybase.Connection(hbhost) t = self.dbh.tables() self.table = self.dbh.table('infrastructure_botnet') self.kickit = threading.Semaphore(0) self.proc_thread = threading.Thread(target=self.run, args=()) self.proc_thread.start() self.botnet_handler = Botnet.Botnet(self.dbh, debug) self.registry = Registry(hbhost, debug) self.num_servers = self.registry.get('hadoop.num_servers') if self.num_servers == None: self.num_servers = 1 self.salt = Salt(self.num_servers, self.debug) def L(self, msg): caller = ".".join([str(__name__), sys._getframe(1).f_code.co_name]) if self.debug != None: print caller + ": " + msg else: syslog.syslog(caller + ": " + msg) def do_some_work(self): self.kickit.release() def getcheckpoint(self): t = self.registry.get('exploder.checkpoint') if t != None: return t return 0 def setcheckpoint(self, ts): self.registry.set('exploder.checkpoint', ts) def run(self): self.L("Exploder running") while True: self.L("waiting for work") self.kickit.acquire() # will block provided kickit is 0 self.L("wakup") co = self.dbh.table('cif_objs') self.L("connected to cif_objs") startts = self.getcheckpoint() endts = int(time.time()) processed = 0 self.L("processing: " + str(startts) + " to " + str(endts)) salt = 0xFF00 # FIX fix in poc-db at the same time (in writeToDb()) srowid = struct.pack(">HIIIII", salt, startts, 0,0,0,0) erowid = struct.pack(">HIIIII", salt, endts, 0,0,0,0) for key, data in co.scan(row_start=srowid, row_stop=erowid): contains = data.keys()[0] obj_data = data[contains] if contains == "cf:RFC5070_IODEF_v1_pb2": iodef = RFC5070_IODEF_v1_pb2.IODEF_DocumentType() try: iodef.ParseFromString(obj_data) #print iodef ii = iodef.Incident[0] table_type = ii.Assessment[0].Impact[0].content.content rowkey = None if table_type == "botnet": self.L("botnet") self.botnet_handler.extract(iodef) self.botnet_handler.commit() elif table_type == "malware": self.L("malware") except Exception as e: print "Failed to parse restored object: ", e traceback.print_exc() self.setcheckpoint(endts+1)