class Exploder(object):
def __init__ (self, connectionPool, thread_tracker, debug):
self.debug = debug
self.pool = connectionPool
if thread_tracker == None:
raise Exception("thread_tracker parameter can not be None")
self.thread_tracker = thread_tracker
self.registry = Registry(connectionPool, debug)
self.num_servers = self.registry.get('hadoop.num_servers')
if self.num_servers == None:
self.num_servers = 1
self.batch_size = self.registry.get('hbase.batch_size')
if self.batch_size == None:
self.batch_size = 1000
"""
We create one exploder thread per hbase server. Each thread has its own
hbase connection.
foreach server (1 .. numservers)
spawn_exploder_thread(server)
"""
self.workers = []
for server in range(0, self.num_servers):
thr_title = "Exploder daemon %d of %d" % (server, self.num_servers-1)
worker_thr = threading.Thread(target=self.run, name=thr_title, args=(server,))
self.workers.append(worker_thr)
worker_thr.daemon = True
worker_thr.start()
while not worker_thr.isAlive():
print "waiting for exploder/worker thread to become alive"
time.sleep(1)
self.thread_tracker.add(id=worker_thr.ident, user='******', host=socket.gethostname(), state='Running', info=thr_title)
def L(self, msg):
caller = ".".join([str(__name__), sys._getframe(1).f_code.co_name])
if self.debug != None:
print caller + ": " + msg
else:
syslog.syslog(caller + ": " + msg)
def do_some_work(self):
self.kickit.release()
def getcheckpoint(self, salt):
t = self.registry.get('exploder.checkpoint.' + str(salt))
if t != None:
return t
return 0
def setcheckpoint(self, salt, ts):
self.registry.set('exploder.checkpoint.' + str(salt), ts)
def run(self, salt):
"""
run(salt)
this routine scans the cif_obj db for rows starting at
"salt" + last checkpoint timestamp
and ending at row
"salt" + now()
each row read in is passed to the Indexer for indexing.
"""
self.L("Exploder thread running for salt: " + str(salt))
with self.pool.connection() as dbh:
index_handler = {} # Indexer.Indexer(self.dbh, "botnet", self.num_servers, self.debug)
while True:
co = dbh.table('cif_objs')
startts = self.getcheckpoint(salt)
endts = int(time.time())
processed = 0
#self.L("processing: " + str(startts) + " to " + str(endts))
if startts == 0:
startts = 1
srowid = struct.pack(">HIIIII", salt, startts-1, 0,0,0,0)
erowid = struct.pack(">HIIIII", salt, endts, 0,0,0,0)
for key, data in co.scan(row_start=srowid, row_stop=erowid):
contains = data.keys()[0]
obj_data = data[contains]
if contains == "cf:RFC5070_IODEF_v1_pb2":
iodef = RFC5070_IODEF_v1_pb2.IODEF_DocumentType()
try:
iodef.ParseFromString(obj_data)
#print "IODEF: ", iodef
ii = iodef.Incident[0]
table_type = ii.Assessment[0].Impact[0].content.content
self.L("\tIndexing: " + table_type)
# check to make sure table_name is in index.secondary
# index.secondary contains a list of configured/permitted secondary index types
if not table_type in index_handler:
self.L("index handler for table type %s doesnt exist, creating a new handler thread" % (table_type))
index_handler[table_type] = Indexer.Indexer(self.pool, table_type, self.num_servers, self.batch_size, self.debug)
index_handler[table_type].extract(key, iodef)
processed = processed + 1
except Exception as e:
print "Failed to parse restored object: ", e
traceback.print_exc()
else:
print "Contains an unsupported object type: ", contains
time.sleep(5)
if processed > 0:
self.setcheckpoint(salt, endts)
elif o == "-D":
debug = a
reg = Registry("localhost", debug)
if del_name != None:
reg.delete(del_name)
kv = reg.get(del_name)
if kv != None:
print "Failed to delete the key: it seems to still be in the database."
elif key_name != None:
if key_type != None and key_value != None:
key_value = cast(key_type, key_value)
reg.set(key_name, key_value)
kv = reg.get(key_name)
if kv == key_value:
print key_name + " has been set to " + str(key_value)
else:
print "Failed? you gave me: " + str(key_value) + " but the database has " + str(kv)
else:
kv = reg.get(key_name)
print key_name + " = " + str(kv) + " " + str(type(kv))
else:
kl = reg.get()
print "Available keys: ", kl
except getopt.GetoptError as err:
class Exploder(object):
def __init__ (self, hbhost, debug):
self.debug = debug
self.dbh = happybase.Connection(hbhost)
t = self.dbh.tables()
self.table = self.dbh.table('infrastructure_botnet')
self.kickit = threading.Semaphore(0)
self.proc_thread = threading.Thread(target=self.run, args=())
self.proc_thread.start()
self.botnet_handler = Botnet.Botnet(self.dbh, debug)
self.registry = Registry(hbhost, debug)
self.num_servers = self.registry.get('hadoop.num_servers')
if self.num_servers == None:
self.num_servers = 1
self.salt = Salt(self.num_servers, self.debug)
def L(self, msg):
caller = ".".join([str(__name__), sys._getframe(1).f_code.co_name])
if self.debug != None:
print caller + ": " + msg
else:
syslog.syslog(caller + ": " + msg)
def do_some_work(self):
self.kickit.release()
def getcheckpoint(self):
t = self.registry.get('exploder.checkpoint')
if t != None:
return t
return 0
def setcheckpoint(self, ts):
self.registry.set('exploder.checkpoint', ts)
def run(self):
self.L("Exploder running")
while True:
self.L("waiting for work")
self.kickit.acquire() # will block provided kickit is 0
self.L("wakup")
co = self.dbh.table('cif_objs')
self.L("connected to cif_objs")
startts = self.getcheckpoint()
endts = int(time.time())
processed = 0
self.L("processing: " + str(startts) + " to " + str(endts))
salt = 0xFF00 # FIX fix in poc-db at the same time (in writeToDb())
srowid = struct.pack(">HIIIII", salt, startts, 0,0,0,0)
erowid = struct.pack(">HIIIII", salt, endts, 0,0,0,0)
for key, data in co.scan(row_start=srowid, row_stop=erowid):
contains = data.keys()[0]
obj_data = data[contains]
if contains == "cf:RFC5070_IODEF_v1_pb2":
iodef = RFC5070_IODEF_v1_pb2.IODEF_DocumentType()
try:
iodef.ParseFromString(obj_data)
#print iodef
ii = iodef.Incident[0]
table_type = ii.Assessment[0].Impact[0].content.content
rowkey = None
if table_type == "botnet":
self.L("botnet")
self.botnet_handler.extract(iodef)
self.botnet_handler.commit()
elif table_type == "malware":
self.L("malware")
except Exception as e:
print "Failed to parse restored object: ", e
traceback.print_exc()
self.setcheckpoint(endts+1)
