def __init__(self, tree, count, inter, iface, strict, capmethod, \ scallback, rcallback, sudata, rudata, excback): """ Create a SequenceConsumer object. @param tree a tree represantation of the packets to send @param count how many time we have to send the tree @param inter the interval to wait between 2 consecutive sends @param iface the interface to send/recv on @param capmethod the method to use 0 for native, 1 for tcpdump, 2 for dumpcap @param scallback the send callback @param rcallback the receive callback @param sudata user data for send callback @param rudata user data for receive callback @param excback exception callback """ assert len(tree) > 0 self.tree = tree self.count = count self.inter = inter self.strict = strict self.iface = iface self.timeout = None self.capmethod = capmethod self.procs = {} self.sockets = [] self.recv_list = defaultdict(list) self.receiving = False self.internal = False self.active_helpers = 0 self.active_helpers_lock = Lock() self.running = Condition() self.pool = ThreadPool(2, 10) self.pool.queue_work(None, self.__notify_exc, self.__check) self.scallback = scallback self.rcallback = rcallback self.excback = excback self.sudata, self.rudata = sudata, rudata log.debug("%d total packets to send for %d times" % (len(tree), self.count))
class SequenceConsumer(Interruptable): # This class most probably suffers when there's repeated sequence # because we destroy and then recreate various process when the refcount # associated to the sockets or process reach 0. Should be optimized to # better handle this situation. def __init__(self, tree, count, inter, iface, strict, capmethod, \ scallback, rcallback, sudata, rudata, excback): """ Create a SequenceConsumer object. @param tree a tree represantation of the packets to send @param count how many time we have to send the tree @param inter the interval to wait between 2 consecutive sends @param iface the interface to send/recv on @param capmethod the method to use 0 for native, 1 for tcpdump, 2 for dumpcap @param scallback the send callback @param rcallback the receive callback @param sudata user data for send callback @param rudata user data for receive callback @param excback exception callback """ assert len(tree) > 0 self.tree = tree self.count = count self.inter = inter self.strict = strict self.iface = iface self.timeout = None self.capmethod = capmethod self.procs = {} self.sockets = [] self.recv_list = defaultdict(list) self.receiving = False self.internal = False self.active_helpers = 0 self.active_helpers_lock = Lock() self.running = Condition() self.pool = ThreadPool(2, 10) self.pool.queue_work(None, self.__notify_exc, self.__check) self.scallback = scallback self.rcallback = rcallback self.excback = excback self.sudata, self.rudata = sudata, rudata log.debug("%d total packets to send for %d times" % (len(tree), self.count)) def isAlive(self): return self.internal def stop(self): self.internal = False self.pool.stop() #self.pool.join_threads() def terminate(self): self.stop() def start(self): if self.internal or self.receiving: log.debug("Pool already started") return self.receiving = True self.internal = True self.pool.start() def __check(self): # This is a function to allow the sequence # to be respawned n times self.receiving = True self.running.acquire() if not self.count: log.debug("This is an infinite loop") self.count = -1 while self.internal and self.count: self.receiving = True if self.count > 0: log.debug("Next step %d" % self.count) else: log.debug("Another loop (infinite)") self.__notify_send(None) if self.capmethod == 0: # If we use native capmethod we should run this self.pool.queue_work(None, self.__notify_exc, self.__recv_worker) for node in self.tree.get_children(): log.debug("Adding first packet of the sequence") self.pool.queue_work(None, self.__notify_exc, self.__send_worker, node) break log.debug("Waiting recv to begin another loop") self.running.wait() if self.count > 0: self.count -= 1 self.running.release() if not self.internal: log.debug("Stopping the thread pool (async)") self.pool.stop() log.debug("Finished") def __recv_helper_worker(self, udata): (process, outfile) = udata self.active_helpers_lock.acquire() if self.active_helpers is not None: self.active_helpers += 1 self.active_helpers_lock.release() if self.timeout is not None: stoptime = time.time() + self.timeout for reader in bind_reader(process, outfile): if self.timeout is not None: remain = stoptime - time.time() if remain <= 0: self.receiving = False log.debug("Timeout here!") break if not self.running or not self.receiving: break if reader: reader, outfile_size, position = reader report_idx = 0 reported_packets = 0 while self.running and self.receiving: if self.timeout is not None: remain = stoptime - time.time() if remain <= 0: self.receiving = False log.debug("Timeout here!") break while report_idx < reported_packets: report_idx = get_n_packets(self.process) r = reader.read_packet() try: # The helper capture packets at L2 so we need to drop the # first protocol to make the match against the packets. r = r[1] except: continue if not r: break is_reply = True my_node = None requested_process = None if self.strict: is_reply = False hashret = r.hashret() if hashret in self.recv_list: for (idx, process, node) in self.recv_list[hashret]: packet = node.get_data().packet.root if r.answers(packet): requested_process = process my_node = node is_reply = True break elif not self.strict and my_node is None: # Get the first packet list = [(v, k) for k, v in self.recv_list.items()] list.sort() requested_process = list[0][0][0][1] my_node = list[0][0][0][2] else: continue # Now cleanup the sockets for key in self.procs: if self.procs[key][0] == requested_process: self.procs[key][2] -= 1 if self.procs[key][2] == 0: process, outfile, refcount = self.procs[key] log.debug("Killing helper %s cause refcount == 0" % \ process) kill_helper(process) del self.procs[key] break if is_reply: self.__notify_recv(my_node, MetaPacket(r), is_reply) # Queue another send thread for node in my_node.get_children(): self.pool.queue_work(None, self.__notify_exc, self.__send_worker, node) else: self.__notify_recv(None, MetaPacket(r), is_reply) # Here we've to check if this current thread is also the last # thread used to receive packets from the helper process. self.active_helpers_lock.acquire() if self.active_helpers is not None: self.active_helpers -= 1 if self.active_helpers == 0: log.debug("Trying to exit") self.running.acquire() self.running.notify() self.running.release() self.receiving = False self.active_helpers = None self.active_helpers_lock.release() self.__notify_recv(None, None, False) return self.active_helpers_lock.release() def __recv_worker(self): # Here we should receive the packet and check against # recv_list if the packet match remove from the list # and start another send_worker if self.timeout is not None: stoptime = time.time() + self.timeout while self.internal and self.receiving: r = [] inmask = [socket for socket, refcount in self.sockets] if self.timeout is not None: remain = stoptime - time.time() if remain <= 0: self.receiving = False log.debug("Timeout here!") break if not inmask: time.sleep(0.05) if FREEBSD or DARWIN: inp, out, err = select(inmask, [], [], 0.05) for sock in inp: r.append(sock.nonblock_recv()) elif WINDOWS: for sock in inmask: r.append(sock.recv(MTU)) else: # FIXME: needs a revision here! possibly packet lost inp, out, err = select(inmask, [], [], 0.05) for sock in inp: r.append(sock.recv(MTU)) if not r: continue if self.timeout is not None: stoptime = time.time() + self.timeout for precv in r: if precv is None: continue is_reply = True my_node = None requested_socket = None if self.strict: is_reply = False hashret = precv.hashret() if hashret in self.recv_list: for (idx, sock, node) in self.recv_list[hashret]: packet = node.get_data().packet.root if precv.answers(packet): requested_socket = sock my_node = node is_reply = True break elif not self.strict and my_node is None: # Get the first packet list = [(v, k) for k, v in self.recv_list.items()] list.sort() requested_socket = list[0][0][0][1] my_node = list[0][0][0][2] else: continue # Now cleanup the sockets for idx in xrange(len(self.sockets)): if self.sockets[idx][0] == requested_socket: self.sockets[idx][1] -= 1 if self.sockets[idx][1] == 0: self.sockets.remove(self.sockets[idx]) break if is_reply: self.__notify_recv(my_node, MetaPacket(precv), is_reply) # Queue another send thread for node in my_node.get_children(): self.pool.queue_work(None, self.__notify_exc, self.__send_worker, node) else: self.__notify_recv(None, MetaPacket(precv), is_reply) log.debug("Trying to exit") self.running.acquire() self.running.notify() self.running.release() self.receiving = False self.__notify_recv(None, None, False) def __send_worker(self, node): if not self.internal: log.debug("Discarding packet") return obj = node.get_data() sock = get_socket_for(obj.packet, iff=self.iface) if node.is_parent(): # Here we should add the node to the dict # to check the the replies for a given time # and continue the sequence with the next # depth. if self.capmethod == 0: try: idx = self.sockets.index(sock) self.sockets[idx][1] += 1 except: self.sockets.append([sock, 1]) key = obj.packet.root.hashret() self.recv_list[key].append((len(self.recv_list), sock, node)) log.debug( "Adding socket to the list for receiving my packet %s" % sock) else: # TODO: here we could create another thread that spawns tcpdump # process. We have to resize also the pool directly leaving n # available threads by having n sniff process and only 1 to send # packets. All controls should be in __check() iface = get_iface_from_ip(obj.packet) if not iface in self.procs: process, outfile = run_helper(self.capmethod - 1, iface) self.procs[iface] = [process, outfile, 1] # Just increase the size of our pool to avoid starvation self.pool.resize(maxthreads=self.pool.max + 1) # And now start a new worker self.pool.queue_work(None, self.__notify_exc, self.__recv_helper_worker, (process, outfile)) else: self.procs[iface][2] += 1 process, outfile = self.procs[iface][0:2] log.debug("A process sniffing on %s exists." % iface) key = obj.packet.root.hashret() self.recv_list[key].append( (len(self.recv_list), process, node)) sock.send(obj.packet.root) self.__notify_send(node) log.debug("Sleeping %f after send" % self.inter) time.sleep(self.inter + obj.inter) if self.internal and node.get_parent(): parent = node.get_parent() next = parent.get_next_of(node) if next: log.debug("Processing next packet") self.pool.queue_work(None, self.__notify_exc, self.__send_worker, next) else: log.debug("Last packet of this level") self.__notify_recv(None, None, False) else: log.debug("Last packet sent") self.__notify_recv(None, None, False) def __notify_exc(self, exc): self.scallback = None self.rcallback = None if isinstance(exc, socket.error): exc = Exception(str(exc[1])) if self.excback: self.excback(exc) else: log.debug("Exception not properly handled. Dumping:") traceback.print_exc(file=sys.stdout) self.stop() def __notify_send(self, node): log.debug("Packet sent") if not self.scallback: return packet = None parent = False if node is not None: packet = node.get_data().packet parent = node.is_parent() if self.scallback(packet, parent, self.sudata): log.debug("send_callback want to exit") self.internal = False def __notify_recv(self, node, reply, is_reply): log.debug("Packet received (is reply? %s)" % is_reply) if not self.rcallback: return packet = None if node is not None: packet = node.get_data().packet if self.rcallback(packet, reply, is_reply, self.rudata): log.debug("recv_callback want to exit") self.internal = False
class SequenceConsumer(Interruptable): # This class most probably suffers when there's repeated sequence # because we destroy and then recreate various process when the refcount # associated to the sockets or process reach 0. Should be optimized to # better handle this situation. def __init__(self, tree, count, inter, iface, strict, capmethod, \ scallback, rcallback, sudata, rudata, excback): """ Create a SequenceConsumer object. @param tree a tree represantation of the packets to send @param count how many time we have to send the tree @param inter the interval to wait between 2 consecutive sends @param iface the interface to send/recv on @param capmethod the method to use 0 for native, 1 for tcpdump, 2 for dumpcap @param scallback the send callback @param rcallback the receive callback @param sudata user data for send callback @param rudata user data for receive callback @param excback exception callback """ assert len(tree) > 0 self.tree = tree self.count = count self.inter = inter self.strict = strict self.iface = iface self.timeout = None self.capmethod = capmethod self.procs = {} self.sockets = [] self.recv_list = defaultdict(list) self.receiving = False self.internal = False self.active_helpers = 0 self.active_helpers_lock = Lock() self.running = Condition() self.pool = ThreadPool(2, 10) self.pool.queue_work(None, self.__notify_exc, self.__check) self.scallback = scallback self.rcallback = rcallback self.excback = excback self.sudata, self.rudata = sudata, rudata log.debug("%d total packets to send for %d times" % (len(tree), self.count)) def isAlive(self): return self.internal def stop(self): self.internal = False self.pool.stop() #self.pool.join_threads() def terminate(self): self.stop() def start(self): if self.internal or self.receiving: log.debug("Pool already started") return self.receiving = True self.internal = True self.pool.start() def __check(self): # This is a function to allow the sequence # to be respawned n times self.receiving = True self.running.acquire() if not self.count: log.debug("This is an infinite loop") self.count = -1 while self.internal and self.count: self.receiving = True if self.count > 0: log.debug("Next step %d" % self.count) else: log.debug("Another loop (infinite)") self.__notify_send(None) if self.capmethod == 0: # If we use native capmethod we should run this self.pool.queue_work(None, self.__notify_exc, self.__recv_worker) for node in self.tree.get_children(): log.debug("Adding first packet of the sequence") self.pool.queue_work(None, self.__notify_exc, self.__send_worker, node) break log.debug("Waiting recv to begin another loop") self.running.wait() if self.count > 0: self.count -= 1 self.running.release() if not self.internal: log.debug("Stopping the thread pool (async)") self.pool.stop() log.debug("Finished") def __recv_helper_worker(self, udata): (process, outfile) = udata self.active_helpers_lock.acquire() if self.active_helpers is not None: self.active_helpers += 1 self.active_helpers_lock.release() if self.timeout is not None: stoptime = time.time() + self.timeout for reader in bind_reader(process, outfile): if self.timeout is not None: remain = stoptime - time.time() if remain <= 0: self.receiving = False log.debug("Timeout here!") break if not self.running or not self.receiving: break if reader: reader, outfile_size, position = reader report_idx = 0 reported_packets = 0 while self.running and self.receiving: if self.timeout is not None: remain = stoptime - time.time() if remain <= 0: self.receiving = False log.debug("Timeout here!") break while report_idx < reported_packets: report_idx = get_n_packets(self.process) r = reader.read_packet() try: # The helper capture packets at L2 so we need to drop the # first protocol to make the match against the packets. r = r[1] except: continue if not r: break is_reply = True my_node = None requested_process = None if self.strict: is_reply = False hashret = r.hashret() if hashret in self.recv_list: for (idx, process, node) in self.recv_list[hashret]: packet = node.get_data().packet.root if r.answers(packet): requested_process = process my_node = node is_reply = True break elif not self.strict and my_node is None: # Get the first packet list = [(v, k) for k, v in self.recv_list.items()] list.sort() requested_process = list[0][0][0][1] my_node = list[0][0][0][2] else: continue # Now cleanup the sockets for key in self.procs: if self.procs[key][0] == requested_process: self.procs[key][2] -= 1 if self.procs[key][2] == 0: process, outfile, refcount = self.procs[key] log.debug("Killing helper %s cause refcount == 0" % \ process) kill_helper(process) del self.procs[key] break if is_reply: self.__notify_recv(my_node, MetaPacket(r), is_reply) # Queue another send thread for node in my_node.get_children(): self.pool.queue_work(None, self.__notify_exc, self.__send_worker, node) else: self.__notify_recv(None, MetaPacket(r), is_reply) # Here we've to check if this current thread is also the last # thread used to receive packets from the helper process. self.active_helpers_lock.acquire() if self.active_helpers is not None: self.active_helpers -= 1 if self.active_helpers == 0: log.debug("Trying to exit") self.running.acquire() self.running.notify() self.running.release() self.receiving = False self.active_helpers = None self.active_helpers_lock.release() self.__notify_recv(None, None, False) return self.active_helpers_lock.release() def __recv_worker(self): # Here we should receive the packet and check against # recv_list if the packet match remove from the list # and start another send_worker if self.timeout is not None: stoptime = time.time() + self.timeout while self.internal and self.receiving: r = [] inmask = [socket for socket, refcount in self.sockets] if self.timeout is not None: remain = stoptime - time.time() if remain <= 0: self.receiving = False log.debug("Timeout here!") break if not inmask: time.sleep(0.05) if FREEBSD or DARWIN: inp, out, err = select(inmask, [], [], 0.05) for sock in inp: r.append(sock.nonblock_recv()) elif WINDOWS: for sock in inmask: r.append(sock.recv(MTU)) else: # FIXME: needs a revision here! possibly packet lost inp, out, err = select(inmask, [], [], 0.05) for sock in inp: r.append(sock.recv(MTU)) if not r: continue if self.timeout is not None: stoptime = time.time() + self.timeout for precv in r: if precv is None: continue is_reply = True my_node = None requested_socket = None if self.strict: is_reply = False hashret = precv.hashret() if hashret in self.recv_list: for (idx, sock, node) in self.recv_list[hashret]: packet = node.get_data().packet.root if precv.answers(packet): requested_socket = sock my_node = node is_reply = True break elif not self.strict and my_node is None: # Get the first packet list = [(v, k) for k, v in self.recv_list.items()] list.sort() requested_socket = list[0][0][0][1] my_node = list[0][0][0][2] else: continue # Now cleanup the sockets for idx in xrange(len(self.sockets)): if self.sockets[idx][0] == requested_socket: self.sockets[idx][1] -= 1 if self.sockets[idx][1] == 0: self.sockets.remove(self.sockets[idx]) break if is_reply: self.__notify_recv(my_node, MetaPacket(precv), is_reply) # Queue another send thread for node in my_node.get_children(): self.pool.queue_work(None, self.__notify_exc, self.__send_worker, node) else: self.__notify_recv(None, MetaPacket(precv), is_reply) log.debug("Trying to exit") self.running.acquire() self.running.notify() self.running.release() self.receiving = False self.__notify_recv(None, None, False) def __send_worker(self, node): if not self.internal: log.debug("Discarding packet") return obj = node.get_data() sock = get_socket_for(obj.packet, iff=self.iface) if node.is_parent(): # Here we should add the node to the dict # to check the the replies for a given time # and continue the sequence with the next # depth. if self.capmethod == 0: try: idx = self.sockets.index(sock) self.sockets[idx][1] += 1 except: self.sockets.append([sock, 1]) key = obj.packet.root.hashret() self.recv_list[key].append((len(self.recv_list), sock, node)) log.debug("Adding socket to the list for receiving my packet %s" % sock) else: # TODO: here we could create another thread that spawns tcpdump # process. We have to resize also the pool directly leaving n # available threads by having n sniff process and only 1 to send # packets. All controls should be in __check() iface = get_iface_from_ip(obj.packet) if not iface in self.procs: process, outfile = run_helper(self.capmethod - 1, iface) self.procs[iface] = [process, outfile, 1] # Just increase the size of our pool to avoid starvation self.pool.resize(maxthreads=self.pool.max + 1) # And now start a new worker self.pool.queue_work(None, self.__notify_exc, self.__recv_helper_worker, (process, outfile)) else: self.procs[iface][2] += 1 process, outfile = self.procs[iface][0:2] log.debug("A process sniffing on %s exists." % iface) key = obj.packet.root.hashret() self.recv_list[key].append((len(self.recv_list), process, node)) sock.send(obj.packet.root) self.__notify_send(node) log.debug("Sleeping %f after send" % self.inter) time.sleep(self.inter + obj.inter) if self.internal and node.get_parent(): parent = node.get_parent() next = parent.get_next_of(node) if next: log.debug("Processing next packet") self.pool.queue_work(None, self.__notify_exc, self.__send_worker, next) else: log.debug("Last packet of this level") self.__notify_recv(None, None, False) else: log.debug("Last packet sent") self.__notify_recv(None, None, False) def __notify_exc(self, exc): self.scallback = None self.rcallback = None if isinstance(exc, socket.error): exc = Exception(str(exc[1])) if self.excback: self.excback(exc) else: log.debug("Exception not properly handled. Dumping:") traceback.print_exc(file=sys.stdout) self.stop() def __notify_send(self, node): log.debug("Packet sent") if not self.scallback: return packet = None parent = False if node is not None: packet = node.get_data().packet parent = node.is_parent() if self.scallback(packet, parent, self.sudata): log.debug("send_callback want to exit") self.internal = False def __notify_recv(self, node, reply, is_reply): log.debug("Packet received (is reply? %s)" % is_reply) if not self.rcallback: return packet = None if node is not None: packet = node.get_data().packet if self.rcallback(packet, reply, is_reply, self.rudata): log.debug("recv_callback want to exit") self.internal = False