def __init__(self, dest_dir, swift_binpath, dispersy_work_dir=u".", sqlite_database=":memory:", swift_work_dir=None, swift_zerostatedir=None, listen=[], peers=[], file_directories=[], files=[], file_timestamp_min=None, run_time=-1, bloomfilter_update=-1, walker=False, gateways={}, mtu=MAX_MTU, callback=None): """ @param dest_dir: Directory in which downloads will be placed as well as logs @param swift_binpath: Path to the swift executable @param dispersy_work_dir: Working directory for Dispersy @param sqlite_database: Location of the sqlite_database, :memory: is in memory @param swift_work_dir: Working directory for Swift @param swift_zerostatedir: Zero state directory for Swift @param listen: Addresses of local sockets to bind to @param peers: Addresses of peers to communicate with @param files_directory: Directory to monitor for files that should be disseminated @param files: Files that should be disseminated @param file_timestamp_min: Minimum file modification time @param run_time: Total run time in seconds @param bloomfilter_update: Period after which another introduction request is sent in seconds @param walker: If enable the Dispersy walker will be enabled @param gateways: Ip addresses of the gateways for specific interfaces (eth0=192.168.0.1) @param mtu: Maximum transmission unit directs the maximum size of messages @param callback: Callback function that will called on certain events """ self._dest_dir = dest_dir self._swift_binpath = swift_binpath self._dispersy_work_dir = unicode(dispersy_work_dir) self._sqlite_database = unicode(sqlite_database) # :memory: is in memory self._swift_work_dir = swift_work_dir if swift_work_dir is not None else dest_dir self._swift_zerostatedir = swift_zerostatedir self._listen = [Address.unknown(l) for l in listen] # Local socket addresses self._peers = [Address.unknown(p) for p in peers] # Peer addresses self._file_directories = file_directories # Directory to monitor for new files (or changes in files) self._files = files # Files to monitor self._file_timestamp_min = file_timestamp_min # Minimum file modification time self._run_time = int(run_time) # Time after which this process stops, -1 is infinite self._bloomfilter_update = float(bloomfilter_update) # Update every # seconds the bloomfilter to peers, -1 for never self._walker = walker # Turn walker on self._api_callback = callback # Subscription to various callbacks self._gateways = {} for g in gateways: a = g.split("=") if len(a) == 2: self._gateways[a[0]] = a[1] self._mtu = mtu self._filepusher = FilePusher(self._register_some_message, self._swift_binpath, directories=self._file_directories, files=self._files, file_size=self._mtu - DISPERSY_MESSAGE_MINIMUM, min_timestamp=self._file_timestamp_min) self._loop_event = Event() # Loop # redirect swift output: sys.stderr = open(self._dest_dir + "/" + str(os.getpid()) + ".err", "w") # redirect standard output: sys.stdout = open(self._dest_dir + "/" + str(os.getpid()) + ".out", "w") self._state = STATE_INITIALIZED
def test_tuple(self): addr = Address.unknown(("1.0.1.0", 1)) # int as port self.assertEqual(addr.port, 1) self.assertEqual(addr.ip, "1.0.1.0") self.assertEqual(addr.family, AF_INET) addr = Address.unknown(("[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]", "1")) # string as port self.assertEqual(addr.port, 1) self.assertEqual(addr.ip, "FEDC:BA98:7654:3210:FEDC:BA98:7654:3210") self.assertEqual(addr.family, AF_INET6) self.assertEqual(addr._flowinfo, 0) self.assertEqual(addr._scopeid, 0)
def test_unknown_port(self): addr = Address.unknown("1") self.assertEqual(addr.port, 1) self.assertEqual(addr.ip, "0.0.0.0") self.assertEqual(addr.family, AF_INET) addr = Address.unknown("12") self.assertEqual(addr.port, 12) self.assertEqual(addr.ip, "0.0.0.0") self.assertEqual(addr.family, AF_INET) addr = Address.unknown(" 12 ") self.assertEqual(addr.port, 12) self.assertEqual(addr.ip, "0.0.0.0") self.assertEqual(addr.family, AF_INET)
def test_unknown_ipv6_scopeid(self): addr = Address.unknown("[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]:1%3") self.assertEqual(addr.port, 1) self.assertEqual(addr.ip, "FEDC:BA98:7654:3210:FEDC:BA98:7654:3210") self.assertEqual(addr.family, AF_INET6) self.assertEqual(addr._flowinfo, 0) self.assertEqual(addr._scopeid, 3)
def test_unknown_ipv6_all(self): addr = Address.unknown("[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]:1/2%3") self.assertEqual(addr.port, 1) self.assertEqual(addr.ip, "FEDC:BA98:7654:3210:FEDC:BA98:7654:3210") self.assertEqual(addr.family, AF_INET6) self.assertEqual(addr._flowinfo, 2) self.assertEqual(addr._scopeid, 3)
def decode_payload(self, placeholder, offset, data): if len(data) < offset + 4: raise DropPacket("Insufficient packet size") data_length, = struct.unpack_from("!L", data, offset) offset += 4 if len(data) < offset + data_length: raise DropPacket("Insufficient packet size") data_payload = data[offset:offset + data_length] id_addrs = data_payload.split(SEPARATOR) id_addresses = zip([i.decode(ENDPOINT_ID_ENCODING) for i in id_addrs[0::3]], # id [Address.unknown(a) for a in id_addrs[1::3]], # lan [Address.unknown(a) for a in id_addrs[2::3]]) # wan offset += data_length return offset, placeholder.meta.payload.implement(id_addresses)
def decode_payload(self, placeholder, offset, data): if len(data) < offset + 4: raise DropPacket("Insufficient packet size") data_length, = struct.unpack_from("!L", data, offset) offset += 4 if len(data) < offset + data_length: raise DropPacket("Insufficient packet size") data_payload = data[offset:offset + data_length] splitted = data_payload.split(SEPARATOR) sender_lan = Address.unknown(splitted[0]) sender_wan = Address.unknown(splitted[1]) address_vote = Address.unknown(splitted[2]) endpoint_id = splitted[3].decode(ENDPOINT_ID_ENCODING) offset += data_length return offset, placeholder.meta.payload.implement(sender_lan, sender_wan, address_vote, endpoint_id)
def read_and_print_out(self, line): # As soon as a TCP connection has been made, will the FastI2I be allowed to start line.strip() listenstr = "swift::Listen addr" if line.find("Creating new TCP listener") != -1: self._last_moreinfo.set() elif line.find(listenstr) != -1: addrstr = line[len(listenstr):] saddr = Address.unknown(addrstr) logger.debug("Found listen address %s", saddr) if saddr != Address(): self.working_sockets.add(saddr) if self._sockaddr_info_callback: self._sockaddr_info_callback(saddr, 0)
def interface_came_up(self, ip, if_name, device, gateway=None, port=0): logger.debug("Interface came up with %s:%d %s %s %s", ip, port, if_name, device, gateway) addr = Address.unknown(ip + ":" + str(port)) # Not very elegant.. But should work for ipv6 as well if addr.resolve_interface(): if addr.interface.name != if_name: logger.warning("Interfaces do not match %s %s", addr.interface.name, if_name) addr.interface.device = device addr.interface.gateway = gateway if addr.interface.address is None: # In case netifaces does not recognize interface such as ppp addr.interface.address = ip self.dispersy_instance._endpoint.interface_came_up(addr) else: addr.set_interface(if_name, ip, None, None, device=device) self.send_message(MESSAGE_KEY_SOCKET_STATE, addr, -2) logger.debug("Bogus interface, cannot locate it")
def __init__(self, stop_event=Event(), name="Network"): ''' @param stop_event: Event that controls the run of this instance, but does not affect this event itself @param name: Name of the instance, must reflect the location of configuration parameters ''' name += ".Dispersy" self.stop_event = stop_event self.db_reader = StatusDbReader() self.cfg = get_config(name) self.status = get_status(name) self.log = get_uav_logger(name) di_kwargs = {} try: di_kwargs = self._get_arguments_from_config() except: logger.exception( "Could not get arguments from config, make do with what you've got" ) finally: self.cfg.close_connection() # Don't need it anymore! API.__init__(self, name, **di_kwargs) self.status["state"] = self.STATES[self.state] self.run_event = Event() self.stop_on_dispersy_stop = True self._stopping = False # dictionary of known interfaces that should be used and the last state and time self.use_interfaces = {} # (timestamp, status, ip) self._listen_args = [ Address.unknown(l) for l in di_kwargs.get("listen", []) ] # Set callbacks self.state_change_callback(self._state_changed) self.swift_state_callback(self._swift_state_changed) # Set signal quit handler signal.signal(signal.SIGQUIT, self.on_quit)
def decode_payload(self, placeholder, offset, data): if len(data) < offset + 4: raise DropPacket("Insufficient packet size") data_length, = struct.unpack_from("!L", data, offset) offset += 4 if len(data) < offset + data_length: raise DropPacket("Insufficient packet size") data_payload = data[offset:offset + data_length] data_pieces = data_payload.split(SEPARATOR) filename = data_pieces[0] directories = data_pieces[1] roothash = data_pieces[2] size = long(data_pieces[3]) timestamp = float(data_pieces[4]) addresses = [Address.unknown(a) for a in data_pieces[5:]] offset += data_length return offset, placeholder.meta.payload.implement(filename, directories, roothash, size, timestamp, addresses)
def __init__(self, stop_event=Event(), name="Network"): ''' @param stop_event: Event that controls the run of this instance, but does not affect this event itself @param name: Name of the instance, must reflect the location of configuration parameters ''' name += ".Dispersy" self.stop_event = stop_event self.db_reader = StatusDbReader() self.cfg = get_config(name) self.status = get_status(name) self.log = get_uav_logger(name) di_kwargs = {} try: di_kwargs = self._get_arguments_from_config() except: logger.exception("Could not get arguments from config, make do with what you've got") finally: self.cfg.close_connection() # Don't need it anymore! API.__init__(self, name, **di_kwargs) self.status["state"] = self.STATES[self.state] self.run_event = Event() self.stop_on_dispersy_stop = True self._stopping = False # dictionary of known interfaces that should be used and the last state and time self.use_interfaces = {} # (timestamp, status, ip) self._listen_args = [Address.unknown(l) for l in di_kwargs.get("listen", [])] # Set callbacks self.state_change_callback(self._state_changed) self.swift_state_callback(self._swift_state_changed) # Set signal quit handler signal.signal(signal.SIGQUIT, self.on_quit)
def add_message(self, message, addresses): if len(message) < self.dispersy_instance._mtu: addrs = [Address.unknown(a) for a in addresses] self.dispersy_instance._register_some_message(APIMessageCarrier(message, addresses=addrs)) else: logger.info("This message of length %d is to big to send with Dispersy", len(message))
def i2ithread_readlinecallback(self, ic, cmd): # logger.debug("CMD IN: %s", cmd) if self.donestate != DONE_STATE_WORKING: return words = cmd.split() assert all(isinstance(word, str) for word in words) if words[0] == "TUNNELRECV": address, session = words[1].split("/") host, port = address.split(":") port = int(port) session = session.decode("HEX") length = int(words[2]) incoming_addr = 0 # None port numbers are ignored if len(words) > 3: incoming_addr = Address.unknown(words[3]) # require LENGTH bytes if len(ic.buffer) < length: return length - len(ic.buffer) data = ic.buffer[:length] ic.buffer = ic.buffer[length:] try: self.roothash2dl["dispersy-endpoint"].i2ithread_data_came_in( session, (host, port), data, incoming_addr) except KeyError: if self._warn_missing_endpoint: self._warn_missing_endpoint = False print >> sys.stderr, "sp: Dispersy endpoint is not available" elif words[0] == "SOCKETINFO": saddr = Address.unknown(words[1]) state = -1 try: state = int(words[2]) except ValueError: pass if saddr != Address(): if state == 0: self.working_sockets.add(saddr) else: self.working_sockets.discard(saddr) if self._sockaddr_info_callback: self._sockaddr_info_callback(saddr, state) else: roothash = binascii.unhexlify(words[1]) if words[0] == "ERROR": error = " ".join(words[2:]) if error == "bad swarm": # bad swarm does not lead to shutdown!!!! logger.debug("This is a bad swarm %s", words[1]) d = self.roothash2dl.get(roothash, None) if d is not None: d.set_bad_swarm() else: error_code = -1 if error == "unknown command": error_code = SWIFT_ERROR_UNKNOWN_COMMAND elif error == "missing parameter": error_code = SWIFT_ERROR_MISSING_PARAMETER elif error == "bad parameter": error_code = SWIFT_ERROR_BAD_PARAMETER else: logger.warning("Unknown Swift Error: %s", error) self.connection_lost(self.get_cmdport(), error_code=error_code) self.splock.acquire() try: d = self.roothash2dl[roothash] except KeyError: logger.debug("Unknown roothash %s", roothash) return finally: self.splock.release() # Hide NSSA interface for SwiftDownloadImpl if words[0] == "INFO": # INFO HASH status dl/total dlstatus = int(words[2]) pargs = words[3].split("/") dynasize = int(pargs[1]) if dynasize == 0: progress = 0.0 else: progress = float(pargs[0]) / float(pargs[1]) dlspeed = float(words[4]) ulspeed = float(words[5]) numleech = int(words[6]) numseeds = int(words[7]) contentdl = 0 # bytes contentul = 0 # bytes if len(words) > 8: contentdl = int(words[8]) contentul = int(words[9]) d.i2ithread_info_callback(dlstatus, progress, dynasize, dlspeed, ulspeed, numleech, numseeds, contentdl, contentul) elif words[0] == "PLAY": # print >>sys.stderr,"sp: i2ithread_readlinecallback: Got PLAY",cmd httpurl = words[2] d.i2ithread_vod_event_callback(VODEVENT_START, httpurl) elif words[0] == "MOREINFO": jsondata = cmd[len("MOREINFO ") + 40 + 1:] midict = json.loads(jsondata) d.i2ithread_moreinfo_callback(midict) elif words[0] == "ERROR": d.i2ithread_info_callback(DLSTATUS_STOPPED_ON_ERROR, 0.0, 0, 0.0, 0.0, 0, 0, 0, 0) elif words[0] == "CHANNELCLOSED": saddr = Address.unknown(words[2]) paddr = Address.unknown(words[3]) if self._channel_closed_callback is not None: self._channel_closed_callback(roothash, saddr, paddr) if d._channel_closed_callback is not None: d._channel_closed_callback(roothash, saddr, paddr)
def __init__(self, dest_dir, swift_binpath, dispersy_work_dir=u".", sqlite_database=":memory:", swift_work_dir=None, swift_zerostatedir=None, listen=[], peers=[], file_directories=[], files=[], file_timestamp_min=None, run_time=-1, bloomfilter_update=-1, walker=False, gateways={}, mtu=MAX_MTU, callback=None): """ @param dest_dir: Directory in which downloads will be placed as well as logs @param swift_binpath: Path to the swift executable @param dispersy_work_dir: Working directory for Dispersy @param sqlite_database: Location of the sqlite_database, :memory: is in memory @param swift_work_dir: Working directory for Swift @param swift_zerostatedir: Zero state directory for Swift @param listen: Addresses of local sockets to bind to @param peers: Addresses of peers to communicate with @param files_directory: Directory to monitor for files that should be disseminated @param files: Files that should be disseminated @param file_timestamp_min: Minimum file modification time @param run_time: Total run time in seconds @param bloomfilter_update: Period after which another introduction request is sent in seconds @param walker: If enable the Dispersy walker will be enabled @param gateways: Ip addresses of the gateways for specific interfaces (eth0=192.168.0.1) @param mtu: Maximum transmission unit directs the maximum size of messages @param callback: Callback function that will called on certain events """ self._dest_dir = dest_dir self._swift_binpath = swift_binpath self._dispersy_work_dir = unicode(dispersy_work_dir) self._sqlite_database = unicode( sqlite_database) # :memory: is in memory self._swift_work_dir = swift_work_dir if swift_work_dir is not None else dest_dir self._swift_zerostatedir = swift_zerostatedir self._listen = [Address.unknown(l) for l in listen] # Local socket addresses self._peers = [Address.unknown(p) for p in peers] # Peer addresses self._file_directories = file_directories # Directory to monitor for new files (or changes in files) self._files = files # Files to monitor self._file_timestamp_min = file_timestamp_min # Minimum file modification time self._run_time = int( run_time) # Time after which this process stops, -1 is infinite self._bloomfilter_update = float( bloomfilter_update ) # Update every # seconds the bloomfilter to peers, -1 for never self._walker = walker # Turn walker on self._api_callback = callback # Subscription to various callbacks self._gateways = {} for g in gateways: a = g.split("=") if len(a) == 2: self._gateways[a[0]] = a[1] self._mtu = mtu self._filepusher = FilePusher(self._register_some_message, self._swift_binpath, directories=self._file_directories, files=self._files, file_size=self._mtu - DISPERSY_MESSAGE_MINIMUM, min_timestamp=self._file_timestamp_min) self._loop_event = Event() # Loop # redirect swift output: sys.stderr = open(self._dest_dir + "/" + str(os.getpid()) + ".err", "w") # redirect standard output: sys.stdout = open(self._dest_dir + "/" + str(os.getpid()) + ".out", "w") self._state = STATE_INITIALIZED
def test_no_port_ipv6(self): addr = Address.unknown("[::0]") self.assertEqual(addr.port, 0) self.assertEqual(addr.ip, "::0") self.assertEqual(addr.family, AF_INET6)
def test_unknown_ipv4(self): addr = Address.unknown("1.0.1.0:1") self.assertEqual(addr.port, 1) self.assertEqual(addr.ip, "1.0.1.0") self.assertEqual(addr.family, AF_INET)
def test_resolve_interface(self): addr = Address.unknown("127.0.0.1:1") addr.resolve_interface() self.assertEqual(addr.interface.name, "lo")
def i2ithread_readlinecallback(self, ic, cmd): # logger.debug("CMD IN: %s", cmd) if self.donestate != DONE_STATE_WORKING: return words = cmd.split() assert all(isinstance(word, str) for word in words) if words[0] == "TUNNELRECV": address, session = words[1].split("/") host, port = address.split(":") port = int(port) session = session.decode("HEX") length = int(words[2]) incoming_addr = 0 # None port numbers are ignored if len(words) > 3: incoming_addr = Address.unknown(words[3]) # require LENGTH bytes if len(ic.buffer) < length: return length - len(ic.buffer) data = ic.buffer[:length] ic.buffer = ic.buffer[length:] try: self.roothash2dl["dispersy-endpoint"].i2ithread_data_came_in(session, (host, port), data, incoming_addr) except KeyError: if self._warn_missing_endpoint: self._warn_missing_endpoint = False print >> sys.stderr, "sp: Dispersy endpoint is not available" elif words[0] == "SOCKETINFO": saddr = Address.unknown(words[1]) state = -1 try: state = int(words[2]) except ValueError: pass if saddr != Address(): if state == 0: self.working_sockets.add(saddr) else: self.working_sockets.discard(saddr) if self._sockaddr_info_callback: self._sockaddr_info_callback(saddr, state) else: roothash = binascii.unhexlify(words[1]) if words[0] == "ERROR": error = " ".join(words[2:]) if error == "bad swarm": # bad swarm does not lead to shutdown!!!! logger.debug("This is a bad swarm %s", words[1]) d = self.roothash2dl.get(roothash, None) if d is not None: d.set_bad_swarm() else: error_code = -1 if error == "unknown command": error_code = SWIFT_ERROR_UNKNOWN_COMMAND elif error == "missing parameter": error_code = SWIFT_ERROR_MISSING_PARAMETER elif error == "bad parameter": error_code = SWIFT_ERROR_BAD_PARAMETER else: logger.warning("Unknown Swift Error: %s", error) self.connection_lost(self.get_cmdport(), error_code=error_code) self.splock.acquire() try: d = self.roothash2dl[roothash] except KeyError: logger.debug("Unknown roothash %s", roothash) return finally: self.splock.release() # Hide NSSA interface for SwiftDownloadImpl if words[0] == "INFO": # INFO HASH status dl/total dlstatus = int(words[2]) pargs = words[3].split("/") dynasize = int(pargs[1]) if dynasize == 0: progress = 0.0 else: progress = float(pargs[0]) / float(pargs[1]) dlspeed = float(words[4]) ulspeed = float(words[5]) numleech = int(words[6]) numseeds = int(words[7]) contentdl = 0 # bytes contentul = 0 # bytes if len(words) > 8: contentdl = int(words[8]) contentul = int(words[9]) d.i2ithread_info_callback(dlstatus, progress, dynasize, dlspeed, ulspeed, numleech, numseeds, contentdl, contentul) elif words[0] == "PLAY": # print >>sys.stderr,"sp: i2ithread_readlinecallback: Got PLAY",cmd httpurl = words[2] d.i2ithread_vod_event_callback(VODEVENT_START, httpurl) elif words[0] == "MOREINFO": jsondata = cmd[len("MOREINFO ") + 40 + 1:] midict = json.loads(jsondata) d.i2ithread_moreinfo_callback(midict) elif words[0] == "ERROR": d.i2ithread_info_callback(DLSTATUS_STOPPED_ON_ERROR, 0.0, 0, 0.0, 0.0, 0, 0, 0, 0) elif words[0] == "CHANNELCLOSED": saddr = Address.unknown(words[2]) paddr = Address.unknown(words[3]) if self._channel_closed_callback is not None: self._channel_closed_callback(roothash, saddr, paddr) if d._channel_closed_callback is not None: d._channel_closed_callback(roothash, saddr, paddr)