def setUp(self):
self._callback = Callback("TestCallback")
self._ports = [12544]
self._addrs = [Address(port=p, ip="127.0.0.1") for p in self._ports]
swift_process = MySwiftProcess(SWIFT_BINPATH, ".", None, self._addrs,
None, None, None)
self._endpoint = MultiEndpoint(swift_process)
self._dispersy = MyDispersy(self._callback, self._endpoint,
DISPERSY_WORKDIR, u":memory:")
self._dispersy.start()
self._community = self._callback.call(self.create_mycommunity,
(self._dispersy, ))
self._swiftcomm = self._community.swift_community
self._directories = "testcase_swift_seed_and_down/"
self._dest_dir = DIRECTORY
self._community.dest_dir = self._dest_dir
self._filename = FILES[0]
self._roothash = get_hash(self._filename, SWIFT_BINPATH)
self._callback2 = Callback("TestCallback2")
self._ports2 = [34254]
self._addrs2 = [Address(port=p, ip="127.0.0.1") for p in self._ports2]
swift_process2 = MySwiftProcess(SWIFT_BINPATH, ".", None, self._addrs2,
None, None, None)
self._endpoint2 = MultiEndpoint(swift_process2)
self._dispersy2 = MyDispersy(self._callback2, self._endpoint2, u".",
u":memory:")
self._dispersy2.start()
self._community2 = self._callback2.call(self.create_mycommunity,
(self._dispersy2, ))
self._community2.dest_dir = self._dest_dir
self._swiftcomm2 = self._community2.swift_community
self._filename2 = FILES[1]
self._roothash2 = get_hash(self._filename2, SWIFT_BINPATH)
def test_puncture_message_conversion(self):
for c in self._conversions:
if isinstance(c, PunctureConversion):
meta = self._di._community.get_meta_message(
PUNCTURE_MESSAGE_NAME)
sender_lan = Address.ipv4("0.0.0.1:1232")
sender_wan = Address.ipv4("23.12.32.123:1232")
sender_id = os.urandom(16)
address_vote = Address.ipv6("[::0]:12145")
endpoint_id = os.urandom(16)
message = meta.impl(
authentication=(self._di._community.my_member, ),
distribution=(self._di._community.claim_global_time(), ),
destination=(Candidate(address_vote.addr(), True), ),
payload=(sender_lan, sender_wan, sender_id, address_vote,
endpoint_id))
encoded = c.encode_payload(message)
placeholder = c.Placeholder(None, meta, 0, encoded, False,
True)
_, x = c.decode_payload(placeholder, 0,
str(encoded[0]) + encoded[1])
self.assertEqual(x.sender_lan, sender_lan)
self.assertEqual(x.sender_wan, sender_wan)
self.assertEqual(x.sender_id, sender_id)
self.assertEqual(x.address_vote, address_vote)
self.assertEqual(x.endpoint_id, endpoint_id)
def swift_info_callback(self, info):
regular = info.get("regular")
base = "swift."
if regular is not None:
for k, v in regular.iteritems():
self.status[base + k] = v
download = info.get("direct")
if download is None:
return
base = "swift.downloads." + download["roothash"] + "."
for k, v in download.iteritems():
if k != "moreinfo":
self.status[base + k] = v
basechannel = "swift.sockets." # Maybe set this to dispersy.endpoint
(channels, total) = download["moreinfo"]
for c in channels:
if_name = self._get_device_by_ip(c["sock_ip"])
peer_name = c["ip"].replace(".","_") + ":" + str(c["port"])
if if_name is None:
addr = Address(ip=c["sock_ip"], port=c["sock_port"])
addr.resolve_interface()
if addr.interface is not None:
if_name = addr.interface.device
self.use_interfaces[if_name] = (time.time(), u"up", c["sock_ip"]) # Insert this interface into the known interfaces
else:
if_name = "unknown"
for k, v in c.iteritems():
self.status[basechannel + if_name + "." + peer_name + "." + download["roothash"] + "." + k] = v
for k, v in total.iteritems():
self.status[base + k] = v
def got_moreinfo(self):
self.set_started()
self._last_moreinfo = datetime.utcnow()
for c in self.downloadimpl.midict.get("channels", []):
self._active_channels.add(
(Address.tuple((c["socket_ip"], c["socket_port"])),
Address.tuple((c["ip"], c["port"]))
)) # Tuple can handle both ipv4 and ipv6 (port can be string)
def __init__(self, dest_dir, swift_binpath, dispersy_work_dir=u".", sqlite_database=":memory:", swift_work_dir=None,
swift_zerostatedir=None, listen=[], peers=[], file_directories=[], files=[], file_timestamp_min=None,
run_time=-1, bloomfilter_update=-1, walker=False, gateways={}, mtu=MAX_MTU, callback=None):
"""
@param dest_dir: Directory in which downloads will be placed as well as logs
@param swift_binpath: Path to the swift executable
@param dispersy_work_dir: Working directory for Dispersy
@param sqlite_database: Location of the sqlite_database, :memory: is in memory
@param swift_work_dir: Working directory for Swift
@param swift_zerostatedir: Zero state directory for Swift
@param listen: Addresses of local sockets to bind to
@param peers: Addresses of peers to communicate with
@param files_directory: Directory to monitor for files that should be disseminated
@param files: Files that should be disseminated
@param file_timestamp_min: Minimum file modification time
@param run_time: Total run time in seconds
@param bloomfilter_update: Period after which another introduction request is sent in seconds
@param walker: If enable the Dispersy walker will be enabled
@param gateways: Ip addresses of the gateways for specific interfaces (eth0=192.168.0.1)
@param mtu: Maximum transmission unit directs the maximum size of messages
@param callback: Callback function that will called on certain events
"""
self._dest_dir = dest_dir
self._swift_binpath = swift_binpath
self._dispersy_work_dir = unicode(dispersy_work_dir)
self._sqlite_database = unicode(sqlite_database) # :memory: is in memory
self._swift_work_dir = swift_work_dir if swift_work_dir is not None else dest_dir
self._swift_zerostatedir = swift_zerostatedir
self._listen = [Address.unknown(l) for l in listen] # Local socket addresses
self._peers = [Address.unknown(p) for p in peers] # Peer addresses
self._file_directories = file_directories # Directory to monitor for new files (or changes in files)
self._files = files # Files to monitor
self._file_timestamp_min = file_timestamp_min # Minimum file modification time
self._run_time = int(run_time) # Time after which this process stops, -1 is infinite
self._bloomfilter_update = float(bloomfilter_update) # Update every # seconds the bloomfilter to peers, -1 for never
self._walker = walker # Turn walker on
self._api_callback = callback # Subscription to various callbacks
self._gateways = {}
for g in gateways:
a = g.split("=")
if len(a) == 2:
self._gateways[a[0]] = a[1]
self._mtu = mtu
self._filepusher = FilePusher(self._register_some_message, self._swift_binpath,
directories=self._file_directories, files=self._files,
file_size=self._mtu - DISPERSY_MESSAGE_MINIMUM,
min_timestamp=self._file_timestamp_min)
self._loop_event = Event() # Loop
# redirect swift output:
sys.stderr = open(self._dest_dir + "/" + str(os.getpid()) + ".err", "w")
# redirect standard output:
sys.stdout = open(self._dest_dir + "/" + str(os.getpid()) + ".out", "w")
self._state = STATE_INITIALIZED
def test_tuple(self):
addr = Address.unknown(("1.0.1.0", 1)) # int as port
self.assertEqual(addr.port, 1)
self.assertEqual(addr.ip, "1.0.1.0")
self.assertEqual(addr.family, AF_INET)
addr = Address.unknown(("[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]", "1")) # string as port
self.assertEqual(addr.port, 1)
self.assertEqual(addr.ip, "FEDC:BA98:7654:3210:FEDC:BA98:7654:3210")
self.assertEqual(addr.family, AF_INET6)
self.assertEqual(addr._flowinfo, 0)
self.assertEqual(addr._scopeid, 0)
def test_private_address(self):
addr = Address(ip="127.2.3.1")
self.assertTrue(addr.is_private_address())
addr = Address(ip="10.12.124.231")
self.assertTrue(addr.is_private_address())
addr = Address(ip="192.168.124.231")
self.assertTrue(addr.is_private_address())
addr = Address(ip="172.28.124.231")
self.assertTrue(addr.is_private_address())
addr = Address(ip="192.180.124.231")
self.assertFalse(addr.is_private_address())
def test_one_contact_peer(self):
# DispersyCandidate is already there
community = FakeCommunity()
mid = "Smoothy"
lan_addresses = [Address(ip="127.3.2.4", port=123)]
wan_addresses = [Address(ip="42.23.2.1", port=332)]
ids = [os.urandom(16)]
comm, first_contact = self.common.update_dispersy_contacts(wan_addresses[0].addr(), ["asdf"], recv=True)
dc = self.common.peer_endpoints_received(community, mid, lan_addresses, wan_addresses, ids)
self.assertEqual(first_contact, dc)
self.assertEqual(dc.peer, Peer(lan_addresses, wan_addresses, ids, mid))
self.assertEqual(len(self.common.dispersy_contacts), 1)
def test_tuple(self):
addr = Address.unknown(("1.0.1.0", 1)) # int as port
self.assertEqual(addr.port, 1)
self.assertEqual(addr.ip, "1.0.1.0")
self.assertEqual(addr.family, AF_INET)
addr = Address.unknown(("[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]",
"1")) # string as port
self.assertEqual(addr.port, 1)
self.assertEqual(addr.ip, "FEDC:BA98:7654:3210:FEDC:BA98:7654:3210")
self.assertEqual(addr.family, AF_INET6)
self.assertEqual(addr._flowinfo, 0)
self.assertEqual(addr._scopeid, 0)
def test_unknown_port(self):
addr = Address.unknown("1")
self.assertEqual(addr.port, 1)
self.assertEqual(addr.ip, "0.0.0.0")
self.assertEqual(addr.family, AF_INET)
addr = Address.unknown("12")
self.assertEqual(addr.port, 12)
self.assertEqual(addr.ip, "0.0.0.0")
self.assertEqual(addr.family, AF_INET)
addr = Address.unknown(" 12 ")
self.assertEqual(addr.port, 12)
self.assertEqual(addr.ip, "0.0.0.0")
self.assertEqual(addr.family, AF_INET)
def test_unknown_port(self):
addr = Address.unknown("1")
self.assertEqual(addr.port, 1)
self.assertEqual(addr.ip, "0.0.0.0")
self.assertEqual(addr.family, AF_INET)
addr = Address.unknown("12")
self.assertEqual(addr.port, 12)
self.assertEqual(addr.ip, "0.0.0.0")
self.assertEqual(addr.family, AF_INET)
addr = Address.unknown(" 12 ")
self.assertEqual(addr.port, 12)
self.assertEqual(addr.ip, "0.0.0.0")
self.assertEqual(addr.family, AF_INET)
def test_occupied_address(self):
interfaces = get_interface_addresses()
self.addresses = []
for i in interfaces: # AF_INET only
addr = Address(ip=i.address, port=random.randint(*ALLOWED_PORTS))
self.addresses.append(addr)
s = socket.socket(addr.family, socket.SOCK_DGRAM)
s.bind(addr.addr())
logger.debug("Created addresses %s", " ".join(map(str, self.addresses)))
addresses = verify_addresses_are_free(self.addresses[:])
for addr in addresses:
self.assertIn(addr.ip, [addr.ip for addr in self.addresses])
def test_two_contacts_one_peer(self):
# here are two DispersyCandidates that represent the same peer
community = FakeCommunity()
mid = "Smoothy"
lan_addresses = [Address(ip="127.3.2.4", port=123)]
wan_addresses = [Address(ip="42.23.2.1", port=332)]
ids = [os.urandom(16)]
comm, first_contact = self.common.update_dispersy_contacts(lan_addresses[0].addr(), ["asdf"], recv=True)
comm, second_contact = self.common.update_dispersy_contacts(wan_addresses[0].addr(), ["asdf"], recv=True)
dc = self.common.peer_endpoints_received(community, mid, lan_addresses, wan_addresses, ids)
self.assertIn(dc, [first_contact, second_contact])
self.assertEqual(dc.peer, Peer(lan_addresses, wan_addresses, ids, mid))
self.assertEqual(len(self.common.dispersy_contacts), 1)
def setUp(self):
self.addressVersion = 6
self.addressLength = 16
self.addressStr = '2001:db8::1428:57ab'
self.addressInt = 42540766411282592856903984951992014763L
self.addressLinkLocalStr = "fe80::1"
self.addressLoopbackStr = "::1"
self.addressMulticastStr = "ff00::1"
self.addressPrivateStr = "fc00::1"
self.addressReservedStr = "::1"
self.addressUnspecifiedStr = "::"
self.address = Address(self.addressStr)
self.ipv4AddressStr = "100.110.120.130"
self.ipv4AddressInt = 1684961410
def read_and_print_out(self, line):
# As soon as a TCP connection has been made, will the FastI2I be allowed to start
line.strip()
listenstr = "swift::Listen addr"
if line.find("Creating new TCP listener") != -1:
self._last_moreinfo.set()
elif line.find(listenstr) != -1:
addrstr = line[len(listenstr):]
saddr = Address.unknown(addrstr)
logger.debug("Found listen address %s", saddr)
if saddr != Address():
self.working_sockets.add(saddr)
if self._sockaddr_info_callback:
self._sockaddr_info_callback(saddr, 0)
def test_occupied_address(self):
interfaces = get_interface_addresses()
self.addresses = []
for i in interfaces: # AF_INET only
addr = Address(ip=i.address, port=random.randint(*ALLOWED_PORTS))
self.addresses.append(addr)
s = socket.socket(addr.family, socket.SOCK_DGRAM)
s.bind(addr.addr())
logger.debug("Created addresses %s", " ".join(map(str,
self.addresses)))
addresses = verify_addresses_are_free(self.addresses[:])
for addr in addresses:
self.assertIn(addr.ip, [addr.ip for addr in self.addresses])
def setUp(self):
self.addressVersion = 4
self.addressLength = 4
self.addressStr = '100.110.120.130'
self.addressInt = 1684961410
self.addressLinkLocalStr = "169.254.0.1"
self.addressLoopbackStr = "127.0.0.1"
self.addressMulticastStr = "224.0.0.1"
self.addressPrivateStr = "192.168.0.1"
self.addressReservedStr = "240.0.0.1"
self.addressUnspecifiedStr = "0.0.0.0"
self.address = Address(self.addressStr)
self.ipv6AddressStr = "fe80:0000:0000:0000:021b:77ff:fbd6:7860"
self.ipv6AddressInt = 338288524927261089654170743795120240736L
def test_uav_message_conversion(self):
for c in self._conversions:
if isinstance(c, APIMessageConversion):
meta = self._di._community.get_meta_message(API_MESSAGE_NAME)
data = "asjfdioewf"
addresses = [Address.ipv4("0.0.0.1:1232"), Address.ipv6("[::0]:12145"), Address(port=32532)]
message = meta.impl(authentication=(self._di._community.my_member,),
distribution=(self._di._community.claim_global_time(),),
destination=(Candidate(addresses[0].addr(), True),),
payload=(data,))
encoded = c.encode_payload(message)
placeholder = c.Placeholder(None, meta, 0, encoded, False, True)
_, x = c.decode_payload(placeholder, 0, str(encoded[0])+encoded[1])
self.assertEqual(x.message, data)
def test_addresses_message_conversion(self):
for c in self._conversions:
if isinstance(c, AddressesConversion):
meta = self._di._community.get_meta_message(ADDRESSES_MESSAGE_NAME)
ids = [os.urandom(16) for _ in range(3)]
addresses = [Address.ipv4("0.0.0.1:1232"), Address.ipv6("[::0]:12145"), Address(port=32532)]
id_addresses = zip(ids, addresses, addresses)
message = meta.impl(authentication=(self._di._community.my_member,),
distribution=(self._di._community.claim_global_time(),),
destination=(Candidate(id_addresses[0][1].addr(), True),),
payload=(id_addresses,))
encoded = c.encode_payload(message)
placeholder = c.Placeholder(None, meta, 0, encoded, False, True)
_, x = c.decode_payload(placeholder, 0, str(encoded[0])+encoded[1])
self.assertEqual(x.id_addresses, id_addresses)
def test_unknown_ipv6_scopeid(self):
addr = Address.unknown("[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]:1%3")
self.assertEqual(addr.port, 1)
self.assertEqual(addr.ip, "FEDC:BA98:7654:3210:FEDC:BA98:7654:3210")
self.assertEqual(addr.family, AF_INET6)
self.assertEqual(addr._flowinfo, 0)
self.assertEqual(addr._scopeid, 3)
def decode_payload(self, placeholder, offset, data):
if len(data) < offset + 4:
raise DropPacket("Insufficient packet size")
data_length, = struct.unpack_from("!L", data, offset)
offset += 4
if len(data) < offset + data_length:
raise DropPacket("Insufficient packet size")
data_payload = data[offset:offset + data_length]
id_addrs = data_payload.split(SEPARATOR)
id_addresses = zip([i.decode(ENDPOINT_ID_ENCODING) for i in id_addrs[0::3]], # id
[Address.unknown(a) for a in id_addrs[1::3]], # lan
[Address.unknown(a) for a in id_addrs[2::3]]) # wan
offset += data_length
return offset, placeholder.meta.payload.implement(id_addresses)
def test_unknown_ipv6_all(self):
addr = Address.unknown("[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]:1/2%3")
self.assertEqual(addr.port, 1)
self.assertEqual(addr.ip, "FEDC:BA98:7654:3210:FEDC:BA98:7654:3210")
self.assertEqual(addr.family, AF_INET6)
self.assertEqual(addr._flowinfo, 2)
self.assertEqual(addr._scopeid, 3)
def get_address_list(self, text):
address_list = []
regex = [
'\d{1,2}\/\d{1,2}\/\d{2}, \d+:\d+ - [a-zA-Z-0-9 ]+:',
'\d{1,2}\/\d{1,2}\/\d{2}, \d+:\d+ [apAP][mM] - [a-zA-Z-0-9 ]+:',
'\[\d{1,2}:\d{1,2}, \d{1,2}\/\d{1,2}\/\d{4}\] [a-zA-Z-0-9 ]+:',
'\d{1,2}\/\d{1,2}\/\d{1,4}, \d{1,2}:\d{1,2} [aAPp][Mm] - \+[0-9 ]+:',
'\d{1,2}\/\d{1,2}\/\d{2}, \d+:\d+ [apAP][mM] - [a-zA-Z-0-9 ]+:[ 0-9-🪀a-zA-Z:\/\.?]+\s\[',
'\d{1,2}\/\d{1,2}\/\d{2}, \d+:\d+ [apAP][mM] - [a-zA-Z-0-9 ]+:[ 0-9-🪀a-zA-Z:\/\.?]+wa\.me\/\d+[ ]{1,4}[^[]',
'\d{1,2}\/\d{1,2}\/\d{2}, \d+:\d+ [apAP][mM] - [a-zA-Z-0-9 ]+:[ 0-9-🪀a-zA-Z:\/\.?]+=Hi\s+',
'\d{1,2}\/\d{1,2}\/\d{2}, \d+:\d+ - [mM]essages and calls',
'\d{1,2}\/\d{1,2}\/\d{2}, \d+:\d+ - [yY]ou'
]
regex_split_main = re.compile("|".join(regex))
string_address_list = re.split(regex_split_main, text)
if len(string_address_list) <= 1:
string_address_list = re.split("\n", text)
for address_text in string_address_list:
address_text = address_text.replace("[\s]+", " ")
self.utility.white_space_cleaner(address_text)
if len(address_text.strip()
) > 0 and not self.utility.whatsapp_text(address_text):
address_obj = Address(address_text.lower(), None, None, None,
None, None)
address_list.append(address_obj)
return address_list
def test_update_contacts(self):
sock_addr = ("1.1.1.1", 3)
packets = ["asdf", "sf"]
comm, dc = self.common.update_dispersy_contacts(sock_addr, packets, recv=False)
first_contact = iter(self.common.dispersy_contacts).next()
self.assertEqual(dc, None)
self.assertEqual(len(self.common.dispersy_contacts), 1)
self.assertEqual(first_contact.address.ip, sock_addr[0])
self.assertEqual(self.common.get_contact(Address.tuple(sock_addr)), first_contact)
more_packets = ["sfjdoiwenfo", "sfjisa"]
comm, dc = self.common.update_dispersy_contacts(sock_addr, more_packets, recv=False)
self.assertEqual(dc, None)
self.assertEqual(len(self.common.dispersy_contacts), 1)
self.assertEqual(first_contact.num_sent(), len(packets + more_packets))
self.assertEqual(first_contact.total_sent(), sum(len(p) for p in packets + more_packets))
comm, dc = self.common.update_dispersy_contacts(sock_addr, more_packets, recv=True)
self.assertEqual(dc, first_contact, str(dc) + " != " + str(first_contact))
self.assertEqual(len(self.common.dispersy_contacts), 1)
self.assertEqual(first_contact.num_sent(), len(packets + more_packets))
self.assertEqual(first_contact.total_sent(), sum(len(p) for p in packets + more_packets))
self.assertEqual(first_contact.num_rcvd(), len(more_packets))
self.assertEqual(first_contact.total_rcvd(), sum(len(p) for p in more_packets))
new_sock_addr = ("2.2.1.2", 5)
comm, dc = self.common.update_dispersy_contacts(new_sock_addr, packets, recv=True)
for c in self.common.dispersy_contacts:
if c != first_contact:
second_contact = c
self.assertEqual(dc, second_contact)
self.assertEqual(len(self.common.dispersy_contacts), 2)
self.assertEqual(first_contact.num_sent(), len(packets + more_packets))
self.assertEqual(first_contact.total_sent(), sum(len(p) for p in packets + more_packets))
self.assertEqual(first_contact.num_rcvd(), len(more_packets))
self.assertEqual(first_contact.total_rcvd(), sum(len(p) for p in more_packets))
self.assertEqual(second_contact.num_rcvd(), len(packets))
self.assertEqual(second_contact.total_rcvd(), sum(len(p) for p in packets))
def test_file_hash_message_conversion(self):
for c in self._conversions:
if isinstance(c, FileHashConversion):
meta = self._di._community.get_meta_message(
FILE_HASH_MESSAGE_NAME)
filename = "asdf.asdf"
dir_ = ""
roothash = "asdfj23j09f09sjfewef"
size = 103425
timestamp = 3232342.2342
address = Address.ipv4("0.0.0.0:1")
message = meta.impl(
authentication=(self._di._community.my_member, ),
distribution=(self._di._community.claim_global_time(),
self._di._community._file_hash_distribution.
claim_sequence_number()),
payload=(filename, dir_, roothash, size, timestamp,
[address]))
encoded = c.encode_payload(message)
placeholder = c.Placeholder(None, meta, 0, encoded, False,
True)
_, x = c.decode_payload(placeholder, 0,
str(encoded[0]) + encoded[1])
self.assertEqual(x.filename, filename)
self.assertEqual(x.directories, dir_)
self.assertEqual(x.roothash, roothash)
self.assertEqual(x.size, size)
self.assertEqual(x.timestamp, timestamp)
self.assertEqual(x.addresses[0], address)
def allowed_addresses(self):
if isinstance(self._destination, CandidateDestination.Implementation):
return [
Address.tuple(c.sock_addr)
for c in self._destination._candidates
]
return None # We're not returning a list if this is not a candidatedestination.. Deal with it
def __init__(self, lang_code):
# Initialize the faker generator
self.fake = faker.Faker(lang_code)
# The customer's name
self.first_name = self.fake.first_name()
self.last_name = self.fake.last_name()
# The customer's mail address
customers_emails = [customer.email for customer in Customer.customers]
while True:
self.email = Email(lang_code, self.first_name,
self.last_name).email
if self.email not in customers_emails:
break
# The customer's phone number
self.phone_number = self.fake.phone_number()
# The customer's postal address
self.address = Address(lang_code,
optional_address=True,
additional_details=True)
# The customer's password
self.password = Password()
Customer.customers.append(self)
def test_to_dict_with_no_address(self):
# arrange
address = Address()
expected_address_dict = {
'street_address_1': None,
'street_address_2': None,
'city': None,
'state': None,
'zip': None,
}
# act
actual_address_dict = address.to_dict()
# assert
self.assertEqual(actual_address_dict, expected_address_dict,
'empty dicts not the same')
def shallow_copy(cls, contact):
assert isinstance(contact, DispersyContact)
# Copies are not really necessary probably
dc = DispersyContact(
Address.copy(contact.address),
peer=Peer.copy(contact.peer) if contact.peer is not None else None)
dc.community_ids = list(contact.community_ids)
return dc
def make_address(self):
pass
city = self.get_city
street = self.get_street
buildnum = self.get_buildnum
address = Address(city, street, buildnum)
return address
def decode_payload(self, placeholder, offset, data):
if len(data) < offset + 4:
raise DropPacket("Insufficient packet size")
data_length, = struct.unpack_from("!L", data, offset)
offset += 4
if len(data) < offset + data_length:
raise DropPacket("Insufficient packet size")
data_payload = data[offset:offset + data_length]
splitted = data_payload.split(SEPARATOR)
sender_lan = Address.unknown(splitted[0])
sender_wan = Address.unknown(splitted[1])
address_vote = Address.unknown(splitted[2])
endpoint_id = splitted[3].decode(ENDPOINT_ID_ENCODING)
offset += data_length
return offset, placeholder.meta.payload.implement(sender_lan, sender_wan, address_vote, endpoint_id)
def send_tunnel(self, session, address, data, addr=Address()):
if addr.port == 0:
SwiftProcess.send_tunnel(self, session, address, data)
else:
self.write("TUNNELSEND %s:%d/%s %d %s\r\n" %
(address[0], address[1], session.encode("HEX"),
len(data), str(addr)))
self.write(data)
def test_unknown_peer(self):
# Case where these addresses do not match any incoming addresses
# This is possible if the peer does not know its proper wan address
community = FakeCommunity()
mid = "Smoothy"
lan_addresses = [Address(ip="127.3.2.4", port=123)]
wan_addresses = [Address(ip="42.23.2.1", port=332)]
ids = [os.urandom(16)]
dc = self.common.peer_endpoints_received(community, mid, lan_addresses, wan_addresses, ids)
self.assertEqual(len(self.common.dispersy_contacts), 1)
self.assertEqual(dc.peer, Peer(lan_addresses, wan_addresses, ids, mid))
self.assertEqual(dc.member_id, mid)
self.assertEqual(len(dc.community_ids), 1)
self.assertEqual(iter(dc.community_ids).next(), community.cid)
self.assertEqual(self.common.get_contact(Address(), mid=mid), dc)
dc2 = self.common.peer_endpoints_received(community, mid, lan_addresses, wan_addresses, ids)
self.assertEqual(dc2, dc)
self.assertEqual(dc2.peer, dc.peer)
def test_to_dict_method(self):
# arrange
ordered_address_dict = collections.OrderedDict(self.address_dict)
expected_address_dict = {
'street_address_1': self.address_dict['address_1'],
'street_address_2': self.address_dict['address_2'],
'city': self.address_dict['city'],
'state': self.address_dict['state'],
'zip': self.address_dict['zip'],
}
# act
address = Address(ordered_address_dict)
actual_address_dict = address.to_dict()
# assert
self.assertEqual(actual_address_dict, expected_address_dict,
'address dicts are not the same')
def test_puncture_response_message_conversion(self):
for c in self._conversions:
if isinstance(c, PunctureResponseConversion):
meta = self._di._community.get_meta_message(PUNCTURE_RESPONSE_MESSAGE_NAME)
sender_lan = Address.ipv4("0.0.0.1:1232")
sender_wan = Address.ipv4("23.12.32.123:1232")
address_vote = Address.ipv6("[::0]:12145")
endpoint_id = os.urandom(16)
message = meta.impl(authentication=(self._di._community.my_member,),
distribution=(self._di._community.claim_global_time(),),
destination=(Candidate(address_vote.addr(), True),),
payload=(sender_lan, sender_wan, address_vote, endpoint_id))
encoded = c.encode_payload(message)
placeholder = c.Placeholder(None, meta, 0, encoded, False, True)
_, x = c.decode_payload(placeholder, 0, str(encoded[0])+encoded[1])
self.assertEqual(x.sender_lan, sender_lan)
self.assertEqual(x.sender_wan, sender_wan)
self.assertEqual(x.address_vote, address_vote)
self.assertEqual(x.endpoint_id, endpoint_id)
def test_addresses_request_message_conversion(self):
for c in self._conversions:
if isinstance(c, AddressesRequestConversion):
meta = self._di._community.get_meta_message(ADDRESSES_REQUEST_MESSAGE_NAME)
sender_lan = Address.ipv4("0.0.0.1:1232")
sender_wan = Address.ipv4("23.12.32.123:1232")
endpoint_id = os.urandom(16)
wan_address = Address.ipv4("12.23.21.21:1234")
message = meta.impl(authentication=(self._di._community.my_member,),
distribution=(self._di._community.claim_global_time(),),
destination=(Candidate(wan_address.addr(), True),),
payload=(sender_lan, sender_wan, endpoint_id, wan_address))
encoded = c.encode_payload(message)
placeholder = c.Placeholder(None, meta, 0, encoded, False, True)
_, x = c.decode_payload(placeholder, 0, str(encoded[0])+encoded[1])
self.assertEqual(x.sender_lan, sender_lan)
self.assertEqual(x.sender_wan, sender_wan)
self.assertEqual(x.endpoint_id, endpoint_id)
self.assertEqual(x.wan_address, wan_address)
def test_uav_message_conversion(self):
for c in self._conversions:
if isinstance(c, APIMessageConversion):
meta = self._di._community.get_meta_message(API_MESSAGE_NAME)
data = "asjfdioewf"
addresses = [
Address.ipv4("0.0.0.1:1232"),
Address.ipv6("[::0]:12145"),
Address(port=32532)
]
message = meta.impl(
authentication=(self._di._community.my_member, ),
distribution=(self._di._community.claim_global_time(), ),
destination=(Candidate(addresses[0].addr(), True), ),
payload=(data, ))
encoded = c.encode_payload(message)
placeholder = c.Placeholder(None, meta, 0, encoded, False,
True)
_, x = c.decode_payload(placeholder, 0,
str(encoded[0]) + encoded[1])
self.assertEqual(x.message, data)
def test_sent_received(self):
addr = Address(ip="127.0.0.1")
self.dc.rcvd(2, 425, addr)
self.dc.rcvd(3, 334, addr)
self.assertEqual(self.dc.num_rcvd(), 5)
self.assertEqual(self.dc.total_rcvd(), 759)
self.assertNotEqual(self.dc.last_contact(addr), datetime.min)
self.dc.sent(2, 425, addr)
self.dc.sent(3, 334, addr)
self.assertEqual(self.dc.num_sent(), 5)
self.assertEqual(self.dc.total_sent(), 759)
self.assertSequenceEqual(self.dc.no_contact_since(), [self.main])
def setUp(self):
self.turn_wlan0_off()
callback = Callback("TestCallback")
self._ports = [12344]
self._addrs = [Address(port=p) for p in self._ports]
swift_process = MySwiftProcess(SWIFT_BINPATH, ".", None, self._addrs, None, None, None)
self._endpoint = MultiEndpoint(swift_process)
self._dispersy = MyDispersy(callback, self._endpoint, DISPERSY_WORKDIR, u":memory:")
self._dispersy.start()
self._directories = "testcase_swift_seed_and_down/"
self._dest_dir = DIRECTORY
self._filename = FILES[0]
self._roothash = get_hash(self._filename, SWIFT_BINPATH)
callback2 = Callback("TestCallback2")
self._ports2 = [34254]
self._addrs2 = [Address(port=p) for p in self._ports2]
swift_process2 = MySwiftProcess(SWIFT_BINPATH, ".", None, self._addrs2, None, None, None)
self._endpoint2 = MultiEndpoint(swift_process2)
self._dispersy2 = MyDispersy(callback2, self._endpoint2, u".", u":memory:")
self._dispersy2.start()
def test_plain_dict_raises_type_error(self):
# arrange
exception = None
address = None
# act
try:
address = Address(self.address_dict)
except Exception as ex:
exception = ex
# assert
self.assertIsInstance(exception, TypeError, 'type not TypeError')
def setUp(self):
self.addressVersion = 6
self.addressLength = 16
self.addressStr = '2001:db8::1428:57ab'
self.addressInt = 42540766411282592856903984951992014763L
self.addressLinkLocalStr = "fe80::1"
self.addressLoopbackStr = "::1"
self.addressMulticastStr = "ff00::1"
self.addressPrivateStr = "fc00::1"
self.addressReservedStr = "::1"
self.addressUnspecifiedStr = "::"
self.address = Address(self.addressStr)
self.ipv4AddressStr = "100.110.120.130"
self.ipv4AddressInt = 1684961410
def read_and_print_out(self, line):
# As soon as a TCP connection has been made, will the FastI2I be allowed to start
line.strip()
listenstr = "swift::Listen addr"
if line.find("Creating new TCP listener") != -1:
self._last_moreinfo.set()
elif line.find(listenstr) != -1:
addrstr = line[len(listenstr):]
saddr = Address.unknown(addrstr)
logger.debug("Found listen address %s", saddr)
if saddr != Address():
self.working_sockets.add(saddr)
if self._sockaddr_info_callback:
self._sockaddr_info_callback(saddr, 0)
def setUp(self):
self.addressVersion = 4
self.addressLength = 4
self.addressStr = '100.110.120.130'
self.addressInt = 1684961410
self.addressLinkLocalStr = "169.254.0.1"
self.addressLoopbackStr = "127.0.0.1"
self.addressMulticastStr = "224.0.0.1"
self.addressPrivateStr = "192.168.0.1"
self.addressReservedStr = "240.0.0.1"
self.addressUnspecifiedStr = "0.0.0.0"
self.address = Address(self.addressStr)
self.ipv6AddressStr = "fe80:0000:0000:0000:021b:77ff:fbd6:7860"
self.ipv6AddressInt = 338288524927261089654170743795120240736L
def test_addresses_message_conversion(self):
for c in self._conversions:
if isinstance(c, AddressesConversion):
meta = self._di._community.get_meta_message(
ADDRESSES_MESSAGE_NAME)
ids = [os.urandom(16) for _ in range(3)]
addresses = [
Address.ipv4("0.0.0.1:1232"),
Address.ipv6("[::0]:12145"),
Address(port=32532)
]
id_addresses = zip(ids, addresses, addresses)
message = meta.impl(
authentication=(self._di._community.my_member, ),
distribution=(self._di._community.claim_global_time(), ),
destination=(Candidate(id_addresses[0][1].addr(), True), ),
payload=(id_addresses, ))
encoded = c.encode_payload(message)
placeholder = c.Placeholder(None, meta, 0, encoded, False,
True)
_, x = c.decode_payload(placeholder, 0,
str(encoded[0]) + encoded[1])
self.assertEqual(x.id_addresses, id_addresses)
def interface_came_up(self, ip, if_name, device, gateway=None, port=0):
logger.debug("Interface came up with %s:%d %s %s %s", ip, port, if_name, device, gateway)
addr = Address.unknown(ip + ":" + str(port)) # Not very elegant.. But should work for ipv6 as well
if addr.resolve_interface():
if addr.interface.name != if_name:
logger.warning("Interfaces do not match %s %s", addr.interface.name, if_name)
addr.interface.device = device
addr.interface.gateway = gateway
if addr.interface.address is None: # In case netifaces does not recognize interface such as ppp
addr.interface.address = ip
self.dispersy_instance._endpoint.interface_came_up(addr)
else:
addr.set_interface(if_name, ip, None, None, device=device)
self.send_message(MESSAGE_KEY_SOCKET_STATE, addr, -2)
logger.debug("Bogus interface, cannot locate it")
def decode_payload(self, placeholder, offset, data):
if len(data) < offset + 4:
raise DropPacket("Insufficient packet size")
data_length, = struct.unpack_from("!L", data, offset)
offset += 4
if len(data) < offset + data_length:
raise DropPacket("Insufficient packet size")
data_payload = data[offset:offset + data_length]
data_pieces = data_payload.split(SEPARATOR)
filename = data_pieces[0]
directories = data_pieces[1]
roothash = data_pieces[2]
size = long(data_pieces[3])
timestamp = float(data_pieces[4])
addresses = [Address.unknown(a) for a in data_pieces[5:]]
offset += data_length
return offset, placeholder.meta.payload.implement(filename, directories, roothash, size, timestamp, addresses)
def test_file_hash_message_conversion(self):
for c in self._conversions:
if isinstance(c, FileHashConversion):
meta = self._di._community.get_meta_message(FILE_HASH_MESSAGE_NAME)
filename = "asdf.asdf"
dir_ = ""
roothash = "asdfj23j09f09sjfewef"
size = 103425
timestamp = 3232342.2342
address = Address.ipv4("0.0.0.0:1")
message = meta.impl(authentication=(self._di._community.my_member,), distribution=(self._di._community.claim_global_time(), self._di._community._file_hash_distribution.claim_sequence_number()),
payload=(filename, dir_, roothash, size, timestamp, [address]))
encoded = c.encode_payload(message)
placeholder = c.Placeholder(None, meta, 0, encoded, False, True)
_, x = c.decode_payload(placeholder, 0, str(encoded[0])+encoded[1])
self.assertEqual(x.filename, filename)
self.assertEqual(x.directories, dir_)
self.assertEqual(x.roothash, roothash)
self.assertEqual(x.size, size)
self.assertEqual(x.timestamp, timestamp)
self.assertEqual(x.addresses[0], address)
def __init__(self, stop_event=Event(), name="Network"):
'''
@param stop_event: Event that controls the run of this instance, but does not affect this event itself
@param name: Name of the instance, must reflect the location of configuration parameters
'''
name += ".Dispersy"
self.stop_event = stop_event
self.db_reader = StatusDbReader()
self.cfg = get_config(name)
self.status = get_status(name)
self.log = get_uav_logger(name)
di_kwargs = {}
try:
di_kwargs = self._get_arguments_from_config()
except:
logger.exception("Could not get arguments from config, make do with what you've got")
finally:
self.cfg.close_connection() # Don't need it anymore!
API.__init__(self, name, **di_kwargs)
self.status["state"] = self.STATES[self.state]
self.run_event = Event()
self.stop_on_dispersy_stop = True
self._stopping = False
# dictionary of known interfaces that should be used and the last state and time
self.use_interfaces = {} # (timestamp, status, ip)
self._listen_args = [Address.unknown(l) for l in di_kwargs.get("listen", [])]
# Set callbacks
self.state_change_callback(self._state_changed)
self.swift_state_callback(self._swift_state_changed)
# Set signal quit handler
signal.signal(signal.SIGQUIT, self.on_quit)
def setUp(self):
self._ports = [12345, 12346, 12347]
self._addrs = [Address.localport(p) for p in self._ports]
swift_process = MySwiftProcess(SWIFT_BINPATH, ".", None, self._addrs, None, None, None)
self._endpoint = MultiEndpoint(swift_process)
self._endpoint.open(FakeDispersy())
def got_moreinfo(self):
self.set_started()
self._last_moreinfo = datetime.utcnow()
for c in self.downloadimpl.midict.get("channels", []):
self._active_channels.add((Address.tuple((c["socket_ip"], c["socket_port"])),
Address.tuple((c["ip"], c["port"])))) # Tuple can handle both ipv4 and ipv6 (port can be string)
def add_message(self, message, addresses):
if len(message) < self.dispersy_instance._mtu:
addrs = [Address.unknown(a) for a in addresses]
self.dispersy_instance._register_some_message(APIMessageCarrier(message, addresses=addrs))
else:
logger.info("This message of length %d is to big to send with Dispersy", len(message))
def test_no_port_ipv4(self):
addr = Address.ipv4("0.0.0.0")
self.assertEqual(addr.port, 0)
self.assertEqual(addr.ip, "0.0.0.0")
self.assertEqual(addr.family, AF_INET)
def test_no_port_ipv6(self):
addr = Address.unknown("[::0]")
self.assertEqual(addr.port, 0)
self.assertEqual(addr.ip, "::0")
self.assertEqual(addr.family, AF_INET6)
def test_unknown_ipv4(self):
addr = Address.unknown("1.0.1.0:1")
self.assertEqual(addr.port, 1)
self.assertEqual(addr.ip, "1.0.1.0")
self.assertEqual(addr.family, AF_INET)
def test_resolve_interface(self):
addr = Address.unknown("127.0.0.1:1")
addr.resolve_interface()
self.assertEqual(addr.interface.name, "lo")
def shallow_copy(cls, contact):
assert isinstance(contact, DispersyContact)
# Copies are not really necessary probably
dc = DispersyContact(Address.copy(contact.address), peer=Peer.copy(contact.peer) if contact.peer is not None else None)
dc.community_ids = list(contact.community_ids)
return dc
def allowed_addresses(self):
if isinstance(self._destination, CandidateDestination.Implementation):
return [Address.tuple(c.sock_addr) for c in self._destination._candidates]
return None # We're not returning a list if this is not a candidatedestination.. Deal with it
