def testFindOptional(self, file_server, site, site_temp): file_server.ip_incoming = {} # Reset flood protection # Init source server site.connection_server = file_server file_server.sites[site.address] = site # Init full source server (has optional files) site_full = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT") file_server_full = FileServer("127.0.0.1", 1546) site_full.connection_server = file_server_full gevent.spawn(lambda: ConnectionServer.start(file_server_full)) time.sleep(0) # Port opening file_server_full.sites[site_full.address] = site_full # Add site site_full.storage.verifyFiles(quick_check=True) # Check optional files site_full_peer = site.addPeer("127.0.0.1", 1546) # Add it to source server assert site_full_peer.updateHashfield() # Update hashfield # Init client server site_temp.connection_server = ConnectionServer("127.0.0.1", 1545) site_temp.announce = mock.MagicMock(return_value=True) # Don't try to find peers from the net site_temp.addPeer("127.0.0.1", 1544) # Add source server # Download normal files site_temp.download(blind_includes=True).join(timeout=5) # Download optional data/optional.txt optional_file_info = site_temp.content_manager.getFileInfo("data/optional.txt") assert not site_temp.storage.isFile("data/optional.txt") assert not site.content_manager.hashfield.hasHash( optional_file_info["sha512"] ) # Source server don't know he has the file assert site_full_peer.hashfield.hasHash( optional_file_info["sha512"] ) # Source full peer on source server has the file assert site_full.content_manager.hashfield.hasHash( optional_file_info["sha512"] ) # Source full server he has the file with Spy.Spy(FileRequest, "route") as requests: # Request 2 file same time threads = [] threads.append(site_temp.needFile("data/optional.txt", blocking=False)) threads.append( site_temp.needFile( "data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif", blocking=False ) ) gevent.joinall(threads) assert ( len([request for request in requests if request[0] == "findHashIds"]) == 1 ) # findHashids should call only once assert site_temp.storage.isFile("data/optional.txt") assert site_temp.storage.isFile("data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif") assert site_temp.storage.deleteFiles() file_server_full.stop() [connection.close() for connection in file_server.connections]
def sitePublish(self, address, peer_ip=None, peer_port=15441, inner_path="content.json"): global file_server from Site import SiteManager from File import FileServer # We need fileserver to handle incoming file requests from Peer import Peer logging.info("Creating FileServer....") file_server = FileServer() file_server_thread = gevent.spawn(file_server.start, check_sites=False) # Dont check every site integrity file_server.openport() site = SiteManager.site_manager.list()[address] site.settings["serving"] = True # Serving the site even if its disabled # Notify local client on new content if config.ip_external: logging.info("Sending siteReload") my_peer = Peer(config.ip_external, config.fileserver_port) logging.info(my_peer.request("siteReload", {"site": site.address, "inner_path": inner_path})) if peer_ip: # Announce ip specificed site.addPeer(peer_ip, peer_port) else: # Just ask the tracker logging.info("Gathering peers from tracker") site.announce() # Gather peers published = site.publish(20, inner_path) # Push to 20 peers if published > 0: time.sleep(3) logging.info("Serving files (max 60s)...") gevent.joinall([file_server_thread], timeout=60) logging.info("Done.") else: logging.info("No peers found, sitePublish command only works if you already have visitors serving your site")
def testPex(self, file_server, site, site_temp): file_server.sites[site.address] = site client = FileServer("127.0.0.1", 1545) client.sites[site_temp.address] = site_temp site_temp.connection_server = client connection = client.getConnection("127.0.0.1", 1544) # Add new fake peer to site fake_peer = site.addPeer("1.2.3.4", 11337, return_peer=True) # Add fake connection to it fake_peer.connection = Connection(file_server, "1.2.3.4", 11337) fake_peer.connection.last_recv_time = time.time() assert fake_peer in site.getConnectablePeers() # Add file_server as peer to client peer_file_server = site_temp.addPeer("127.0.0.1", 1544) assert "1.2.3.4:11337" not in site_temp.peers assert peer_file_server.pex() assert "1.2.3.4:11337" in site_temp.peers # Should not exchange private peers from local network fake_peer_private = site.addPeer("192.168.0.1", 11337, return_peer=True) assert fake_peer_private not in site.getConnectablePeers(allow_private=False) fake_peer_private.connection = Connection(file_server, "192.168.0.1", 11337) fake_peer_private.connection.last_recv_time = time.time() assert "192.168.0.1:11337" not in site_temp.peers assert not peer_file_server.pex() assert "192.168.0.1:11337" not in site_temp.peers connection.close() client.stop()
def file_server(request): request.addfinalizer(CryptConnection.manager.removeCerts) # Remove cert files after end file_server = FileServer("127.0.0.1", 1544) gevent.spawn(lambda: ConnectionServer.start(file_server)) # Wait for port opening for retry in range(10): time.sleep(0.1) # Port opening try: conn = file_server.getConnection("127.0.0.1", 1544) conn.close() break except Exception, err: print err
def testHashfieldExchange(self, file_server, site, site_temp): server1 = file_server server1.ip_incoming = {} # Reset flood protection server1.sites[site.address] = site server2 = FileServer("127.0.0.1", 1545) server2.sites[site_temp.address] = site_temp site_temp.connection_server = server2 site.storage.verifyFiles(quick_check=True) # Find what optional files we have # Add file_server as peer to client server2_peer1 = site_temp.addPeer("127.0.0.1", 1544) # Check if hashfield has any files assert len(site.content_manager.hashfield) > 0 # Testing hashfield sync assert len(server2_peer1.hashfield) == 0 assert server2_peer1.updateHashfield() # Query hashfield from peer assert len(server2_peer1.hashfield) > 0 # Test force push new hashfield site_temp.content_manager.hashfield.appendHash("AABB") server1_peer2 = site.addPeer("127.0.0.1", 1545, return_peer=True) with Spy.Spy(FileRequest, "route") as requests: assert len(server1_peer2.hashfield) == 0 server2_peer1.sendMyHashfield() assert len(server1_peer2.hashfield) == 1 server2_peer1.sendMyHashfield() # Hashfield not changed, should be ignored assert len(requests) == 1 time.sleep(0.01) # To make hashfield change date different site_temp.content_manager.hashfield.appendHash("AACC") server2_peer1.sendMyHashfield() # Push hashfield assert len(server1_peer2.hashfield) == 2 assert len(requests) == 2 site_temp.content_manager.hashfield.appendHash("AADD") assert server1_peer2.updateHashfield(force=True) # Request hashfield assert len(server1_peer2.hashfield) == 3 assert len(requests) == 3 assert not server2_peer1.sendMyHashfield() # Not changed, should be ignored assert len(requests) == 3 server2.stop()
def main(self): global ui_server, file_server from File import FileServer from Ui import UiServer logging.info("Creating FileServer....") file_server = FileServer() logging.info("Creating UiServer....") ui_server = UiServer() file_server.ui_server = ui_server logging.info("Removing old SSL certs...") from Crypt import CryptConnection CryptConnection.manager.removeCerts() logging.info("Starting servers....") gevent.joinall([gevent.spawn(ui_server.start), gevent.spawn(file_server.start)])
def sitePublish(self, address, peer_ip=None, peer_port=15441, inner_path="content.json", diffs={}): global file_server from Site import Site from Site import SiteManager from File import FileServer # We need fileserver to handle incoming file requests from Peer import Peer SiteManager.site_manager.load() logging.info("Loading site...") site = Site(address, allow_create=False) site.settings["serving"] = True # Serving the site even if its disabled logging.info("Creating FileServer....") file_server = FileServer() site.connection_server = file_server file_server_thread = gevent.spawn(file_server.start, check_sites=False) # Dont check every site integrity time.sleep(0.001) if not file_server_thread.ready(): # Started fileserver file_server.openport() if peer_ip: # Announce ip specificed site.addPeer(peer_ip, peer_port) else: # Just ask the tracker logging.info("Gathering peers from tracker") site.announce() # Gather peers published = site.publish(5, inner_path, diffs=diffs) # Push to peers if published > 0: time.sleep(3) logging.info("Serving files (max 60s)...") gevent.joinall([file_server_thread], timeout=60) logging.info("Done.") else: logging.info("No peers found, sitePublish command only works if you already have visitors serving your site") else: # Already running, notify local client on new content logging.info("Sending siteReload") if config.fileserver_ip == "*": my_peer = Peer("127.0.0.1", config.fileserver_port) else: my_peer = Peer(config.fileserver_ip, config.fileserver_port) logging.info(my_peer.request("siteReload", {"site": site.address, "inner_path": inner_path})) logging.info("Sending sitePublish") logging.info(my_peer.request("sitePublish", {"site": site.address, "inner_path": inner_path, "diffs": diffs})) logging.info("Done.")
def testPing(self, file_server, site, site_temp): file_server.ip_incoming = {} # Reset flood protection file_server.sites[site.address] = site client = FileServer("127.0.0.1", 1545) client.sites[site_temp.address] = site_temp site_temp.connection_server = client connection = client.getConnection("127.0.0.1", 1544) # Add file_server as peer to client peer_file_server = site_temp.addPeer("127.0.0.1", 1544) assert peer_file_server.ping() assert peer_file_server in site_temp.peers.values() peer_file_server.remove() assert peer_file_server not in site_temp.peers.values() connection.close() client.stop()
def testRangedFileRequest(self, file_server, site, site_temp): inner_path = self.createBigfile(site) file_server.sites[site.address] = site client = FileServer("127.0.0.1", 1545) client.sites[site_temp.address] = site_temp site_temp.connection_server = client connection = client.getConnection("127.0.0.1", 1544) # Add file_server as peer to client peer_file_server = site_temp.addPeer("127.0.0.1", 1544) buff = peer_file_server.getFile(site_temp.address, "%s|%s-%s" % (inner_path, 5 * 1024 * 1024, 6 * 1024 * 1024)) assert len(buff.getvalue()) == 1 * 1024 * 1024 # Correct block size assert buff.getvalue().startswith("Test524") # Correct data buff.seek(0) assert site.content_manager.verifyPiece(inner_path, 5 * 1024 * 1024, buff) # Correct hash connection.close() client.stop()
def sitePublish(self, address, peer_ip=None, peer_port=15441, inner_path="content.json"): global file_server from Site import Site from File import FileServer # We need fileserver to handle incoming file requests logging.info("Creating FileServer....") file_server = FileServer() file_server_thread = gevent.spawn(file_server.start, check_sites=False) # Dont check every site integrity file_server.openport() site = file_server.sites[address] site.settings["serving"] = True # Serving the site even if its disabled if peer_ip: # Announce ip specificed site.addPeer(peer_ip, peer_port) else: # Just ask the tracker logging.info("Gathering peers from tracker") site.announce() # Gather peers site.publish(20, inner_path) # Push to 20 peers time.sleep(3) logging.info("Serving files...") gevent.joinall([file_server_thread]) logging.info("Done.")
def testDownloadFile(self, file_server, site, site_temp): file_server.ip_incoming = {} # Reset flood protection file_server.sites[site.address] = site client = FileServer("127.0.0.1", 1545) client.sites[site_temp.address] = site_temp site_temp.connection_server = client connection = client.getConnection("127.0.0.1", 1544) # Add file_server as peer to client peer_file_server = site_temp.addPeer("127.0.0.1", 1544) # Testing streamFile buff = peer_file_server.streamFile(site_temp.address, "content.json") assert "sign" in buff.getvalue() # Testing getFile buff = peer_file_server.getFile(site_temp.address, "content.json") assert "sign" in buff.getvalue() connection.close() client.stop()
def testFindOptional(self, file_server, site, site_temp): file_server.ip_incoming = {} # Reset flood protection # Init source server site.connection_server = file_server file_server.sites[site.address] = site # Init full source server (has optional files) site_full = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT") file_server_full = FileServer("127.0.0.1", 1546) site_full.connection_server = file_server_full gevent.spawn(lambda: ConnectionServer.start(file_server_full)) time.sleep(0) # Port opening file_server_full.sites[site_full.address] = site_full # Add site site_full.storage.verifyFiles(quick_check=True) # Check optional files site_full_peer = site.addPeer("127.0.0.1", 1546) # Add it to source server assert site_full_peer.updateHashfield() # Update hashfield # Init client server site_temp.connection_server = ConnectionServer("127.0.0.1", 1545) site_temp.announce = mock.MagicMock(return_value=True) # Don't try to find peers from the net site_temp.addPeer("127.0.0.1", 1544) # Add source server # Download normal files site_temp.download(blind_includes=True).join(timeout=5) # Download optional data/optional.txt optional_file_info = site_temp.content_manager.getFileInfo("data/optional.txt") assert not site_temp.storage.isFile("data/optional.txt") assert not site.content_manager.hashfield.hasHash(optional_file_info["sha512"]) # Source server don't know he has the file assert site_full_peer.hashfield.hasHash(optional_file_info["sha512"]) # Source full peer on source server has the file assert site_full.content_manager.hashfield.hasHash(optional_file_info["sha512"]) # Source full server he has the file with Spy.Spy(FileRequest, "route") as requests: site_temp.needFile("data/optional.txt") print requests assert site_temp.storage.deleteFiles() file_server_full.stop() [connection.close() for connection in file_server.connections]
def testHashfieldExchange(self, file_server, site, site_temp): file_server.ip_incoming = {} # Reset flood protection file_server.sites[site.address] = site client = FileServer("127.0.0.1", 1545) client.sites[site_temp.address] = site_temp site_temp.connection_server = client connection = client.getConnection("127.0.0.1", 1544) site.storage.verifyFiles(quick_check=True) # Find what optional files we have # Add file_server as peer to client peer_file_server = site_temp.addPeer("127.0.0.1", 1544) # Check if hashfield has any files assert len(site.content_manager.hashfield) > 0 # Testing hashfield sync assert len(peer_file_server.hashfield) == 0 assert peer_file_server.getHashfield() assert len(peer_file_server.hashfield) > 0 connection.close() client.stop()
def testPex(self, file_server, site, site_temp): file_server.sites[site.address] = site client = FileServer("127.0.0.1", 1545) client.sites[site_temp.address] = site_temp site_temp.connection_server = client connection = client.getConnection("127.0.0.1", 1544) # Add new fake peer to site fake_peer = site.addPeer("1.2.3.4", 11337, return_peer=True) # Add fake connection to it fake_peer.connection = Connection(file_server, "1.2.3.4", 11337) fake_peer.connection.last_recv_time = time.time() assert fake_peer in site.getConnectablePeers() # Add file_server as peer to client peer_file_server = site_temp.addPeer("127.0.0.1", 1544) assert "1.2.3.4:11337" not in site_temp.peers assert peer_file_server.pex() assert "1.2.3.4:11337" in site_temp.peers connection.close() client.stop()
def siteNeedFile(self, address, inner_path): from Site import Site from Site import SiteManager SiteManager.site_manager.load() address = getPublicKey(address) def checker(): while 1: s = time.time() time.sleep(1) print "Switch time:", time.time() - s gevent.spawn(checker) logging.info("Opening a simple connection server") global file_server from File import FileServer file_server = FileServer("127.0.0.1", 1234) file_server_thread = gevent.spawn(file_server.start, check_sites=False) site = Site(address) site.announce() print site.needFile(inner_path, update=True)
def testFindHash(self, file_server, site, site_temp): file_server.sites[site.address] = site client = FileServer(file_server.ip, 1545) client.sites[site_temp.address] = site_temp site_temp.connection_server = client # Add file_server as peer to client peer_file_server = site_temp.addPeer(file_server.ip, 1544) assert peer_file_server.findHashIds([1234]) == {} # Add fake peer with requred hash fake_peer_1 = site.addPeer(file_server.ip_external, 1544) fake_peer_1.hashfield.append(1234) fake_peer_2 = site.addPeer("1.2.3.5", 1545) fake_peer_2.hashfield.append(1234) fake_peer_2.hashfield.append(1235) fake_peer_3 = site.addPeer("1.2.3.6", 1546) fake_peer_3.hashfield.append(1235) fake_peer_3.hashfield.append(1236) res = peer_file_server.findHashIds([1234, 1235]) assert sorted(res[1234]) == sorted([(file_server.ip_external, 1544), ("1.2.3.5", 1545)]) assert sorted(res[1235]) == sorted([("1.2.3.5", 1545), ("1.2.3.6", 1546)]) # Test my address adding site.content_manager.hashfield.append(1234) res = peer_file_server.findHashIds([1234, 1235]) assert sorted(res[1234]) == sorted([(file_server.ip_external, 1544), ("1.2.3.5", 1545), (file_server.ip, 1544)]) assert sorted(res[1235]) == sorted([("1.2.3.5", 1545), ("1.2.3.6", 1546)])
def testWorkerManagerPiecefieldDownload(self, file_server, site, site_temp): inner_path = self.createBigfile(site) server1 = file_server server1.sites[site.address] = site server2 = FileServer("127.0.0.1", 1545) server2.sites[site_temp.address] = site_temp site_temp.connection_server = server2 sha512 = site.content_manager.getFileInfo(inner_path)["sha512"] # Create 10 fake peer for each piece for i in range(10): peer = Peer("127.0.0.1", 1544, site_temp, server2) peer.piecefields[sha512][i] = "1" peer.updateHashfield = mock.MagicMock(return_value=False) peer.updatePiecefields = mock.MagicMock(return_value=False) peer.findHashIds = mock.MagicMock(return_value={"nope": []}) peer.hashfield = site.content_manager.hashfield peer.has_hashfield = True peer.key = "Peer:%s" % i site_temp.peers["Peer:%s" % i] = peer site_temp.downloadContent("content.json", download_files=False) site_temp.needFile("data/optional.any.iso.piecemap.msgpack") with Spy.Spy(Peer, "getFile") as requests: for i in range(10): site_temp.needFile("%s|%s-%s" % (inner_path, i * 1024 * 1024, (i + 1) * 1024 * 1024)) assert len(requests) == 10 for i in range(10): assert requests[i][0] == site_temp.peers[ "Peer:%s" % i] # Every part should be requested from piece owner peer
def testPex(self, file_server, site, site_temp): file_server.sites[site.address] = site client = FileServer(file_server.ip, 1545) client.sites = {site_temp.address: site_temp} site_temp.connection_server = client connection = client.getConnection(file_server.ip, 1544) # Add new fake peer to site fake_peer = site.addPeer(file_server.ip_external, 11337, return_peer=True) # Add fake connection to it fake_peer.connection = Connection(file_server, file_server.ip_external, 11337) fake_peer.connection.last_recv_time = time.time() assert fake_peer in site.getConnectablePeers() # Add file_server as peer to client peer_file_server = site_temp.addPeer(file_server.ip, 1544) assert "%s:11337" % file_server.ip_external not in site_temp.peers assert peer_file_server.pex() assert "%s:11337" % file_server.ip_external in site_temp.peers # Should not exchange private peers from local network fake_peer_private = site.addPeer("192.168.0.1", 11337, return_peer=True) assert fake_peer_private not in site.getConnectablePeers( allow_private=False) fake_peer_private.connection = Connection(file_server, "192.168.0.1", 11337) fake_peer_private.connection.last_recv_time = time.time() assert "192.168.0.1:11337" not in site_temp.peers assert not peer_file_server.pex() assert "192.168.0.1:11337" not in site_temp.peers connection.close() client.stop()
def testDownloadFile(self, file_server, site, site_temp): file_server.sites[site.address] = site client = FileServer(file_server.ip, 1545) client.sites = {site_temp.address: site_temp} site_temp.connection_server = client connection = client.getConnection(file_server.ip, 1544) # Add file_server as peer to client peer_file_server = site_temp.addPeer(file_server.ip, 1544) # Testing streamFile buff = peer_file_server.getFile(site_temp.address, "content.json", streaming=True) assert b"sign" in buff.getvalue() # Testing getFile buff = peer_file_server.getFile(site_temp.address, "content.json") assert b"sign" in buff.getvalue() connection.close() client.stop()
def testArchivedBeforeDownload(self, file_server, site, site_temp): # Init source server site.connection_server = file_server file_server.sites[site.address] = site # Init client server client = FileServer(file_server.ip, 1545) client.sites[site_temp.address] = site_temp site_temp.connection_server = client # Download normally site_temp.addPeer(file_server.ip, 1544) site_temp.download(blind_includes=True).join(timeout=5) bad_files = site_temp.storage.verifyFiles( quick_check=True)["bad_files"] assert not bad_files assert "data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json" in site_temp.content_manager.contents assert site_temp.storage.isFile( "data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json") assert len(list(site_temp.storage.query("SELECT * FROM comment"))) == 2 # Add archived data assert not "archived_before" in site.content_manager.contents[ "data/users/content.json"]["user_contents"] assert not site.content_manager.isArchived( "data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json", time.time() - 1) content_modification_time = site.content_manager.contents[ "data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json"][ "modified"] site.content_manager.contents["data/users/content.json"][ "user_contents"]["archived_before"] = content_modification_time site.content_manager.sign( "data/users/content.json", privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv") date_archived = site.content_manager.contents[ "data/users/content.json"]["user_contents"]["archived_before"] assert site.content_manager.isArchived( "data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json", date_archived - 1) assert site.content_manager.isArchived( "data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json", date_archived) assert not site.content_manager.isArchived( "data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json", date_archived + 1) # Allow user to update archived data later # Push archived update assert not "archived_before" in site_temp.content_manager.contents[ "data/users/content.json"]["user_contents"] site.publish() time.sleep(0.1) site_temp.download(blind_includes=True).join( timeout=5) # Wait for download # The archived content should disappear from remote client assert "archived_before" in site_temp.content_manager.contents[ "data/users/content.json"]["user_contents"] assert "data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json" not in site_temp.content_manager.contents assert not site_temp.storage.isDir( "data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q") assert len(list(site_temp.storage.query("SELECT * FROM comment"))) == 1 assert len( list( site_temp.storage.query( "SELECT * FROM json WHERE directory LIKE '%1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q%'" ))) == 0 assert site_temp.storage.deleteFiles() [connection.close() for connection in file_server.connections]
def sitePublish(self, address, peer_ip=None, peer_port=15441, inner_path="content.json", diffs={}): global file_server from Site import Site from Site import SiteManager from File import FileServer # We need fileserver to handle incoming file requests from Peer import Peer SiteManager.site_manager.load() logging.info("Loading site...") site = Site(address, allow_create=False) site.settings[ "serving"] = True # Serving the site even if its disabled logging.info("Creating FileServer....") file_server = FileServer() site.connection_server = file_server file_server_thread = gevent.spawn( file_server.start, check_sites=False) # Dont check every site integrity time.sleep(0.001) if not file_server_thread.ready(): # Started fileserver file_server.openport() if peer_ip: # Announce ip specificed site.addPeer(peer_ip, peer_port) else: # Just ask the tracker logging.info("Gathering peers from tracker") site.announce() # Gather peers published = site.publish(5, inner_path, diffs=diffs) # Push to peers if published > 0: time.sleep(3) logging.info("Serving files (max 60s)...") gevent.joinall([file_server_thread], timeout=60) logging.info("Done.") else: logging.info( "No peers found, sitePublish command only works if you already have visitors serving your site" ) else: # Already running, notify local client on new content logging.info("Sending siteReload") my_peer = Peer("127.0.0.1", config.fileserver_port) logging.info( my_peer.request("siteReload", { "site": site.address, "inner_path": inner_path })) logging.info("Sending sitePublish") logging.info( my_peer.request("sitePublish", { "site": site.address, "inner_path": inner_path, "diffs": diffs })) logging.info("Done.")
def testFileRename(self, file_server, site, site_temp): inner_path = self.createBigfile(site) # Init source server site.connection_server = file_server file_server.sites[site.address] = site # Init client server site_temp.connection_server = FileServer(file_server.ip, 1545) site_temp.connection_server.sites[site_temp.address] = site_temp site_temp.addPeer(file_server.ip, 1544) # Download site site_temp.download(blind_includes=True).join(timeout=5) with Spy.Spy(FileRequest, "route") as requests: site_temp.needFile("%s|%s-%s" % (inner_path, 0, 1 * self.piece_size)) assert len([req for req in requests if req[1] == "streamFile" ]) == 2 # 1 piece + piecemap # Rename the file inner_path_new = inner_path.replace(".iso", "-new.iso") site.storage.rename(inner_path, inner_path_new) site.storage.delete("data/optional.any.iso.piecemap.msgpack") assert site.content_manager.sign("content.json", self.privatekey, remove_missing_optional=True) files_optional = site.content_manager.contents["content.json"][ "files_optional"].keys() assert "data/optional.any-new.iso.piecemap.msgpack" in files_optional assert "data/optional.any.iso.piecemap.msgpack" not in files_optional assert "data/optional.any.iso" not in files_optional with Spy.Spy(FileRequest, "route") as requests: site.publish() time.sleep(0.1) site_temp.download(blind_includes=True).join( timeout=5) # Wait for download assert len([req[1] for req in requests if req[1] == "streamFile"]) == 0 with site_temp.storage.openBigfile(inner_path_new, prebuffer=0) as f: f.read(1024) # First piece already downloaded assert [req for req in requests if req[1] == "streamFile"] == [] # Second piece needs to be downloaded + changed piecemap f.seek(self.piece_size) f.read(1024) assert [ req[3]["inner_path"] for req in requests if req[1] == "streamFile" ] == [inner_path_new + ".piecemap.msgpack", inner_path_new]
def testArchivedDownload(self, file_server, site, site_temp): file_server.ip_incoming = {} # Reset flood protection # Init source server site.connection_server = file_server file_server.sites[site.address] = site # Init client server client = FileServer("127.0.0.1", 1545) client.sites[site_temp.address] = site_temp site_temp.connection_server = client # Don't try to find peers from the net site.announce = mock.MagicMock(return_value=True) site_temp.announce = mock.MagicMock(return_value=True) # Download normally site_temp.addPeer("127.0.0.1", 1544) site_temp.download(blind_includes=True).join(timeout=5) bad_files = site_temp.storage.verifyFiles(quick_check=True) assert not bad_files assert "data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json" in site_temp.content_manager.contents assert site_temp.storage.isFile( "data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json") assert len(list(site_temp.storage.query("SELECT * FROM comment"))) == 2 # Add archived data assert not "archived" in site.content_manager.contents[ "data/users/content.json"]["user_contents"] assert not site.content_manager.isArchived( "data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json", time.time() - 1) site.content_manager.contents["data/users/content.json"][ "user_contents"]["archived"] = { "1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q": time.time() } site.content_manager.sign( "data/users/content.json", privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv") date_archived = site.content_manager.contents[ "data/users/content.json"]["user_contents"]["archived"][ "1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q"] assert site.content_manager.isArchived( "data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json", date_archived - 1) assert site.content_manager.isArchived( "data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json", date_archived) assert not site.content_manager.isArchived( "data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json", date_archived + 1) # Allow user to update archived data later # Push archived update assert not "archived" in site_temp.content_manager.contents[ "data/users/content.json"]["user_contents"] site.publish() site_temp.download(blind_includes=True).join( timeout=5) # Wait for download # The archived content should disappear from remote client assert "archived" in site_temp.content_manager.contents[ "data/users/content.json"]["user_contents"] assert "data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q/content.json" not in site_temp.content_manager.contents assert not site_temp.storage.isDir( "data/users/1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q") assert len(list(site_temp.storage.query("SELECT * FROM comment"))) == 1 assert len( list( site_temp.storage.query( "SELECT * FROM json WHERE directory LIKE '%1C5sgvWaSgfaTpV5kjBCnCiKtENNMYo69q%'" ))) == 0 assert site_temp.storage.deleteFiles() [connection.close() for connection in file_server.connections]
def testUpdate(self, file_server, site, site_temp): file_server.ip_incoming = {} # Reset flood protection assert site.storage.directory == config.data_dir + "/" + site.address assert site_temp.storage.directory == config.data_dir + "-temp/" + site.address # Init source server site.connection_server = file_server file_server.sites[site.address] = site # Init client server client = FileServer("127.0.0.1", 1545) client.sites[site_temp.address] = site_temp site_temp.connection_server = client # Don't try to find peers from the net site.announce = mock.MagicMock(return_value=True) site_temp.announce = mock.MagicMock(return_value=True) # Connect peers site_temp.addPeer("127.0.0.1", 1544) # Download site from site to site_temp site_temp.download(blind_includes=True).join(timeout=5) # Update file data_original = site.storage.open("data/data.json").read() data_new = data_original.replace('"ZeroBlog"', '"UpdatedZeroBlog"') assert data_original != data_new site.storage.open("data/data.json", "wb").write(data_new) assert site.storage.open("data/data.json").read() == data_new assert site_temp.storage.open("data/data.json").read() == data_original # Publish without patch with Spy.Spy(FileRequest, "route") as requests: site.content_manager.sign( "content.json", privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv" ) site.publish() site_temp.download(blind_includes=True).join(timeout=5) assert len([ request for request in requests if request[0] in ("getFile", "streamFile") ]) == 1 assert site_temp.storage.open("data/data.json").read() == data_new # Close connection to avoid update spam limit site.peers.values()[0].remove() site.addPeer("127.0.0.1", 1545) site_temp.peers.values()[0].ping() # Connect back time.sleep(0.1) # Update with patch data_new = data_original.replace('"ZeroBlog"', '"PatchedZeroBlog"') assert data_original != data_new site.storage.open("data/data.json-new", "wb").write(data_new) assert site.storage.open("data/data.json-new").read() == data_new assert site_temp.storage.open("data/data.json").read() != data_new # Generate diff diffs = site.content_manager.getDiffs("content.json") assert not site.storage.isFile( "data/data.json-new") # New data file removed assert site.storage.open( "data/data.json").read() == data_new # -new postfix removed assert "data/data.json" in diffs assert diffs["data/data.json"] == [ ('=', 2), ('-', 29), ('+', ['\t"title": "PatchedZeroBlog",\n']), ('=', 31102) ] # Publish with patch with Spy.Spy(FileRequest, "route") as requests: site.content_manager.sign( "content.json", privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv" ) site.publish(diffs=diffs) site_temp.download(blind_includes=True).join(timeout=5) assert len([ request for request in requests if request[0] in ("getFile", "streamFile") ]) == 0 assert site_temp.storage.open("data/data.json").read() == data_new assert site_temp.storage.deleteFiles() [connection.close() for connection in file_server.connections]
def testFindOptional(self, file_server, site, site_temp): file_server.ip_incoming = {} # Reset flood protection # Init source server site.connection_server = file_server file_server.sites[site.address] = site # Init full source server (has optional files) site_full = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT") file_server_full = FileServer("127.0.0.1", 1546) site_full.connection_server = file_server_full gevent.spawn(lambda: ConnectionServer.start(file_server_full)) time.sleep(0.001) # Port opening file_server_full.sites[site_full.address] = site_full # Add site site_full.storage.verifyFiles(quick_check=True) # Check optional files site_full_peer = site.addPeer("127.0.0.1", 1546) # Add it to source server assert site_full_peer.updateHashfield() # Update hashfield # Init client server site_temp.connection_server = ConnectionServer("127.0.0.1", 1545) site_temp.announce = mock.MagicMock( return_value=True) # Don't try to find peers from the net site_temp.addPeer("127.0.0.1", 1544) # Add source server # Download normal files site_temp.download(blind_includes=True).join(timeout=5) # Download optional data/optional.txt optional_file_info = site_temp.content_manager.getFileInfo( "data/optional.txt") assert not site_temp.storage.isFile("data/optional.txt") assert not site.content_manager.hashfield.hasHash( optional_file_info["sha512"] ) # Source server don't know he has the file assert site_full_peer.hashfield.hasHash( optional_file_info["sha512"] ) # Source full peer on source server has the file assert site_full.content_manager.hashfield.hasHash( optional_file_info["sha512"]) # Source full server he has the file with Spy.Spy(FileRequest, "route") as requests: # Request 2 file same time threads = [] threads.append( site_temp.needFile("data/optional.txt", blocking=False)) threads.append( site_temp.needFile( "data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif", blocking=False)) gevent.joinall(threads) assert len([ request for request in requests if request[0] == "findHashIds" ]) == 1 # findHashids should call only once assert site_temp.storage.isFile("data/optional.txt") assert site_temp.storage.isFile( "data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif" ) assert site_temp.storage.deleteFiles() file_server_full.stop() [connection.close() for connection in file_server.connections]
def testBigUpdate(self, file_server, site, site_temp): # Init source server site.connection_server = file_server file_server.sites[site.address] = site # Init client server client = FileServer(file_server.ip, 1545) client.sites = {site_temp.address: site_temp} site_temp.connection_server = client # Connect peers site_temp.addPeer(file_server.ip, 1544) # Download site from site to site_temp assert site_temp.download(blind_includes=True, retry_bad_files=False).get(timeout=10) assert list(site_temp.bad_files.keys()) == [ "data/users/1J6UrZMkarjVg5ax9W4qThir3BFUikbW6C/content.json" ] # Update file data_original = site.storage.open("data/data.json").read() data_new = data_original.replace(b'"ZeroBlog"', b'"PatchedZeroBlog"') assert data_original != data_new site.storage.open("data/data.json-new", "wb").write(data_new) assert site.storage.open("data/data.json-new").read() == data_new assert site_temp.storage.open("data/data.json").read() != data_new # Generate diff diffs = site.content_manager.getDiffs("content.json") assert not site.storage.isFile( "data/data.json-new") # New data file removed assert site.storage.open( "data/data.json").read() == data_new # -new postfix removed assert "data/data.json" in diffs content_json = site.storage.loadJson("content.json") content_json["description"] = "BigZeroBlog" * 1024 * 10 site.storage.writeJson("content.json", content_json) site.content_manager.loadContent("content.json", force=True) # Publish with patch site.log.info("Publish new data.json with patch") with Spy.Spy(FileRequest, "route") as requests: site.content_manager.sign( "content.json", privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv" ) assert site.storage.getSize( "content.json") > 10 * 1024 # Make it a big content.json site.publish(diffs=diffs) time.sleep(0.1) assert site_temp.download(blind_includes=True, retry_bad_files=False).get(timeout=10) file_requests = [ request for request in requests if request[1] in ("getFile", "streamFile") ] assert len(file_requests) == 1 assert site_temp.storage.open("data/data.json").read() == data_new assert site_temp.storage.open( "content.json").read() == site.storage.open("content.json").read()
def testFindOptional(self, file_server, site, site_temp): # Init source server site.connection_server = file_server file_server.sites[site.address] = site # Init full source server (has optional files) site_full = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT") file_server_full = FileServer(file_server.ip, 1546) site_full.connection_server = file_server_full def listen(): ConnectionServer.start(file_server_full) ConnectionServer.listen(file_server_full) gevent.spawn(listen) time.sleep(0.001) # Port opening file_server_full.sites[site_full.address] = site_full # Add site site_full.storage.verifyFiles(quick_check=True) # Check optional files site_full_peer = site.addPeer(file_server.ip, 1546) # Add it to source server hashfield = site_full_peer.updateHashfield() # Update hashfield assert len(site_full.content_manager.hashfield) == 8 assert hashfield assert site_full.storage.isFile("data/optional.txt") assert site_full.storage.isFile( "data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif" ) assert len(site_full_peer.hashfield) == 8 # Remove hashes from source server for hash in list(site.content_manager.hashfield): site.content_manager.hashfield.remove(hash) # Init client server site_temp.connection_server = ConnectionServer(file_server.ip, 1545) site_temp.addPeer(file_server.ip, 1544) # Add source server # Download normal files site_temp.log.info("Start Downloading site") site_temp.download(blind_includes=True).join(timeout=5) # Download optional data/optional.txt optional_file_info = site_temp.content_manager.getFileInfo( "data/optional.txt") optional_file_info2 = site_temp.content_manager.getFileInfo( "data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif" ) assert not site_temp.storage.isFile("data/optional.txt") assert not site_temp.storage.isFile( "data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif" ) assert not site.content_manager.hashfield.hasHash( optional_file_info["sha512"] ) # Source server don't know he has the file assert not site.content_manager.hashfield.hasHash( optional_file_info2["sha512"] ) # Source server don't know he has the file assert site_full_peer.hashfield.hasHash( optional_file_info["sha512"] ) # Source full peer on source server has the file assert site_full_peer.hashfield.hasHash( optional_file_info2["sha512"] ) # Source full peer on source server has the file assert site_full.content_manager.hashfield.hasHash( optional_file_info["sha512"]) # Source full server he has the file assert site_full.content_manager.hashfield.hasHash( optional_file_info2["sha512"] ) # Source full server he has the file site_temp.log.info("Request optional files") with Spy.Spy(FileRequest, "route") as requests: # Request 2 file same time threads = [] threads.append( site_temp.needFile("data/optional.txt", blocking=False)) threads.append( site_temp.needFile( "data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif", blocking=False)) gevent.joinall(threads) assert len([ request for request in requests if request[1] == "findHashIds" ]) == 1 # findHashids should call only once assert site_temp.storage.isFile("data/optional.txt") assert site_temp.storage.isFile( "data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif" ) assert site_temp.storage.deleteFiles() file_server_full.stop() [connection.close() for connection in file_server.connections] site_full.content_manager.contents.db.close()
def testRenameOptional(self, file_server, site, site_temp): assert site.storage.directory == config.data_dir + "/" + site.address assert site_temp.storage.directory == config.data_dir + "-temp/" + site.address # Init source server site.connection_server = file_server file_server.sites[site.address] = site # Init client server client = FileServer(file_server.ip, 1545) client.sites[site_temp.address] = site_temp site_temp.connection_server = client site_temp.announce = mock.MagicMock( return_value=True) # Don't try to find peers from the net site_temp.addPeer(file_server.ip, 1544) site_temp.download(blind_includes=True).join(timeout=5) assert site_temp.settings["optional_downloaded"] == 0 site_temp.needFile("data/optional.txt") assert site_temp.settings["optional_downloaded"] > 0 settings_before = site_temp.settings hashfield_before = site_temp.content_manager.hashfield.tobytes() # Rename optional file os.rename(site.storage.getPath("data/optional.txt"), site.storage.getPath("data/optional-new.txt")) site.content_manager.sign( "content.json", privatekey="5KUh3PvNm5HUWoCfSUfcYvfQ2g3PrRNJWr6Q9eqdBGu23mtMntv", remove_missing_optional=True) content = site.storage.loadJson("content.json") assert "data/optional-new.txt" in content["files_optional"] assert "data/optional.txt" not in content["files_optional"] assert not site_temp.storage.isFile("data/optional-new.txt") assert site_temp.storage.isFile("data/optional.txt") with Spy.Spy(FileRequest, "route") as requests: site.publish() time.sleep(0.1) site_temp.download(blind_includes=True).join( timeout=5) # Wait for download assert "streamFile" not in [req[1] for req in requests] content = site_temp.storage.loadJson("content.json") assert "data/optional-new.txt" in content["files_optional"] assert "data/optional.txt" not in content["files_optional"] assert site_temp.storage.isFile("data/optional-new.txt") assert not site_temp.storage.isFile("data/optional.txt") assert site_temp.settings["size"] == settings_before["size"] assert site_temp.settings["size_optional"] == settings_before[ "size_optional"] assert site_temp.settings["optional_downloaded"] == settings_before[ "optional_downloaded"] assert site_temp.content_manager.hashfield.tobytes( ) == hashfield_before assert site_temp.storage.deleteFiles() [connection.close() for connection in file_server.connections]