def test_putcontent_unlinked(self): """Try to put content in an unlinked file.""" empty_hash = content_hash_factory().content_hash() data = "*" size = 1 hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) def auth(client): # setup d = client.dummy_authenticate("open sesame") d.addCallback(lambda r: client.get_root()) # create file and remove it d.addCallback(lambda r: client.make_file(request.ROOT, r, "hola")) d.addCallback(lambda req: self._save_state("file", req.new_id)) d.addCallback( lambda _: client.unlink(request.ROOT, self._state.file)) # try to put content d.addCallback(lambda _: client.put_content( request.ROOT, self._state.file, empty_hash, hash_value, crc32_value, size, StringIO(data))) d.addCallbacks(client.test_fail, lambda x: client.test_done("ok")) return self.callback_test(auth)
def test_proxy_producer_not_streaming(self): """Test ProxyHashingProducer.""" data = os.urandom(1024 * 10) message = zlib.compress(data) ds = diskstorage.DiskStorage(os.path.join(self.tmpdir, "testfile")) consumer = ds.put("somenode") producer = upload.ProxyHashingProducer(consumer, False) # add chunks, but see that nothing is really being calculated chunk_sz = 10 for part in range(0, len(message), chunk_sz): yield producer.dataReceived(message[part:part + chunk_sz]) self.assertEqual(producer.deflated_size, 0) self.assertEqual(producer.inflated_size, 0) self.assertEqual(producer.crc32, 0) # stop and re-check producer.stopProducing() consumer.commit() yield producer.flush_decompressor() with open(consumer.filepath, "rb") as fh: self.assertEqual(fh.read(), message) hasher = content_hash_factory() hasher.update(data) self.assertEqual(producer.hash_object.content_hash(), hasher.content_hash()) magic_hasher = magic_hash_factory() magic_hasher.update(data) self.assertEqual(producer.magic_hash_object.content_hash()._magic_hash, magic_hasher.content_hash()._magic_hash) self.assertEqual(producer.inflated_size, len(data)) self.assertEqual(producer.crc32, crc32(data))
def test_get_content_on_share(self): """read a file on a share.""" data = "" deflated_data = zlib.compress(data) hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) deflated_size = len(deflated_data) def auth(client): """auth""" d = client.dummy_authenticate("open sesame") # need to put data to be able to retrieve it! d.addCallback( lambda r: client.put_content( self.share_modify, self.filerw, NO_CONTENT_HASH, hash_value, crc32_value, size, deflated_size, StringIO(deflated_data))) d.addCallback( lambda r: client.get_content( self.share_modify, self.filerw, EMPTY_HASH)) d.addCallbacks(client.test_done, client.test_fail) return self.callback_test(auth)
def test_getcontent_file_slow(self): """Get content from a file with very low BW and fail with timeout.""" data = os.urandom(300000) deflated_data = zlib.compress(data) hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) deflated_size = len(deflated_data) @defer.inlineCallbacks def auth(client): """Test.""" yield client.dummy_authenticate("open sesame") root = yield client.get_root() # make a file and put content in it mkfile_req = yield client.make_file(request.ROOT, root, "hola") yield client.put_content(request.ROOT, mkfile_req.new_id, NO_CONTENT_HASH, hash_value, crc32_value, size, deflated_size, StringIO(deflated_data)) # set the read limit, and get content client.factory.factory.readLimit = 1000 yield client.get_content(request.ROOT, mkfile_req.new_id, hash_value) d = self.callback_test(auth, add_default_callbacks=True, timeout=0.1) err = yield self.assertFailure(d, Exception) self.assertEqual(str(err), "timeout")
def test_getcontent_file(self, check_file_content=True): """Get the content from a file.""" data = os.urandom(300000) deflated_data = zlib.compress(data) hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) deflated_size = len(deflated_data) def check_file(req): if req.data != deflated_data: raise Exception("data does not match") def auth(client): d = client.dummy_authenticate("open sesame") d.addCallbacks(lambda _: client.get_root(), client.test_fail) d.addCallbacks( lambda root: client.make_file(request.ROOT, root, "hola"), client.test_fail) d.addCallback(self.save_req, 'req') d.addCallbacks( lambda mkfile_req: client. put_content(request.ROOT, mkfile_req.new_id, NO_CONTENT_HASH, hash_value, crc32_value, size, deflated_size, StringIO(deflated_data)), client.test_fail) d.addCallback(lambda _: client.get_content( request.ROOT, self._state.req.new_id, hash_value)) if check_file_content: d.addCallback(check_file) d.addCallbacks(client.test_done, client.test_fail) return self.callback_test(auth, timeout=5)
def test_put_content(self): """Write a file.""" data = "*" * 10000 deflated_data = zlib.compress(data) hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) deflated_size = len(deflated_data) def auth(client): """Authenticate and test.""" d = client.dummy_authenticate("open sesame") d.addCallback(lambda _: client.create_udf(u"~", u"myudf")) d.addCallback(self.save_req, "udf") # create a file with content d.addCallback(lambda r: client.make_file( self._state.udf.volume_id, self._state.udf.node_id, "foo")) # put content d.addCallback(lambda req: client.put_content( self._state.udf.volume_id, req.new_id, NO_CONTENT_HASH, hash_value, crc32_value, size, deflated_size, StringIO(deflated_data))) d.addCallbacks(client.test_done, client.test_fail) return self.callback_test(auth)
def test_upload(self): """Hiccup the network in the middle of an upload.""" data = os.urandom(1000) hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) self.patch(self.main.fs, 'open_file', lambda mdid: StringIO(data)) mdid, node_id = yield self._mkfile('hola') def worker(): """Async worker.""" self.aq.upload('', node_id, NO_CONTENT_HASH, hash_value, crc32_value, size, mdid) return self.hiccup() yield self.wait_for_nirvana() yield self.nuke_client_method( 'put_content_request', worker, lambda: self.connlost_deferred) self.assertInListenerEvents( 'AQ_UPLOAD_FINISHED', {'share_id': '', 'hash': hash_value, 'node_id': anUUID, 'new_generation': 2})
def _get_data(self, data_len=1000): """Get the hash, crc and size of a chunk of data.""" data = os.urandom(data_len) # not terribly compressible hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) return NoCloseStringIO(data), data, hash_value, crc32_value, size
def __init__(self, consumer, streaming): self.decompressor = zlib.decompressobj() self.hash_object = content_hash_factory() self.magic_hash_object = magic_hash_factory() self.crc32 = 0 self.inflated_size = 0 self.deflated_size = 0 self.consumer = consumer self.streaming = streaming
def test_putcontent_slow(self, num_files=1): """Test putting content to a file with very low bandwidth and fail with timeout. """ data = os.urandom(30000) deflated_data = zlib.compress(data) hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) deflated_size = len(deflated_data) def auth(client): def check_file(result): @transaction.atomic def _check_file(): content_blob = services.get_object_or_none(ContentBlob, hash=hash_value) if not content_blob: raise ValueError("content blob is not there") d = threads.deferToThread(_check_file) return d d = client.dummy_authenticate("open sesame") filename = 'hola_1' d.addCallbacks(lambda _: client.get_root(), client.test_fail) d.addCallbacks( lambda root: client.make_file(request.ROOT, root, filename), client.test_fail) def set_write_limit(r): client.factory.factory.writeLimit = 100 return r d.addCallback(set_write_limit) d.addCallbacks( lambda mkfile_req: client. put_content(request.ROOT, mkfile_req.new_id, NO_CONTENT_HASH, hash_value, crc32_value, size, deflated_size, StringIO(deflated_data)), client.test_fail) return d d1 = defer.Deferred() test_d = self.callback_test(auth, timeout=1) test_d.addCallbacks(d1.errback, lambda r: d1.callback(None)) return d1
def test_shutdown_upload(self): """Stop and restart the server.""" service = yield self.create_service() client = yield self.connect_client(service) root = yield client.get_root() mkfile_req = yield client.make_file(request.ROOT, root, "hola") # try to upload something that will fail when sending data empty_hash = content_hash_factory().content_hash() # lose the connection if something wrong try: yield client.put_content(request.ROOT, mkfile_req.new_id, empty_hash, "fake_hash", 1234, 1000, None) except Exception: client.transport.loseConnection() ujobs = self.usr0.get_uploadjobs(node_id=mkfile_req.new_id) self.assertEqual(ujobs, [])
def put(self, local, remote): """Put local file into remote file.""" try: node_id = self.get_id_from_filename(remote) except ValueError: parent_id = self.get_cwd_id() r = self.defer_from_thread(self.factory.current_protocol.make_file, self.volume, parent_id, remote.split("/")[-1]) node_id = r.new_id old_hash = self.get_hash(node_id) ho = content_hash_factory() zipper = zlib.compressobj() crc32_value = 0 size = 0 deflated_size = 0 temp_file_name = None with open(local) as fh: with tempfile.NamedTemporaryFile(mode='w', prefix='cmd_client-', delete=False) as dest: temp_file_name = dest.name while True: cont = fh.read(1024**2) if not cont: dest.write(zipper.flush()) deflated_size = dest.tell() break ho.update(cont) crc32_value = crc32(cont, crc32_value) size += len(cont) dest.write(zipper.compress(cont)) hash_value = ho.content_hash() try: self.defer_from_thread(self.factory.current_protocol.put_content, self.volume, node_id, old_hash, hash_value, crc32_value, size, deflated_size, open(temp_file_name, 'r')) finally: if os.path.exists(temp_file_name): os.unlink(temp_file_name)
def test_put_content_on_share_ro(self): """Write a file on a share thats read only.""" data = "*" * 100000 hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) def auth(client): """auth""" d = client.dummy_authenticate("open sesame") d.addCallback( lambda r: client.put_content( self.share, self.filero, NO_CONTENT_HASH, hash_value, crc32_value, size, StringIO(data))) d.addCallbacks(client.test_fail, lambda x: client.test_done()) return self.callback_test(auth)
def test_putcontent(self, num_files=1): """Test putting content to a file.""" data = os.urandom(300000) deflated_data = zlib.compress(data) hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) deflated_size = len(deflated_data) def auth(client): def check_file(result): @transaction.atomic def _check_file(): content_blob = services.get_object_or_none(ContentBlob, hash=hash_value) if not content_blob: raise ValueError("content blob is not there") d = threads.deferToThread(_check_file) return d d = client.dummy_authenticate("open sesame") filenames = iter('hola_%d' % i for i in range(num_files)) for i in range(num_files): d.addCallbacks(lambda _: client.get_root(), client.test_fail) d.addCallbacks( lambda root: client.make_file(request.ROOT, root, filenames.next()), client.test_fail) d.addCallbacks( lambda mkfile_req: client.put_content( request.ROOT, mkfile_req.new_id, NO_CONTENT_HASH, hash_value, crc32_value, size, deflated_size, StringIO(deflated_data)), client.test_fail) d.addCallback(check_file) d.addCallbacks(client.test_done, client.test_fail) return d return self.callback_test(auth, timeout=1)
def do_create_lots_of_files(self, suffix=''): """A helper that creates N files.""" # data for putcontent ho = content_hash_factory() hash_value = ho.content_hash() crc32_value = crc32("") deflated_content = zlib.compress("") deflated_size = len(deflated_content) mk = yield self.client.make_file(request.ROOT, self.root_id, "test_first" + suffix) yield self.client.put_content(request.ROOT, mk.new_id, NO_CONTENT_HASH, hash_value, crc32_value, 0, deflated_size, StringIO(deflated_content)) for i in range(self.N): mk = yield self.client.make_file(request.ROOT, self.root_id, "test_%03x%s" % (i, suffix)) yield self.client.put_content(request.ROOT, mk.new_id, NO_CONTENT_HASH, hash_value, crc32_value, 0, deflated_size, StringIO(deflated_content))
def test_mkfile_already_exists_content(self): """Create a file on a file that already exists and have content.""" data = "*" * 100 deflated_data = zlib.compress(data) hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) deflated_size = len(deflated_data) def auth(client): d = client.dummy_authenticate("open sesame") d.addCallback(lambda r: client.get_root()) d.addCallback(self.save_req, "root") d.addCallback(lambda r: client.make_file(request.ROOT, r, "hola")) d.addCallback(lambda req: client.put_content(request.ROOT, req.new_id, NO_CONTENT_HASH, hash_value, crc32_value, size, deflated_size, StringIO(deflated_data))) d.addCallback(lambda r: client.make_file(request.ROOT, self._state.root, "hola")) d.addCallbacks( lambda x: client.test_done("ok"), client.test_fail) return self.callback_test(auth)
# For further info, check http://launchpad.net/magicicada-server """Test User Defined Folder operations.""" import uuid import zlib from StringIO import StringIO from magicicadaprotocol import request from magicicadaprotocol.content_hash import content_hash_factory, crc32 from twisted.internet import reactor, defer from magicicada.filesync.errors import DoesNotExist from magicicada.server.testing.testcase import FactoryHelper, TestWithDatabase EMPTY_HASH = content_hash_factory().content_hash() NO_CONTENT_HASH = "" class TestManageUDF(TestWithDatabase): """Test udf creation and deletion.""" def test_create_udf_simple(self): """Create an UDF.""" def auth(client): """Authenticate and test.""" d = client.dummy_authenticate("open sesame") def check_result(req): """Check the request result.""" self.assertFalse(req.volume_id is None) self.assertFalse(req.node_id is None)