def test_putcontent_unlinked(self): """Try to put content in an unlinked file.""" empty_hash = content_hash_factory().content_hash() data = "*" size = 1 hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) def auth(client): # setup d = client.dummy_authenticate("open sesame") d.addCallback(lambda r: client.get_root()) # create file and remove it d.addCallback(lambda r: client.make_file(request.ROOT, r, "hola")) d.addCallback(lambda req: self._save_state("file", req.new_id)) d.addCallback( lambda _: client.unlink(request.ROOT, self._state.file)) # try to put content d.addCallback(lambda _: client.put_content( request.ROOT, self._state.file, empty_hash, hash_value, crc32_value, size, StringIO(data))) d.addCallbacks(client.test_fail, lambda x: client.test_done("ok")) return self.callback_test(auth)
def test_putcontent_unlinked(self): """Try to put content in an unlinked file.""" empty_hash = content_hash_factory().content_hash() data = "*" size = 1 hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) def auth(client): # setup d = client.dummy_authenticate("open sesame") d.addCallback(lambda r: client.get_root()) # create file and remove it d.addCallback(lambda r: client.make_file(request.ROOT, r, "hola")) d.addCallback(lambda req: self._save_state("file", req.new_id)) d.addCallback(lambda _: client.unlink(request.ROOT, self._state.file)) # try to put content d.addCallback(lambda _: client.put_content( request.ROOT, self._state.file, empty_hash, hash_value, crc32_value, size, StringIO(data))) d.addCallbacks(client.test_fail, lambda x: client.test_done("ok")) return self.callback_test(auth)
def test_getcontent_file_slow(self): """Get content from a file with very low BW and fail with timeout.""" data = os.urandom(300000) deflated_data = zlib.compress(data) hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) deflated_size = len(deflated_data) @defer.inlineCallbacks def auth(client): """Test.""" yield client.dummy_authenticate("open sesame") root = yield client.get_root() # make a file and put content in it mkfile_req = yield client.make_file(request.ROOT, root, "hola") yield client.put_content(request.ROOT, mkfile_req.new_id, NO_CONTENT_HASH, hash_value, crc32_value, size, deflated_size, StringIO(deflated_data)) # set the read limit, and get content client.factory.factory.readLimit = 1000 yield client.get_content(request.ROOT, mkfile_req.new_id, hash_value) d = self.callback_test(auth, add_default_callbacks=True, timeout=0.1) err = yield self.assertFailure(d, Exception) self.assertEqual(str(err), "timeout")
def test_put_content(self): """Write a file.""" data = "*" * 10000 deflated_data = zlib.compress(data) hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) deflated_size = len(deflated_data) def auth(client): """Authenticate and test.""" d = client.dummy_authenticate("open sesame") d.addCallback(lambda _: client.create_udf(u"~", u"myudf")) d.addCallback(self.save_req, "udf") # create a file with content d.addCallback(lambda r: client.make_file(self._state.udf.volume_id, self._state.udf.node_id, "foo")) # put content d.addCallback(lambda req: client.put_content( self._state.udf.volume_id, req.new_id, NO_CONTENT_HASH, hash_value, crc32_value, size, deflated_size, StringIO(deflated_data))) d.addCallbacks(client.test_done, client.test_fail) return self.callback_test(auth)
def test_getcontent_file_slow(self): """Get content from a file with very low BW and fail with timeout.""" data = os.urandom(300000) deflated_data = zlib.compress(data) hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) deflated_size = len(deflated_data) @defer.inlineCallbacks def auth(client): """Test.""" yield client.dummy_authenticate("open sesame") root = yield client.get_root() # make a file and put content in it mkfile_req = yield client.make_file(request.ROOT, root, "hola") yield client.put_content(request.ROOT, mkfile_req.new_id, NO_CONTENT_HASH, hash_value, crc32_value, size, deflated_size, StringIO(deflated_data)) # set the read limit, and get content client.factory.factory.readLimit = 1000 yield client.get_content(request.ROOT, mkfile_req.new_id, hash_value) d = self.callback_test(auth, add_default_callbacks=True, timeout=0.1) err = yield self.assertFailure(d, Exception) self.assertEqual(str(err), "timeout")
def test_proxy_producer_streaming(self): """Test ProxyHashingProducer.""" data = os.urandom(1024 * 10) message = zlib.compress(data) ds = diskstorage.DiskStorage(os.path.join(self.tmpdir, "testfile")) consumer = ds.put("somenode") producer = upload.ProxyHashingProducer(consumer, True) chunk_sz = 10 for part in xrange(0, len(message), chunk_sz): yield producer.dataReceived(message[part:part + chunk_sz]) producer.stopProducing() yield producer.flush_decompressor() with open(consumer.filepath, "rb") as fh: self.assertEqual(fh.read(), message) hasher = content_hash_factory() hasher.update(data) self.assertEqual(producer.hash_object.content_hash(), hasher.content_hash()) magic_hasher = magic_hash_factory() magic_hasher.update(data) self.assertEqual(producer.magic_hash_object.content_hash()._magic_hash, magic_hasher.content_hash()._magic_hash) self.assertEqual(producer.inflated_size, len(data)) self.assertEqual(producer.crc32, crc32(data))
def test_get_content_on_share(self): """read a file on a share.""" data = "" deflated_data = zlib.compress(data) hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) deflated_size = len(deflated_data) def auth(client): """auth""" d = client.dummy_authenticate("open sesame") # need to put data to be able to retrieve it! d.addCallback( lambda r: client.put_content( self.share_modify, self.filerw, NO_CONTENT_HASH, hash_value, crc32_value, size, deflated_size, StringIO(deflated_data))) d.addCallback( lambda r: client.get_content( self.share_modify, self.filerw, EMPTY_HASH)) d.addCallbacks(client.test_done, client.test_fail) return self.callback_test(auth)
def test_getcontent_file(self, check_file_content=True): """Get the content from a file.""" data = os.urandom(300000) deflated_data = zlib.compress(data) hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) deflated_size = len(deflated_data) def check_file(req): if req.data != deflated_data: raise Exception("data does not match") def auth(client): d = client.dummy_authenticate("open sesame") d.addCallbacks(lambda _: client.get_root(), client.test_fail) d.addCallbacks( lambda root: client.make_file(request.ROOT, root, "hola"), client.test_fail) d.addCallback(self.save_req, 'req') d.addCallbacks( lambda mkfile_req: client.put_content( request.ROOT, mkfile_req.new_id, NO_CONTENT_HASH, hash_value, crc32_value, size, deflated_size, StringIO(deflated_data)), client.test_fail) d.addCallback(lambda _: client.get_content( request.ROOT, self._state.req.new_id, hash_value)) if check_file_content: d.addCallback(check_file) d.addCallbacks(client.test_done, client.test_fail) return self.callback_test(auth, timeout=1.5)
def test_getcontent_file(self, check_file_content=True): """Get the content from a file.""" data = os.urandom(300000) deflated_data = zlib.compress(data) hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) deflated_size = len(deflated_data) def check_file(req): if req.data != deflated_data: raise Exception("data does not match") def auth(client): d = client.dummy_authenticate("open sesame") d.addCallbacks(lambda _: client.get_root(), client.test_fail) d.addCallbacks( lambda root: client.make_file(request.ROOT, root, "hola"), client.test_fail) d.addCallback(self.save_req, 'req') d.addCallbacks( lambda mkfile_req: client. put_content(request.ROOT, mkfile_req.new_id, NO_CONTENT_HASH, hash_value, crc32_value, size, deflated_size, StringIO(deflated_data)), client.test_fail) d.addCallback(lambda _: client.get_content( request.ROOT, self._state.req.new_id, hash_value)) if check_file_content: d.addCallback(check_file) d.addCallbacks(client.test_done, client.test_fail) return self.callback_test(auth, timeout=1.5)
def _hash(self, path): """Actually hashes a file.""" hasher = content_hash_factory() crc = 0 size = 0 try: initial_stat = stat_path(path) with open_file(path, 'rb') as fh: while True: # stop hashing if path_to_cancel = path or _stopped is True with self.mutex: path_to_cancel = self._should_cancel if path_to_cancel == path or self._stopped: raise StopHashing('hashing of %r was cancelled' % path) cont = fh.read(self.chunk_size) if not cont: break hasher.update(cont) crc = crc32(cont, crc) size += len(cont) finally: with self.mutex: self._should_cancel = None return hasher.content_hash(), crc, size, initial_stat
def test_shutdown_while_hashing(self): """Test that the HashQueue is shutdown ASAP while it's hashing.""" # create large data in order to test testinfo = os.urandom(500000) hasher = content_hash_factory() hasher.hash_object.update(testinfo) testfile = os.path.join(self.test_dir, "testfile") # send what to hash with open_file(testfile, "wb") as fh: fh.write(testinfo) class Helper(object): """Helper class.""" def push(self, event, **kwargs): """Callback.""" receiver = Helper() hq = hash_queue.HashQueue(receiver) # read in small chunks, so we have more iterations hq.hasher.chunk_size = 2**10 hq.insert(testfile, "mdid") time.sleep(0.1) hq.shutdown() # block until the hash is stopped and the queue is empty # a shutdown clears the queue hq._queue.join() self.assertFalse(hq.hasher.hashing) self.assertTrue(hq.hasher._stopped) #self.assertFalse(hq.hasher.isAlive()) self.assertTrue(hq._queue.empty())
def test_upload(self): """Hiccup the network in the middle of an upload.""" data = os.urandom(1000) hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) self.patch(self.main.fs, 'open_file', lambda mdid: StringIO(data)) mdid, node_id = yield self._mkfile('hola') def worker(): """Async worker.""" self.aq.upload('', node_id, NO_CONTENT_HASH, hash_value, crc32_value, size, mdid) return self.hiccup() d = self.wait_for_nirvana() d.addCallback(lambda _: self.nuke_client_method( 'put_content_request', worker, lambda: self.connlost_deferred)) self.assertInQ(d, lambda: ('AQ_UPLOAD_FINISHED', {'share_id': '', 'hash': hash_value, 'node_id': anUUID, 'new_generation': 2L})) yield d
def make_file_with_content(root, name, mimetype=None): """Make a file with content.""" hash = content_hash.content_hash_factory() hash.update(str(uuid.uuid4())) hashstr = hash.content_hash() root.make_file_with_content(name, hashstr, 10, 100, 100, uuid.uuid4(), mimetype=mimetype)
def _get_data(self, data_len=1000): """Get the hash, crc and size of a chunk of data.""" data = os.urandom(data_len) # not terribly compressible hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) return NoCloseStringIO(data), data, hash_value, crc32_value, size
def test_putcontent_slow(self, num_files=1): """Test putting content to a file with very low bandwidth and fail with timeout. """ data = os.urandom(30000) deflated_data = zlib.compress(data) hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) deflated_size = len(deflated_data) def auth(client): def check_file(result): def _check_file(): filesync_tm.begin() try: store = get_filesync_store() content_blob = store.get(ContentBlob, hash_value) if not content_blob: raise ValueError("content blob is not there") finally: filesync_tm.abort() d = threads.deferToThread(_check_file) return d d = client.dummy_authenticate("open sesame") filename = "hola_1" d.addCallbacks(lambda _: client.get_root(), client.test_fail) d.addCallbacks(lambda root: client.make_file(request.ROOT, root, filename), client.test_fail) def set_write_limit(r): client.factory.factory.writeLimit = 100 return r d.addCallback(set_write_limit) d.addCallbacks( lambda mkfile_req: client.put_content( request.ROOT, mkfile_req.new_id, NO_CONTENT_HASH, hash_value, crc32_value, size, deflated_size, StringIO(deflated_data), ), client.test_fail, ) return d d1 = defer.Deferred() test_d = self.callback_test(auth, timeout=1) test_d.addCallbacks(d1.errback, lambda r: d1.callback(None)) return d1
def _get_data(self, data_len=1000): """Get the hash, crc and size of a chunk of data.""" data = os.urandom(data_len) # not terribly compressible hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) return NoCloseStringIO(data), data, hash_value, crc32_value, size
def __init__(self, consumer, streaming): self.decompressor = zlib.decompressobj() self.hash_object = content_hash_factory() self.magic_hash_object = magic_hash_factory() self.crc32 = 0 self.inflated_size = 0 self.deflated_size = 0 self.consumer = consumer self.streaming = streaming
def test_unique(self): """The hasher should return in order.""" # calculate what we should receive should_be = [] for i in range(10): hasher = content_hash_factory() text = "supercalifragilistico"+str(i) hasher.hash_object.update(text) tfile = os.path.join(self.test_dir, "tfile"+str(i)) with open_file(tfile, "wb") as fh: fh.write("supercalifragilistico"+str(i)) d = dict(path=tfile, hash=hasher.content_hash(), crc32=crc32(text), size=len(text), stat=stat_path(tfile)) should_be.append(("HQ_HASH_NEW", d)) d = defer.Deferred() class Helper(object): """Helper class.""" # class-closure, cannot use self, pylint: disable-msg=E0213 def __init__(innerself): innerself.store = [] def push(innerself, event, **kwargs): """Callback.""" innerself.store.append((event, kwargs)) if len(innerself.store) == 10: if innerself.store == should_be: d.callback(True) else: d.errback(Exception("are different!")) receiver = Helper() hq = hash_queue.HashQueue(receiver) self.addCleanup(hq.shutdown) # stop the hasher so we can test the unique items in the queue hq.hasher.stop() self.log.debug('Hasher stopped (forced)') # allow the hasher to fully stop time.sleep(0.1) # create a new hasher just like the HashQueue creates it hq.hasher = hash_queue._Hasher(hq._queue, hq._end_mark, receiver) hq.hasher.setDaemon(True) # send to hash twice for i in range(10): tfile = os.path.join(self.test_dir, "tfile"+str(i)) hq.insert(tfile, "mdid") hq.insert(tfile, "mdid") # start the hasher self.log.debug('Hasher started (forced)') hq.hasher.start() # insert the last item to check the uniqueness in the queue while # the hasher is running for i in range(9, 10): tfile = os.path.join(self.test_dir, "tfile"+str(i)) hq.insert(tfile, "mdid") return d
def make_file_with_content(root, name, mimetype=None): """Make a file with content.""" hash = content_hash.content_hash_factory() hash.update(str(uuid.uuid4())) hashstr = hash.content_hash() root.make_file_with_content(name, hashstr, 10, 100, 100, uuid.uuid4(), mimetype=mimetype)
def test_putcontent_slow(self, num_files=1): """Test putting content to a file with very low bandwidth and fail with timeout. """ data = os.urandom(30000) deflated_data = zlib.compress(data) hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) deflated_size = len(deflated_data) def auth(client): def check_file(result): def _check_file(): storage_tm.begin() try: store = get_storage_store() content_blob = store.get(model.ContentBlob, hash_value) if not content_blob: raise ValueError("content blob is not there") finally: storage_tm.abort() d = threads.deferToThread(_check_file) return d d = client.dummy_authenticate("open sesame") filename = 'hola_1' d.addCallbacks(lambda _: client.get_root(), client.test_fail) d.addCallbacks( lambda root: client.make_file(request.ROOT, root, filename), client.test_fail) def set_write_limit(r): client.factory.factory.writeLimit = 100 return r d.addCallback(set_write_limit) d.addCallbacks( lambda mkfile_req: client. put_content(request.ROOT, mkfile_req.new_id, NO_CONTENT_HASH, hash_value, crc32_value, size, deflated_size, StringIO(deflated_data)), client.test_fail) return d d1 = defer.Deferred() test_d = self.callback_test(auth, timeout=1) test_d.addCallbacks(d1.errback, lambda r: d1.callback(None)) return d1
def check_info((event, kwargs)): """check the info pushed by the hasher""" # pylint: disable-msg=W0612 self.assertEqual(event, "HQ_HASH_NEW") # calculate what we should receive realh = content_hash_factory() realh.hash_object.update("foobar") should_be = realh.content_hash() curr_stat = stat_path(testfile) self.assertEquals(should_be, kwargs['hash']) for attr in ('st_mode', 'st_ino', 'st_dev', 'st_nlink', 'st_uid', 'st_gid', 'st_size', 'st_ctime', 'st_mtime'): self.assertEquals(getattr(curr_stat, attr), getattr(kwargs['stat'], attr))
def test_putcontent(self, num_files=1): """Test putting content to a file.""" data = os.urandom(300000) deflated_data = zlib.compress(data) hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) deflated_size = len(deflated_data) def auth(client): def check_file(result): def _check_file(): filesync_tm.begin() try: store = get_filesync_store() content_blob = store.get(ContentBlob, hash_value) if not content_blob: raise ValueError("content blob is not there") finally: filesync_tm.abort() d = threads.deferToThread(_check_file) return d d = client.dummy_authenticate("open sesame") filenames = iter("hola_%d" % i for i in xrange(num_files)) for i in range(num_files): d.addCallbacks(lambda _: client.get_root(), client.test_fail) d.addCallbacks(lambda root: client.make_file(request.ROOT, root, filenames.next()), client.test_fail) d.addCallbacks( lambda mkfile_req: client.put_content( request.ROOT, mkfile_req.new_id, NO_CONTENT_HASH, hash_value, crc32_value, size, deflated_size, StringIO(deflated_data), ), client.test_fail, ) d.addCallback(check_file) d.addCallbacks(client.test_done, client.test_fail) return d return self.callback_test(auth, timeout=1)
def test_order(self): """The hasher should return in order.""" # calculate what we should receive should_be = [] for i in range(10): hasher = content_hash_factory() text = "supercalifragilistico"+str(i) hasher.hash_object.update(text) tfile = os.path.join(self.test_dir, "tfile"+str(i)) with open_file(tfile, "wb") as fh: fh.write("supercalifragilistico"+str(i)) d = dict(path=tfile, hash=hasher.content_hash(), crc32=crc32(text), size=len(text), stat=stat_path(tfile)) should_be.append(("HQ_HASH_NEW", d)) # create the hasher mark = object() queue = hash_queue.UniqueQueue() d = defer.Deferred() class Helper(object): """Helper class.""" # class-closure, cannot use self, pylint: disable-msg=E0213 def __init__(innerself): innerself.store = [] def push(innerself, event, **kwargs): """Callback.""" innerself.store.append((event, kwargs)) if len(innerself.store) == 10: hasher.stop() hasher.join(timeout=5) if innerself.store == should_be: d.callback(True) else: d.errback(Exception("are different!")) receiver = Helper() hasher = hash_queue._Hasher(queue, mark, receiver) # send what to hash for i in range(10): tfile = os.path.join(self.test_dir, "tfile"+str(i)) item = ((tfile, "mdid"), FAKE_TIMESTAMP) queue.put(item) # start the hasher after putting the work items hasher.start() return d
def put(self, local, remote): """Put local file into remote file.""" try: node_id = self.get_id_from_filename(remote) except ValueError: parent_id = self.get_cwd_id() r = self.defer_from_thread( self.factory.current_protocol.make_file, self.volume, parent_id, remote.split("/")[-1] ) node_id = r.new_id old_hash = self.get_hash(node_id) ho = content_hash_factory() zipper = zlib.compressobj() crc32_value = 0 size = 0 deflated_size = 0 temp_file_name = None with open(local) as fh: with tempfile.NamedTemporaryFile(mode="w", prefix="cmd_client-", delete=False) as dest: temp_file_name = dest.name while True: cont = fh.read(1024 ** 2) if not cont: dest.write(zipper.flush()) deflated_size = dest.tell() break ho.update(cont) crc32_value = crc32(cont, crc32_value) size += len(cont) dest.write(zipper.compress(cont)) hash_value = ho.content_hash() try: self.defer_from_thread( self.factory.current_protocol.put_content, self.volume, node_id, old_hash, hash_value, crc32_value, size, deflated_size, open(temp_file_name, "r"), ) finally: if os.path.exists(temp_file_name): os.unlink(temp_file_name)
def test_shutdown_upload(self): """Stop and restart the server.""" service = yield self.create_service() client = yield self.connect_client(service) root = yield client.get_root() mkfile_req = yield client.make_file(request.ROOT, root, "hola") # try to upload something that will fail when sending data empty_hash = content_hash_factory().content_hash() # lose the connection if something wrong try: yield client.put_content(request.ROOT, mkfile_req.new_id, empty_hash, "fake_hash", 1234, 1000, None) except: client.transport.loseConnection() ujobs = self.usr0.get_uploadjobs(node_id=mkfile_req.new_id) self.assertEqual(ujobs, [])
def test_putcontent(self, num_files=1): """Test putting content to a file.""" data = os.urandom(300000) deflated_data = zlib.compress(data) hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) deflated_size = len(deflated_data) def auth(client): def check_file(result): def _check_file(): storage_tm.begin() try: store = get_storage_store() content_blob = store.get(model.ContentBlob, hash_value) if not content_blob: raise ValueError("content blob is not there") finally: storage_tm.abort() d = threads.deferToThread(_check_file) return d d = client.dummy_authenticate("open sesame") filenames = iter('hola_%d' % i for i in xrange(num_files)) for i in range(num_files): d.addCallbacks(lambda _: client.get_root(), client.test_fail) d.addCallbacks( lambda root: client.make_file(request.ROOT, root, filenames.next()), client.test_fail) d.addCallbacks( lambda mkfile_req: client.put_content( request.ROOT, mkfile_req.new_id, NO_CONTENT_HASH, hash_value, crc32_value, size, deflated_size, StringIO(deflated_data)), client.test_fail) d.addCallback(check_file) d.addCallbacks(client.test_done, client.test_fail) return d return self.callback_test(auth, timeout=1)
def test_put_content_on_share_ro(self): """Write a file on a share thats read only.""" data = "*" * 100000 hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) def auth(client): """auth""" d = client.dummy_authenticate("open sesame") d.addCallback( lambda r: client.put_content( self.share, self.filero, NO_CONTENT_HASH, hash_value, crc32_value, size, StringIO(data))) d.addCallbacks(client.test_fail, lambda x: client.test_done()) return self.callback_test(auth)
def put(self, local, remote): """Put local file into remote file.""" try: node_id = self.get_id_from_filename(remote) except ValueError: parent_id = self.get_cwd_id() r = self.defer_from_thread( self.factory.current_protocol.make_file, self.volume, parent_id, remote.split("/")[-1]) node_id = r.new_id old_hash = self.get_hash(node_id) ho = content_hash_factory() zipper = zlib.compressobj() crc32_value = 0 size = 0 deflated_size = 0 temp_file_name = None with open(local) as fh: with tempfile.NamedTemporaryFile(mode='w', prefix='cmd_client-', delete=False) as dest: temp_file_name = dest.name while True: cont = fh.read(1024 ** 2) if not cont: dest.write(zipper.flush()) deflated_size = dest.tell() break ho.update(cont) crc32_value = crc32(cont, crc32_value) size += len(cont) dest.write(zipper.compress(cont)) hash_value = ho.content_hash() try: self.defer_from_thread( self.factory.current_protocol.put_content, self.volume, node_id, old_hash, hash_value, crc32_value, size, deflated_size, open(temp_file_name, 'r')) finally: if os.path.exists(temp_file_name): os.unlink(temp_file_name)
def do_fileops(bench, base, sufix): """Do a bunch of file operations for a directory.""" with bench("makefile_" + sufix): file = base.make_file(unicode(uuid.uuid4())) with bench("makedir_" + sufix): dir = base.make_subdirectory(unicode(uuid.uuid4())) with bench("delfile_" + sufix): file.delete() with bench("deldir_" + sufix): dir.delete() with bench("5050_flat_" + sufix): for i in range(50): base.make_file(unicode(uuid.uuid4())) base.make_subdirectory(unicode(uuid.uuid4())) with bench("getchildren_" + sufix): base.get_children() first = dir = base.make_subdirectory(unicode(uuid.uuid4())) with bench("5050_nested_" + sufix): for i in range(50): dir.make_file(unicode(uuid.uuid4())) dir = dir.make_subdirectory(unicode(uuid.uuid4())) first.make_file(unicode(uuid.uuid4())) with bench("getchildren_sparse_" + sufix): first.get_children() hash = content_hash.content_hash_factory() hash.update(str(uuid.uuid4())) hashstr = hash.content_hash() storage_key = uuid.uuid4() with bench("make_with_content_" + sufix): base.make_file_with_content(unicode(uuid.uuid4()), hashstr, 100, 100, 12, storage_key) with bench("make_with_content_repeated_" + sufix): base.make_file_with_content(unicode(uuid.uuid4()), hashstr, 100, 100, 12, storage_key)
def test_shutdown_upload(self): """Stop and restart the server.""" # create a server service = StorageServerService(0, "localhost", self.s4_conn.getHost().port, False, s4.AWS_DEFAULT_ACCESS_KEY_ID, s4.AWS_DEFAULT_SECRET_ACCESS_KEY, auth_provider_class=DummyAuthProvider, heartbeat_interval=0) yield service.startService() # create a user, connect a client usr0 = make_storage_user(0, u"dummy", u"", 2**20) d = defer.Deferred() f = TestClientFactory() f.connected = d.callback reactor.connectTCP("localhost", service.port, f) client = yield d # auth, get root, create a file yield client.dummy_authenticate("open sesame") root = yield client.get_root() mkfile_req = yield client.make_file(request.ROOT, root, "hola") # try to upload something that will fail when sending data empty_hash = content_hash_factory().content_hash() # lose the connection if something wrong try: yield client.put_content(request.ROOT, mkfile_req.new_id, empty_hash, "fake_hash", 1234, 1000, None) except: client.transport.loseConnection() # see that the server has not protocols alive yield service.factory.wait_for_shutdown() ujobs = usr0.get_uploadjobs(node_id=mkfile_req.new_id) self.assertEqual(ujobs, []) yield service.stopService()
def test_large_content(self): """The hasher works ok for a lot of info.""" # calculate what we should receive testinfo = "".join(chr(random.randint(0, 255)) for i in range(100000)) hasher = content_hash_factory() hasher.hash_object.update(testinfo) testfile = os.path.join(self.test_dir, "testfile") testhash = hasher.content_hash() # create the hasher mark = object() queue = hash_queue.UniqueQueue() d = defer.Deferred() class Helper(object): """Helper class.""" def push(self, event, path, hash, crc32, size, stat): """callback""" hasher.stop() hasher.join(timeout=5) if event != "HQ_HASH_NEW": d.errback(Exception("envent is not HQ_HASH_NEW")) elif path != testfile: d.errback(Exception("path is not the original one")) elif hash != testhash: d.errback(Exception("the hashes are different!")) else: d.callback(True) receiver = Helper() hasher = hash_queue._Hasher(queue, mark, receiver) # send what to hash with open_file(testfile, "wb") as fh: fh.write(testinfo) item = ((testfile, "mdid"), FAKE_TIMESTAMP) queue.put(item) # start the hasher after putting the work items hasher.start() return d
def test_putcontent(self): """Test putting content to a file.""" data = "*" * 100000 deflated_data = zlib.compress(data) hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) deflated_size = len(deflated_data) def auth(client): def check_file(result): def _check_file(): try: content_blob = self.usr0.volume().get_content( hash_value) except errors.DoesNotExist: raise ValueError("content blob is not there") assert self.s4_site.resource.buckets["test"] \ .bucket_children[str(content_blob.storage_key)] \ .contents == deflated_data d = threads.deferToThread(_check_file) return d d = client.dummy_authenticate("open sesame") d.addCallbacks(lambda _: client.get_root(), client.test_fail) d.addCallbacks( lambda root: client.make_file(request.ROOT, root, "hola"), client.test_fail) d.addCallbacks( lambda mkfile_req: client.put_content( request.ROOT, mkfile_req.new_id, NO_CONTENT_HASH, hash_value, crc32_value, size, deflated_size, StringIO(deflated_data)), client.test_fail) d.addCallback(check_file) d.addCallbacks(client.test_done, client.test_fail) return self.callback_test(auth)
def test_order(self): """The hasher should return in order.""" # calculate what we should receive should_be = [] for i in range(10): hasher = content_hash_factory() text = "supercalifragilistico"+str(i) hasher.hash_object.update(text) tfile = os.path.join(self.test_dir, "tfile"+str(i)) with open_file(tfile, "wb") as fh: fh.write("supercalifragilistico"+str(i)) d = dict(path=tfile, hash=hasher.content_hash(), crc32=crc32(text), size=len(text), stat=stat_path(tfile)) should_be.append(("HQ_HASH_NEW", d)) d = defer.Deferred() class Helper(object): """Helper class.""" # class-closure, cannot use self, pylint: disable-msg=E0213 def __init__(innerself): innerself.store = [] def push(innerself, event, **kwargs): """Callback.""" innerself.store.append((event, kwargs)) if len(innerself.store) == 10: if innerself.store[:-1] == should_be[:-1]: d.callback(True) else: d.errback(Exception("are different! ")) receiver = Helper() hq = hash_queue.HashQueue(receiver) self.addCleanup(hq.shutdown) # send what to hash for i in range(10): tfile = os.path.join(self.test_dir, "tfile"+str(i)) hq.insert(tfile, "mdid") return d
def test_shutdown_upload(self): """Stop and restart the server.""" # create a server service = StorageServerService( 0, "localhost", self.s4_conn.getHost().port, False, s4.AWS_DEFAULT_ACCESS_KEY_ID, s4.AWS_DEFAULT_SECRET_ACCESS_KEY, auth_provider_class=DummyAuthProvider, heartbeat_interval=0) yield service.startService() # create a user, connect a client usr0 = make_storage_user(0, u"dummy", u"", 2 ** 20) d = defer.Deferred() f = TestClientFactory() f.connected = d.callback reactor.connectTCP("localhost", service.port, f) client = yield d # auth, get root, create a file yield client.dummy_authenticate("open sesame") root = yield client.get_root() mkfile_req = yield client.make_file(request.ROOT, root, "hola") # try to upload something that will fail when sending data empty_hash = content_hash_factory().content_hash() # lose the connection if something wrong try: yield client.put_content(request.ROOT, mkfile_req.new_id, empty_hash, "fake_hash", 1234, 1000, None) except: client.transport.loseConnection() # see that the server has not protocols alive yield service.factory.wait_for_shutdown() ujobs = usr0.get_uploadjobs(node_id=mkfile_req.new_id) self.assertEqual(ujobs, []) yield service.stopService()
def do_create_lots_of_files(self, suffix=''): """A helper that creates N files.""" # data for putcontent ho = content_hash_factory() hash_value = ho.content_hash() crc32_value = crc32("") deflated_content = zlib.compress("") deflated_size = len(deflated_content) mk = yield self.client.make_file(request.ROOT, self.root_id, "test_first" + suffix) yield self.client.put_content( request.ROOT, mk.new_id, NO_CONTENT_HASH, hash_value, crc32_value, 0, deflated_size, StringIO(deflated_content)) for i in xrange(self.N): mk = yield self.client.make_file(request.ROOT, self.root_id, "test_%03x%s" % (i, suffix)) yield self.client.put_content(request.ROOT, mk.new_id, NO_CONTENT_HASH, hash_value, crc32_value, 0, deflated_size, StringIO(deflated_content))
def do_create_lots_of_files(self, suffix=''): """A helper that creates N files.""" # data for putcontent ho = content_hash_factory() hash_value = ho.content_hash() crc32_value = crc32("") deflated_content = zlib.compress("") deflated_size = len(deflated_content) mk = yield self.client.make_file(request.ROOT, self.root_id, "test_first" + suffix) yield self.client.put_content(request.ROOT, mk.new_id, NO_CONTENT_HASH, hash_value, crc32_value, 0, deflated_size, StringIO(deflated_content)) for i in xrange(self.N): mk = yield self.client.make_file(request.ROOT, self.root_id, "test_%03x%s" % (i, suffix)) yield self.client.put_content(request.ROOT, mk.new_id, NO_CONTENT_HASH, hash_value, crc32_value, 0, deflated_size, StringIO(deflated_content))
def test_mkfile_already_exists_content(self): """Create a file on a file that already exists and have content.""" data = "*" * 100 deflated_data = zlib.compress(data) hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) deflated_size = len(deflated_data) def auth(client): d = client.dummy_authenticate("open sesame") d.addCallback(lambda r: client.get_root()) d.addCallback(self.save_req, "root") d.addCallback(lambda r: client.make_file(request.ROOT, r, "hola")) d.addCallback(lambda req: client.put_content(request.ROOT, req.new_id, NO_CONTENT_HASH, hash_value, crc32_value, size, deflated_size, StringIO(deflated_data))) d.addCallback(lambda r: client.make_file(request.ROOT, self._state.root, "hola")) d.addCallbacks( lambda x: client.test_done("ok"), client.test_fail) return self.callback_test(auth)
def test_mkfile_already_exists_content(self): """Create a file on a file that already exists and have content.""" data = "*" * 100 deflated_data = zlib.compress(data) hash_object = content_hash_factory() hash_object.update(data) hash_value = hash_object.content_hash() crc32_value = crc32(data) size = len(data) deflated_size = len(deflated_data) def auth(client): d = client.dummy_authenticate("open sesame") d.addCallback(lambda r: client.get_root()) d.addCallback(self.save_req, "root") d.addCallback(lambda r: client.make_file(request.ROOT, r, "hola")) d.addCallback(lambda req: client.put_content( request.ROOT, req.new_id, NO_CONTENT_HASH, hash_value, crc32_value, size, deflated_size, StringIO(deflated_data))) d.addCallback(lambda r: client.make_file(request.ROOT, self._state. root, "hola")) d.addCallbacks(lambda x: client.test_done("ok"), client.test_fail) return self.callback_test(auth)
| When a system has autenticated a user: | controller = SystemServices() | user = controller.get_user(user_id) | | Later in the code: | | | user.share(share_id).dir(dir_id).make_file(u"file_name") | | In this example, no database access is performed until the call | to make_file. This way, this would translate to the following | steps within a single transaction: | * get the user and verify it's Live | * get the share_id and verify it's Live and accepted | * get the directory from the share and verify that it's Live | * if the user has write access to the share, make the file and | return it | | The benefit if this method is that methods like make_file make_subdir are | only methods of a directory node and we don't end up with a user class with | lots of functions that behave differently based on the parameters passed | """ from ubuntuone.storageprotocol.content_hash import content_hash_factory from backends.filesync.data.dbmanager import get_storage_store # NOQA from backends.filesync.data.dbmanager import storage_tm # NOQA EMPTY_CONTENT_HASH = content_hash_factory().content_hash()
def test_content_hash_factory(self): """Check the factory for the normal content hash.""" o = content_hash_factory() self.assertTrue(isinstance(o, SHA1ContentHash))
| When a system has autenticated a user: | controller = SystemServices() | user = controller.get_user(user_id) | | Later in the code: | | | user.share(share_id).dir(dir_id).make_file(u"file_name") | | In this example, no database access is performed until the call | to make_file. This way, this would translate to the following | steps within a single transaction: | * get the user and verify it's Live | * get the share_id and verify it's Live and accepted | * get the directory from the share and verify that it's Live | * if the user has write access to the share, make the file and | return it | | The benefit if this method is that methods like make_file make_subdir are | only methods of a directory node and we don't end up with a user class with | lots of functions that behave differently based on the parameters passed | """ from ubuntuone.storageprotocol.content_hash import content_hash_factory from backends.filesync.data.dbmanager import get_storage_store # NOQA from backends.filesync.data.dbmanager import storage_tm # NOQA EMPTY_CONTENT_HASH = content_hash_factory().content_hash()
"""Test sharing operations.""" import uuid import zlib from StringIO import StringIO from twisted.internet import reactor, defer from ubuntuone.storageprotocol import errors as protocol_errors, request from ubuntuone.storageprotocol.content_hash import content_hash_factory, crc32 from magicicada.filesync import errors from magicicada.filesync.models import STATUS_LIVE, Share from magicicada.server.testing.testcase import FactoryHelper, TestWithDatabase EMPTY_HASH = content_hash_factory().content_hash() NO_CONTENT_HASH = "" class TestCreateShare(TestWithDatabase): """Test crate_share command.""" def test_delete_share(self): """Test deletion of an offered by me share.""" @defer.inlineCallbacks def do_delete(client): """Callback to do delete.""" # authenticate and create a root to share yield client.dummy_authenticate("open sesame") root = yield client.get_root()