def file_length(digest, file_cacher=None, file_lengther=None): """Compute the length of the file identified by digest. digest (string): the digest of the file. file_cacher (FileCacher): the cacher to use, or None. file_lengther (type): a File-like object that tell the dimension of the input (see example above for how to write one). return (int): the length of the tile. """ if file_cacher is None: file_cacher = FileCacher() if file_lengther is None: file_lengther = FileLengther lengther = file_lengther() file_cacher.get_file_to_fobj(digest, lengther) return lengther.tell()
class TestFileCacher(TestService): """Service that performs automatically some tests for the FileCacher service. """ def __init__(self, shard): logger.initialize(ServiceCoord("TestFileCacher", shard)) TestService.__init__(self, shard, custom_logger=logger) # Assume we store the cache in "./cache/fs-cache-TestFileCacher-0/" self.cache_base_path = os.path.join(config.cache_dir, "fs-cache-TestFileCacher-0") self.cache_path = None self.content = None self.fake_content = None self.digest = None self.file_obj = None self.file_cacher = FileCacher(self) #self.file_cacher = FileCacher(self, path="fs-storage") def prepare(self): """Initialization for the test code - make sure that the cache is empty before testing. """ logger.info("Please delete directory %s before." % self.cache_base_path) ### TEST 000 ### def test_000(self): """Send a ~100B random binary file to the storage through FileCacher as a file-like object. FC should cache the content locally. """ self.size = 100 self.content = "".join(chr(random.randint(0, 255)) for unused_i in xrange(self.size)) logger.info(" I am sending the ~100B binary file to FileCacher") try: data = self.file_cacher.put_file_from_fobj(StringIO(self.content), u"Test #000") except Exception as error: self.test_end(False, "Error received: %r." % error) return if not os.path.exists(os.path.join(self.cache_base_path, data)): self.test_end(False, "File not stored in local cache.") elif open(os.path.join(self.cache_base_path, data), "rb").read() != \ self.content: self.test_end(False, "Local cache's content differ " "from original file.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data self.test_end(True, "Data sent and cached without error.") ### TEST 001 ### def test_001(self): """Retrieve the file. """ logger.info(" I am retrieving the ~100B binary file from FileCacher") self.fake_content = "Fake content.\n" with open(self.cache_path, "wb") as cached_file: cached_file.write(self.fake_content) try: data = self.file_cacher.get_file(self.digest) except Exception as error: self.test_end(False, "Error received: %r." % error) return received = data.read() data.close() if received != self.fake_content: if received == self.content: self.test_end(False, "Did not use the cache even if it could.") else: self.test_end(False, "Content differ.") else: self.test_end(True, "Data object received correctly.") ### TEST 002 ### def test_002(self): """Check the size of the file. """ logger.info(" I am checking the size of the ~100B binary file") try: size = self.file_cacher.get_size(self.digest) except Exception as error: self.test_end(False, "Error received: %r." % error) return if size == self.size: self.test_end(True, "The size is correct.") else: self.test_end(False, "The size is wrong: %d instead of %d" % (size, self.size)) ### TEST 003 ### def test_003(self): """Get file from FileCacher. """ logger.info(" I am retrieving the file from FileCacher " + "after deleting the cache.") os.unlink(self.cache_path) try: data = self.file_cacher.get_file(self.digest) except Exception as error: self.test_end(False, "Error received: %r." % error) return received = data.read() data.close() if received != self.content: self.test_end(False, "Content differ.") elif not os.path.exists(self.cache_path): self.test_end(False, "File not stored in local cache.") elif open(self.cache_path).read() != self.content: self.test_end(False, "Local cache's content differ " + "from original file.") else: self.test_end(True, "Content object received " + "and cached correctly.") ### TEST 004 ### def test_004(self): """Delete the file through FS and tries to get it again through FC. """ logger.info(" I am deleting the file from FileCacher.") try: self.file_cacher.delete(digest=self.digest) except Exception as error: self.test_end(False, "Error received: %s." % error) return else: logger.info(" File deleted correctly.") logger.info(" I am getting the file from FileCacher.") try: self.file_cacher.get_file(self.digest) except Exception as error: self.test_end(True, "Correctly received an error: %r." % error) else: self.test_end(False, "Did not receive error.") ### TEST 005 ### def test_005(self): """Get unexisting file from FileCacher. """ logger.info(" I am retrieving an unexisting file from FileCacher.") try: self.file_cacher.get_file(self.digest) except Exception as error: self.test_end(True, "Correctly received an error: %r." % error) else: self.test_end(False, "Did not receive error.") ### TEST 006 ### def test_006(self): """Send a ~100B random binary file to the storage through FileCacher as a string. FC should cache the content locally. """ self.content = "".join(chr(random.randint(0, 255)) for unused_i in xrange(100)) logger.info(" I am sending the ~100B binary file to FileCacher") try: data = self.file_cacher.put_file_content(self.content, u"Test #005") except Exception as error: self.test_end(False, "Error received: %r." % error) return if not os.path.exists(os.path.join(self.cache_base_path, data)): self.test_end(False, "File not stored in local cache.") elif open(os.path.join(self.cache_base_path, data), "rb").read() != self.content: self.test_end(False, "Local cache's content differ " "from original file.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data self.test_end(True, "Data sent and cached without error.") ### TEST 007 ### def test_007(self): """Retrieve the file as a string. """ logger.info(" I am retrieving the ~100B binary file from FileCacher " "using get_file_to_string()") self.fake_content = "Fake content.\n" with open(self.cache_path, "wb") as cached_file: cached_file.write(self.fake_content) try: data = self.file_cacher.get_file_content(self.digest) except Exception as error: self.test_end(False, "Error received: %r." % error) return if data != self.fake_content: if data == self.content: self.test_end(False, "Did not use the cache even if it could.") else: self.test_end(False, "Content differ.") else: self.test_end(True, "Data received correctly.") ### TEST 008 ### def test_008(self): """Put a ~100MB file into the storage (using a specially crafted file-like object). """ logger.info(" I am sending the ~100MB binary file to FileCacher") rand_file = RandomFile(100000000) try: data = self.file_cacher.put_file_from_fobj(rand_file, u"Test #007") except Exception as error: self.test_end(False, "Error received: %r." % error) return if rand_file.dim != 0: self.test_end(False, "The input file wasn't read completely.") my_digest = rand_file.digest rand_file.close() if not os.path.exists(os.path.join(self.cache_base_path, data)): self.test_end(False, "File not stored in local cache.") elif my_digest != data: self.test_end(False, "File received with wrong hash.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data self.test_end(True, "Data sent and cached without error.") ### TEST 009 ### def test_009(self): """Get the ~100MB file from FileCacher. """ logger.info(" I am retrieving the ~100MB file from FileCacher " + "after deleting the cache.") os.unlink(self.cache_path) hash_file = HashingFile() try: self.file_cacher.get_file_to_fobj(self.digest, hash_file) except Exception as error: self.test_end(False, "Error received: %r." % error) return my_digest = hash_file.digest hash_file.close() try: if self.digest != my_digest: self.test_end(False, "Content differs.") elif not os.path.exists(self.cache_path): self.test_end(False, "File not stored in local cache.") else: self.test_end(True, "Content object received " + "and cached correctly.") finally: self.file_cacher.delete(self.digest)
class TestFileCacher(unittest.TestCase): """Service that performs automatically some tests for the FileCacher service. """ def setUp(self): self.file_cacher = FileCacher() #self.file_cacher = FileCacher(self, path="fs-storage") self.cache_base_path = self.file_cacher.file_dir self.cache_path = None self.content = None self.fake_content = None self.digest = None self.file_obj = None def tearDown(self): shutil.rmtree(self.cache_base_path, ignore_errors=True) def test_file_life(self): """Send a ~100B random binary file to the storage through FileCacher as a file-like object. FC should cache the content locally. Then retrieve it. Then check its size. Then get it back. Then delete it. """ self.size = 100 self.content = b"".join( chr(random.randint(0, 255)) for unused_i in xrange(self.size)) data = self.file_cacher.put_file_from_fobj(StringIO(self.content), u"Test #000") if not os.path.exists(os.path.join(self.cache_base_path, data)): self.fail("File not stored in local cache.") elif io.open(os.path.join(self.cache_base_path, data), "rb").read() != self.content: self.fail("Local cache's content differ " "from original file.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data # Retrieve the file. self.fake_content = "Fake content.\n" with io.open(self.cache_path, "wb") as cached_file: cached_file.write(self.fake_content) try: data = self.file_cacher.get_file(self.digest) except Exception as error: self.fail("Error received: %r." % error) return received = data.read() data.close() if received != self.fake_content: if received == self.content: self.fail("Did not use the cache even if it could.") else: self.fail("Content differ.") # Check the size of the file. try: size = self.file_cacher.get_size(self.digest) except Exception as error: self.fail("Error received: %r." % error) return if size != self.size: self.fail("The size is wrong: %d instead of %d" % (size, self.size)) # Get file from FileCacher. os.unlink(self.cache_path) try: data = self.file_cacher.get_file(self.digest) except Exception as error: self.fail("Error received: %r." % error) return received = data.read() data.close() if received != self.content: self.fail("Content differ.") elif not os.path.exists(self.cache_path): self.fail("File not stored in local cache.") elif io.open(self.cache_path, "rb").read() != self.content: self.fail("Local cache's content differ " + "from original file.") # Delete the file through FS and tries to get it again through # FC. try: self.file_cacher.delete(digest=self.digest) except Exception as error: self.fail("Error received: %s." % error) return else: with self.assertRaises(Exception): self.file_cacher.get_file(self.digest) def test_fetch_missing_file(self): """Get unexisting file from FileCacher. """ with self.assertRaises(Exception): self.file_cacher.get_file(self.digest) def test_file_as_content(self): """Send a ~100B random binary file to the storage through FileCacher as a string. FC should cache the content locally. Then retrieve it as a string. """ self.content = b"".join( chr(random.randint(0, 255)) for unused_i in xrange(100)) try: data = self.file_cacher.put_file_content(self.content, u"Test #005") except Exception as error: self.fail("Error received: %r." % error) return if not os.path.exists(os.path.join(self.cache_base_path, data)): self.fail("File not stored in local cache.") elif io.open(os.path.join(self.cache_base_path, data), "rb").read() != self.content: self.fail("Local cache's content differ " "from original file.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data # Retrieve the file as a string. self.fake_content = "Fake content.\n" with io.open(self.cache_path, "wb") as cached_file: cached_file.write(self.fake_content) try: data = self.file_cacher.get_file_content(self.digest) except Exception as error: self.fail("Error received: %r." % error) return if data != self.fake_content: if data == self.content: self.fail("Did not use the cache even if it could.") else: self.fail("Content differ.") def test_big_file(self): """Put a ~10MB file into the storage (using a specially crafted file-like object). Then get it back. """ rand_file = RandomFile(10000000) try: data = self.file_cacher.put_file_from_fobj(rand_file, u"Test #007") except Exception as error: self.fail("Error received: %r." % error) return if rand_file.dim != 0: self.fail("The input file wasn't read completely.") my_digest = rand_file.digest rand_file.close() if not os.path.exists(os.path.join(self.cache_base_path, data)): self.fail("File not stored in local cache.") elif my_digest != data: self.fail("File received with wrong hash.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data # Get the ~100MB file from FileCacher. os.unlink(self.cache_path) hash_file = HashingFile() try: self.file_cacher.get_file_to_fobj(self.digest, hash_file) except Exception as error: self.fail("Error received: %r." % error) return my_digest = hash_file.digest hash_file.close() try: if self.digest != my_digest: self.fail("Content differs.") elif not os.path.exists(self.cache_path): self.fail("File not stored in local cache.") finally: self.file_cacher.delete(self.digest)
class TestFileCacher(unittest.TestCase): """Service that performs automatically some tests for the FileCacher service. """ def setUp(self): self.file_cacher = FileCacher() #self.file_cacher = FileCacher(self, path="fs-storage") self.cache_base_path = self.file_cacher.file_dir self.cache_path = None self.content = None self.fake_content = None self.digest = None self.file_obj = None def tearDown(self): shutil.rmtree(self.cache_base_path, ignore_errors=True) def test_file_life(self): """Send a ~100B random binary file to the storage through FileCacher as a file-like object. FC should cache the content locally. Then retrieve it. Then check its size. Then get it back. Then delete it. """ self.size = 100 self.content = b"".join(chr(random.randint(0, 255)) for unused_i in xrange(self.size)) data = self.file_cacher.put_file_from_fobj(StringIO(self.content), u"Test #000") if not os.path.exists(os.path.join(self.cache_base_path, data)): self.fail("File not stored in local cache.") elif io.open(os.path.join(self.cache_base_path, data), "rb").read() != self.content: self.fail("Local cache's content differ " "from original file.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data # Retrieve the file. self.fake_content = "Fake content.\n" with io.open(self.cache_path, "wb") as cached_file: cached_file.write(self.fake_content) try: data = self.file_cacher.get_file(self.digest) except Exception as error: self.fail("Error received: %r." % error) return received = data.read() data.close() if received != self.fake_content: if received == self.content: self.fail("Did not use the cache even if it could.") else: self.fail("Content differ.") # Check the size of the file. try: size = self.file_cacher.get_size(self.digest) except Exception as error: self.fail("Error received: %r." % error) return if size != self.size: self.fail("The size is wrong: %d instead of %d" % (size, self.size)) # Get file from FileCacher. os.unlink(self.cache_path) try: data = self.file_cacher.get_file(self.digest) except Exception as error: self.fail("Error received: %r." % error) return received = data.read() data.close() if received != self.content: self.fail("Content differ.") elif not os.path.exists(self.cache_path): self.fail("File not stored in local cache.") elif io.open(self.cache_path, "rb").read() != self.content: self.fail("Local cache's content differ " + "from original file.") # Delete the file through FS and tries to get it again through # FC. try: self.file_cacher.delete(digest=self.digest) except Exception as error: self.fail("Error received: %s." % error) return else: with self.assertRaises(Exception): self.file_cacher.get_file(self.digest) def test_fetch_missing_file(self): """Get unexisting file from FileCacher. """ with self.assertRaises(Exception): self.file_cacher.get_file(self.digest) def test_file_as_content(self): """Send a ~100B random binary file to the storage through FileCacher as a string. FC should cache the content locally. Then retrieve it as a string. """ self.content = b"".join(chr(random.randint(0, 255)) for unused_i in xrange(100)) try: data = self.file_cacher.put_file_content(self.content, u"Test #005") except Exception as error: self.fail("Error received: %r." % error) return if not os.path.exists(os.path.join(self.cache_base_path, data)): self.fail("File not stored in local cache.") elif io.open(os.path.join(self.cache_base_path, data), "rb").read() != self.content: self.fail("Local cache's content differ " "from original file.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data # Retrieve the file as a string. self.fake_content = "Fake content.\n" with io.open(self.cache_path, "wb") as cached_file: cached_file.write(self.fake_content) try: data = self.file_cacher.get_file_content(self.digest) except Exception as error: self.fail("Error received: %r." % error) return if data != self.fake_content: if data == self.content: self.fail("Did not use the cache even if it could.") else: self.fail("Content differ.") def test_big_file(self): """Put a ~10MB file into the storage (using a specially crafted file-like object). Then get it back. """ rand_file = RandomFile(10000000) try: data = self.file_cacher.put_file_from_fobj(rand_file, u"Test #007") except Exception as error: self.fail("Error received: %r." % error) return if rand_file.dim != 0: self.fail("The input file wasn't read completely.") my_digest = rand_file.digest rand_file.close() if not os.path.exists(os.path.join(self.cache_base_path, data)): self.fail("File not stored in local cache.") elif my_digest != data: self.fail("File received with wrong hash.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data # Get the ~100MB file from FileCacher. os.unlink(self.cache_path) hash_file = HashingFile() try: self.file_cacher.get_file_to_fobj(self.digest, hash_file) except Exception as error: self.fail("Error received: %r." % error) return my_digest = hash_file.digest hash_file.close() try: if self.digest != my_digest: self.fail("Content differs.") elif not os.path.exists(self.cache_path): self.fail("File not stored in local cache.") finally: self.file_cacher.delete(self.digest)
class TestFileCacher(TestService): """Service that performs automatically some tests for the FileCacher service. """ def __init__(self, shard): logger.initialize(ServiceCoord("TestFileCacher", shard)) TestService.__init__(self, shard, custom_logger=logger) # Assume we store the cache in "./cache/fs-cache-TestFileCacher-0/" self.cache_base_path = os.path.join(config.cache_dir, "fs-cache-TestFileCacher-0") self.cache_path = None self.content = None self.fake_content = None self.digest = None self.file_obj = None self.file_cacher = FileCacher(self) #self.file_cacher = FileCacher(self, path="fs-storage") def prepare(self): """Initialization for the test code - make sure that the cache is empty before testing. """ logger.info("Please delete directory %s before." % self.cache_base_path) ### TEST 000 ### def test_000(self): """Send a ~100B random binary file to the storage through FileCacher as a file-like object. FC should cache the content locally. """ self.size = 100 self.content = "".join( chr(random.randint(0, 255)) for unused_i in xrange(self.size)) logger.info(" I am sending the ~100B binary file to FileCacher") try: data = self.file_cacher.put_file_from_fobj(StringIO(self.content), u"Test #000") except Exception as error: self.test_end(False, "Error received: %r." % error) return if not os.path.exists(os.path.join(self.cache_base_path, data)): self.test_end(False, "File not stored in local cache.") elif open(os.path.join(self.cache_base_path, data), "rb").read() != \ self.content: self.test_end( False, "Local cache's content differ " "from original file.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data self.test_end(True, "Data sent and cached without error.") ### TEST 001 ### def test_001(self): """Retrieve the file. """ logger.info(" I am retrieving the ~100B binary file from FileCacher") self.fake_content = "Fake content.\n" with open(self.cache_path, "wb") as cached_file: cached_file.write(self.fake_content) try: data = self.file_cacher.get_file(self.digest) except Exception as error: self.test_end(False, "Error received: %r." % error) return received = data.read() data.close() if received != self.fake_content: if received == self.content: self.test_end(False, "Did not use the cache even if it could.") else: self.test_end(False, "Content differ.") else: self.test_end(True, "Data object received correctly.") ### TEST 002 ### def test_002(self): """Check the size of the file. """ logger.info(" I am checking the size of the ~100B binary file") try: size = self.file_cacher.get_size(self.digest) except Exception as error: self.test_end(False, "Error received: %r." % error) return if size == self.size: self.test_end(True, "The size is correct.") else: self.test_end( False, "The size is wrong: %d instead of %d" % (size, self.size)) ### TEST 003 ### def test_003(self): """Get file from FileCacher. """ logger.info(" I am retrieving the file from FileCacher " + "after deleting the cache.") os.unlink(self.cache_path) try: data = self.file_cacher.get_file(self.digest) except Exception as error: self.test_end(False, "Error received: %r." % error) return received = data.read() data.close() if received != self.content: self.test_end(False, "Content differ.") elif not os.path.exists(self.cache_path): self.test_end(False, "File not stored in local cache.") elif open(self.cache_path).read() != self.content: self.test_end( False, "Local cache's content differ " + "from original file.") else: self.test_end(True, "Content object received " + "and cached correctly.") ### TEST 004 ### def test_004(self): """Delete the file through FS and tries to get it again through FC. """ logger.info(" I am deleting the file from FileCacher.") try: self.file_cacher.delete(digest=self.digest) except Exception as error: self.test_end(False, "Error received: %s." % error) return else: logger.info(" File deleted correctly.") logger.info(" I am getting the file from FileCacher.") try: self.file_cacher.get_file(self.digest) except Exception as error: self.test_end(True, "Correctly received an error: %r." % error) else: self.test_end(False, "Did not receive error.") ### TEST 005 ### def test_005(self): """Get unexisting file from FileCacher. """ logger.info(" I am retrieving an unexisting file from FileCacher.") try: self.file_cacher.get_file(self.digest) except Exception as error: self.test_end(True, "Correctly received an error: %r." % error) else: self.test_end(False, "Did not receive error.") ### TEST 006 ### def test_006(self): """Send a ~100B random binary file to the storage through FileCacher as a string. FC should cache the content locally. """ self.content = "".join( chr(random.randint(0, 255)) for unused_i in xrange(100)) logger.info(" I am sending the ~100B binary file to FileCacher") try: data = self.file_cacher.put_file_content(self.content, u"Test #005") except Exception as error: self.test_end(False, "Error received: %r." % error) return if not os.path.exists(os.path.join(self.cache_base_path, data)): self.test_end(False, "File not stored in local cache.") elif open(os.path.join(self.cache_base_path, data), "rb").read() != self.content: self.test_end( False, "Local cache's content differ " "from original file.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data self.test_end(True, "Data sent and cached without error.") ### TEST 007 ### def test_007(self): """Retrieve the file as a string. """ logger.info(" I am retrieving the ~100B binary file from FileCacher " "using get_file_to_string()") self.fake_content = "Fake content.\n" with open(self.cache_path, "wb") as cached_file: cached_file.write(self.fake_content) try: data = self.file_cacher.get_file_content(self.digest) except Exception as error: self.test_end(False, "Error received: %r." % error) return if data != self.fake_content: if data == self.content: self.test_end(False, "Did not use the cache even if it could.") else: self.test_end(False, "Content differ.") else: self.test_end(True, "Data received correctly.") ### TEST 008 ### def test_008(self): """Put a ~100MB file into the storage (using a specially crafted file-like object). """ logger.info(" I am sending the ~100MB binary file to FileCacher") rand_file = RandomFile(100000000) try: data = self.file_cacher.put_file_from_fobj(rand_file, u"Test #007") except Exception as error: self.test_end(False, "Error received: %r." % error) return if rand_file.dim != 0: self.test_end(False, "The input file wasn't read completely.") my_digest = rand_file.digest rand_file.close() if not os.path.exists(os.path.join(self.cache_base_path, data)): self.test_end(False, "File not stored in local cache.") elif my_digest != data: self.test_end(False, "File received with wrong hash.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data self.test_end(True, "Data sent and cached without error.") ### TEST 009 ### def test_009(self): """Get the ~100MB file from FileCacher. """ logger.info(" I am retrieving the ~100MB file from FileCacher " + "after deleting the cache.") os.unlink(self.cache_path) hash_file = HashingFile() try: self.file_cacher.get_file_to_fobj(self.digest, hash_file) except Exception as error: self.test_end(False, "Error received: %r." % error) return my_digest = hash_file.digest hash_file.close() try: if self.digest != my_digest: self.test_end(False, "Content differs.") elif not os.path.exists(self.cache_path): self.test_end(False, "File not stored in local cache.") else: self.test_end( True, "Content object received " + "and cached correctly.") finally: self.file_cacher.delete(self.digest)