class TestFileCacher(TestService): """Service that performs automatically some tests for the FileCacher service. """ def __init__(self, shard): logger.initialize(ServiceCoord("TestFileCacher", shard)) TestService.__init__(self, shard, custom_logger=logger) # Assume we store the cache in "./cache/fs-cache-TestFileCacher-0/" self.cache_base_path = os.path.join(config.cache_dir, "fs-cache-TestFileCacher-0") self.cache_path = None self.content = None self.fake_content = None self.digest = None self.file_obj = None self.file_cacher = FileCacher(self) #self.file_cacher = FileCacher(self, path="fs-storage") def prepare(self): """Initialization for the test code - make sure that the cache is empty before testing. """ logger.info("Please delete directory %s before." % self.cache_base_path) ### TEST 000 ### def test_000(self): """Send a ~100B random binary file to the storage through FileCacher as a file-like object. FC should cache the content locally. """ self.size = 100 self.content = "".join(chr(random.randint(0, 255)) for unused_i in xrange(self.size)) logger.info(" I am sending the ~100B binary file to FileCacher") try: data = self.file_cacher.put_file_from_fobj(StringIO(self.content), u"Test #000") except Exception as error: self.test_end(False, "Error received: %r." % error) return if not os.path.exists(os.path.join(self.cache_base_path, data)): self.test_end(False, "File not stored in local cache.") elif open(os.path.join(self.cache_base_path, data), "rb").read() != \ self.content: self.test_end(False, "Local cache's content differ " "from original file.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data self.test_end(True, "Data sent and cached without error.") ### TEST 001 ### def test_001(self): """Retrieve the file. """ logger.info(" I am retrieving the ~100B binary file from FileCacher") self.fake_content = "Fake content.\n" with open(self.cache_path, "wb") as cached_file: cached_file.write(self.fake_content) try: data = self.file_cacher.get_file(self.digest) except Exception as error: self.test_end(False, "Error received: %r." % error) return received = data.read() data.close() if received != self.fake_content: if received == self.content: self.test_end(False, "Did not use the cache even if it could.") else: self.test_end(False, "Content differ.") else: self.test_end(True, "Data object received correctly.") ### TEST 002 ### def test_002(self): """Check the size of the file. """ logger.info(" I am checking the size of the ~100B binary file") try: size = self.file_cacher.get_size(self.digest) except Exception as error: self.test_end(False, "Error received: %r." % error) return if size == self.size: self.test_end(True, "The size is correct.") else: self.test_end(False, "The size is wrong: %d instead of %d" % (size, self.size)) ### TEST 003 ### def test_003(self): """Get file from FileCacher. """ logger.info(" I am retrieving the file from FileCacher " + "after deleting the cache.") os.unlink(self.cache_path) try: data = self.file_cacher.get_file(self.digest) except Exception as error: self.test_end(False, "Error received: %r." % error) return received = data.read() data.close() if received != self.content: self.test_end(False, "Content differ.") elif not os.path.exists(self.cache_path): self.test_end(False, "File not stored in local cache.") elif open(self.cache_path).read() != self.content: self.test_end(False, "Local cache's content differ " + "from original file.") else: self.test_end(True, "Content object received " + "and cached correctly.") ### TEST 004 ### def test_004(self): """Delete the file through FS and tries to get it again through FC. """ logger.info(" I am deleting the file from FileCacher.") try: self.file_cacher.delete(digest=self.digest) except Exception as error: self.test_end(False, "Error received: %s." % error) return else: logger.info(" File deleted correctly.") logger.info(" I am getting the file from FileCacher.") try: self.file_cacher.get_file(self.digest) except Exception as error: self.test_end(True, "Correctly received an error: %r." % error) else: self.test_end(False, "Did not receive error.") ### TEST 005 ### def test_005(self): """Get unexisting file from FileCacher. """ logger.info(" I am retrieving an unexisting file from FileCacher.") try: self.file_cacher.get_file(self.digest) except Exception as error: self.test_end(True, "Correctly received an error: %r." % error) else: self.test_end(False, "Did not receive error.") ### TEST 006 ### def test_006(self): """Send a ~100B random binary file to the storage through FileCacher as a string. FC should cache the content locally. """ self.content = "".join(chr(random.randint(0, 255)) for unused_i in xrange(100)) logger.info(" I am sending the ~100B binary file to FileCacher") try: data = self.file_cacher.put_file_content(self.content, u"Test #005") except Exception as error: self.test_end(False, "Error received: %r." % error) return if not os.path.exists(os.path.join(self.cache_base_path, data)): self.test_end(False, "File not stored in local cache.") elif open(os.path.join(self.cache_base_path, data), "rb").read() != self.content: self.test_end(False, "Local cache's content differ " "from original file.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data self.test_end(True, "Data sent and cached without error.") ### TEST 007 ### def test_007(self): """Retrieve the file as a string. """ logger.info(" I am retrieving the ~100B binary file from FileCacher " "using get_file_to_string()") self.fake_content = "Fake content.\n" with open(self.cache_path, "wb") as cached_file: cached_file.write(self.fake_content) try: data = self.file_cacher.get_file_content(self.digest) except Exception as error: self.test_end(False, "Error received: %r." % error) return if data != self.fake_content: if data == self.content: self.test_end(False, "Did not use the cache even if it could.") else: self.test_end(False, "Content differ.") else: self.test_end(True, "Data received correctly.") ### TEST 008 ### def test_008(self): """Put a ~100MB file into the storage (using a specially crafted file-like object). """ logger.info(" I am sending the ~100MB binary file to FileCacher") rand_file = RandomFile(100000000) try: data = self.file_cacher.put_file_from_fobj(rand_file, u"Test #007") except Exception as error: self.test_end(False, "Error received: %r." % error) return if rand_file.dim != 0: self.test_end(False, "The input file wasn't read completely.") my_digest = rand_file.digest rand_file.close() if not os.path.exists(os.path.join(self.cache_base_path, data)): self.test_end(False, "File not stored in local cache.") elif my_digest != data: self.test_end(False, "File received with wrong hash.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data self.test_end(True, "Data sent and cached without error.") ### TEST 009 ### def test_009(self): """Get the ~100MB file from FileCacher. """ logger.info(" I am retrieving the ~100MB file from FileCacher " + "after deleting the cache.") os.unlink(self.cache_path) hash_file = HashingFile() try: self.file_cacher.get_file_to_fobj(self.digest, hash_file) except Exception as error: self.test_end(False, "Error received: %r." % error) return my_digest = hash_file.digest hash_file.close() try: if self.digest != my_digest: self.test_end(False, "Content differs.") elif not os.path.exists(self.cache_path): self.test_end(False, "File not stored in local cache.") else: self.test_end(True, "Content object received " + "and cached correctly.") finally: self.file_cacher.delete(self.digest)
def test_testcases(base_dir, soluzione, language, assume=None): global task, file_cacher # Use a disabled FileCacher with a FSBackend in order to avoid to fill # the database with junk and to save up space. if file_cacher is None: file_cacher = FileCacher(path=os.path.join(config.cache_dir, 'cmsMake'), enabled=False) # Load the task if task is None: loader = YamlLoader(os.path.realpath(os.path.join(base_dir, "..")), file_cacher) # Normally we should import the contest before, but YamlLoader # accepts get_task() even without previous get_contest() calls task = loader.get_task(os.path.split(os.path.realpath(base_dir))[1]) # Prepare the EvaluationJob dataset = task.active_dataset if dataset.task_type != "OutputOnly": digest = file_cacher.put_file_from_path( os.path.join(base_dir, soluzione), "Solution %s for task %s" % (soluzione, task.name)) executables = { task.name: Executable(filename=task.name, digest=digest) } jobs = [(t, EvaluationJob(language=language, task_type=dataset.task_type, task_type_parameters=json.loads( dataset.task_type_parameters), managers=dict(dataset.managers), executables=executables, input=dataset.testcases[t].input, output=dataset.testcases[t].output, time_limit=dataset.time_limit, memory_limit=dataset.memory_limit)) for t in dataset.testcases] tasktype = get_task_type(dataset=dataset) else: print("Generating outputs...", end='') files = {} for t in sorted(dataset.testcases.keys()): with file_cacher.get_file(dataset.testcases[t].input) as fin: with TemporaryFile() as fout: print(str(t), end='') call(soluzione, stdin=fin, stdout=fout, cwd=base_dir) fout.seek(0) digest = file_cacher.put_file_from_fobj(fout) outname = "output_%s.txt" % t files[outname] = File(filename=outname, digest=digest) jobs = [(t, EvaluationJob(task_type=dataset.task_type, task_type_parameters=json.loads( dataset.task_type_parameters), managers=dict(dataset.managers), files=files, input=dataset.testcases[t].input, output=dataset.testcases[t].output, time_limit=dataset.time_limit, memory_limit=dataset.memory_limit)) for t in dataset.testcases] for k, job in jobs: job._key = k tasktype = get_task_type(dataset=dataset) print() ask_again = True last_status = "ok" status = "ok" stop = False info = [] points = [] comments = [] tcnames = [] for jobinfo in sorted(jobs): print(jobinfo[0], end='') sys.stdout.flush() job = jobinfo[1] # Skip the testcase if we decide to consider everything to # timeout if stop: info.append("Time limit exceeded") points.append(0.0) comments.append("Timeout.") continue # Evaluate testcase last_status = status tasktype.evaluate(job, file_cacher) if dataset.task_type != "OutputOnly": status = job.plus["exit_status"] info.append("Time: %5.3f Wall: %5.3f Memory: %s" % (job.plus["execution_time"], job.plus["execution_wall_clock_time"], mem_human(job.plus["execution_memory"]))) else: status = "ok" info.append("N/A") points.append(float(job.outcome)) comments.append(format_status_text(job.text)) tcnames.append(jobinfo[0]) # If we saw two consecutive timeouts, ask wether we want to # consider everything to timeout if ask_again and status == "timeout" and last_status == "timeout": print() print("Want to stop and consider everything to timeout? [y/N]", end='') if assume is not None: print(assume) tmp = assume else: tmp = raw_input().lower() if tmp in ['y', 'yes']: stop = True else: ask_again = False # Result pretty printing print() clen = max(len(c) for c in comments) ilen = max(len(i) for i in info) for (i, p, c, b) in zip(tcnames, points, comments, info): print("%s) %5.2lf --- %s [%s]" % (i, p, c.ljust(clen), b.center(ilen))) return zip(points, comments, info)
class TestFileCacher(unittest.TestCase): """Service that performs automatically some tests for the FileCacher service. """ def setUp(self): self.file_cacher = FileCacher() #self.file_cacher = FileCacher(self, path="fs-storage") self.cache_base_path = self.file_cacher.file_dir self.cache_path = None self.content = None self.fake_content = None self.digest = None self.file_obj = None def tearDown(self): shutil.rmtree(self.cache_base_path, ignore_errors=True) def test_file_life(self): """Send a ~100B random binary file to the storage through FileCacher as a file-like object. FC should cache the content locally. Then retrieve it. Then check its size. Then get it back. Then delete it. """ self.size = 100 self.content = b"".join( chr(random.randint(0, 255)) for unused_i in xrange(self.size)) data = self.file_cacher.put_file_from_fobj(StringIO(self.content), u"Test #000") if not os.path.exists(os.path.join(self.cache_base_path, data)): self.fail("File not stored in local cache.") elif io.open(os.path.join(self.cache_base_path, data), "rb").read() != self.content: self.fail("Local cache's content differ " "from original file.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data # Retrieve the file. self.fake_content = "Fake content.\n" with io.open(self.cache_path, "wb") as cached_file: cached_file.write(self.fake_content) try: data = self.file_cacher.get_file(self.digest) except Exception as error: self.fail("Error received: %r." % error) return received = data.read() data.close() if received != self.fake_content: if received == self.content: self.fail("Did not use the cache even if it could.") else: self.fail("Content differ.") # Check the size of the file. try: size = self.file_cacher.get_size(self.digest) except Exception as error: self.fail("Error received: %r." % error) return if size != self.size: self.fail("The size is wrong: %d instead of %d" % (size, self.size)) # Get file from FileCacher. os.unlink(self.cache_path) try: data = self.file_cacher.get_file(self.digest) except Exception as error: self.fail("Error received: %r." % error) return received = data.read() data.close() if received != self.content: self.fail("Content differ.") elif not os.path.exists(self.cache_path): self.fail("File not stored in local cache.") elif io.open(self.cache_path, "rb").read() != self.content: self.fail("Local cache's content differ " + "from original file.") # Delete the file through FS and tries to get it again through # FC. try: self.file_cacher.delete(digest=self.digest) except Exception as error: self.fail("Error received: %s." % error) return else: with self.assertRaises(Exception): self.file_cacher.get_file(self.digest) def test_fetch_missing_file(self): """Get unexisting file from FileCacher. """ with self.assertRaises(Exception): self.file_cacher.get_file(self.digest) def test_file_as_content(self): """Send a ~100B random binary file to the storage through FileCacher as a string. FC should cache the content locally. Then retrieve it as a string. """ self.content = b"".join( chr(random.randint(0, 255)) for unused_i in xrange(100)) try: data = self.file_cacher.put_file_content(self.content, u"Test #005") except Exception as error: self.fail("Error received: %r." % error) return if not os.path.exists(os.path.join(self.cache_base_path, data)): self.fail("File not stored in local cache.") elif io.open(os.path.join(self.cache_base_path, data), "rb").read() != self.content: self.fail("Local cache's content differ " "from original file.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data # Retrieve the file as a string. self.fake_content = "Fake content.\n" with io.open(self.cache_path, "wb") as cached_file: cached_file.write(self.fake_content) try: data = self.file_cacher.get_file_content(self.digest) except Exception as error: self.fail("Error received: %r." % error) return if data != self.fake_content: if data == self.content: self.fail("Did not use the cache even if it could.") else: self.fail("Content differ.") def test_big_file(self): """Put a ~10MB file into the storage (using a specially crafted file-like object). Then get it back. """ rand_file = RandomFile(10000000) try: data = self.file_cacher.put_file_from_fobj(rand_file, u"Test #007") except Exception as error: self.fail("Error received: %r." % error) return if rand_file.dim != 0: self.fail("The input file wasn't read completely.") my_digest = rand_file.digest rand_file.close() if not os.path.exists(os.path.join(self.cache_base_path, data)): self.fail("File not stored in local cache.") elif my_digest != data: self.fail("File received with wrong hash.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data # Get the ~100MB file from FileCacher. os.unlink(self.cache_path) hash_file = HashingFile() try: self.file_cacher.get_file_to_fobj(self.digest, hash_file) except Exception as error: self.fail("Error received: %r." % error) return my_digest = hash_file.digest hash_file.close() try: if self.digest != my_digest: self.fail("Content differs.") elif not os.path.exists(self.cache_path): self.fail("File not stored in local cache.") finally: self.file_cacher.delete(self.digest)
def test_testcases(base_dir, soluzione, language, assume=None): global task, file_cacher # Use a FileCacher with a NullBackend in order to avoid to fill # the database with junk if file_cacher is None: file_cacher = FileCacher(null=True) # Load the task # TODO - This implies copying a lot of data to the FileCacher, # which is annoying if you have to do it continuously; it would be # better to use a persistent cache (although local, possibly # filesystem-based instead of database-based) and somehow detect # when the task has already been loaded if task is None: loader = YamlLoader( os.path.realpath(os.path.join(base_dir, "..")), file_cacher) # Normally we should import the contest before, but YamlLoader # accepts get_task() even without previous get_contest() calls task = loader.get_task(os.path.split(os.path.realpath(base_dir))[1]) # Prepare the EvaluationJob dataset = task.active_dataset if dataset.task_type != "OutputOnly": digest = file_cacher.put_file_from_path( os.path.join(base_dir, soluzione), "Solution %s for task %s" % (soluzione, task.name)) executables = {task.name: Executable(filename=task.name, digest=digest)} jobs = [(t, EvaluationJob( language=language, task_type=dataset.task_type, task_type_parameters=json.loads(dataset.task_type_parameters), managers=dict(dataset.managers), executables=executables, input=dataset.testcases[t].input, output=dataset.testcases[t].output, time_limit=dataset.time_limit, memory_limit=dataset.memory_limit)) for t in dataset.testcases] tasktype = get_task_type(dataset=dataset) else: print("Generating outputs...", end='') files = {} for t in sorted(dataset.testcases.keys()): with file_cacher.get_file(dataset.testcases[t].input) as fin: with TemporaryFile() as fout: print("%s" % t, end='') call(soluzione, stdin=fin, stdout=fout, cwd=base_dir) fout.seek(0) digest = file_cacher.put_file_from_fobj(fout) outname = "output_%s.txt" % t files[outname] = File(filename=outname, digest=digest) jobs = [(t, EvaluationJob( task_type=dataset.task_type, task_type_parameters=json.loads(dataset.task_type_parameters), managers=dict(dataset.managers), files=files, input=dataset.testcases[t].input, output=dataset.testcases[t].output, time_limit=dataset.time_limit, memory_limit=dataset.memory_limit)) for t in dataset.testcases] for k, job in jobs: job._key = k tasktype = get_task_type(dataset=dataset) print() ask_again = True last_status = "ok" status = "ok" stop = False info = [] points = [] comments = [] tcnames = [] for jobinfo in sorted(jobs): print(jobinfo[0], end='') sys.stdout.flush() job = jobinfo[1] # Skip the testcase if we decide to consider everything to # timeout if stop: info.append("Time limit exceeded") points.append(0.0) comments.append("Timeout.") continue # Evaluate testcase last_status = status tasktype.evaluate(job, file_cacher) if dataset.task_type != "OutputOnly": status = job.plus["exit_status"] info.append("Time: %5.3f Wall: %5.3f Memory: %s" % (job.plus["execution_time"], job.plus["execution_wall_clock_time"], mem_human(job.plus["execution_memory"]))) else: status = "ok" info.append("N/A") points.append(float(job.outcome)) comments.append(format_status_text(job.text)) tcnames.append(jobinfo[0]) # If we saw two consecutive timeouts, ask wether we want to # consider everything to timeout if ask_again and status == "timeout" and last_status == "timeout": print() print("Want to stop and consider everything to timeout? [y/N]", end='') if assume is not None: print(assume) tmp = assume else: tmp = raw_input().lower() if tmp in ['y', 'yes']: stop = True else: ask_again = False # Result pretty printing print() clen = max(len(c) for c in comments) ilen = max(len(i) for i in info) for (i, p, c, b) in zip(tcnames, points, comments, info): print("%s) %5.2lf --- %s [%s]" % (i, p, c.ljust(clen), b.center(ilen))) return zip(points, comments, info)
class TestFileCacher(unittest.TestCase): """Service that performs automatically some tests for the FileCacher service. """ def setUp(self): self.file_cacher = FileCacher() #self.file_cacher = FileCacher(self, path="fs-storage") self.cache_base_path = self.file_cacher.file_dir self.cache_path = None self.content = None self.fake_content = None self.digest = None self.file_obj = None def tearDown(self): shutil.rmtree(self.cache_base_path, ignore_errors=True) def test_file_life(self): """Send a ~100B random binary file to the storage through FileCacher as a file-like object. FC should cache the content locally. Then retrieve it. Then check its size. Then get it back. Then delete it. """ self.size = 100 self.content = b"".join(chr(random.randint(0, 255)) for unused_i in xrange(self.size)) data = self.file_cacher.put_file_from_fobj(StringIO(self.content), u"Test #000") if not os.path.exists(os.path.join(self.cache_base_path, data)): self.fail("File not stored in local cache.") elif io.open(os.path.join(self.cache_base_path, data), "rb").read() != self.content: self.fail("Local cache's content differ " "from original file.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data # Retrieve the file. self.fake_content = "Fake content.\n" with io.open(self.cache_path, "wb") as cached_file: cached_file.write(self.fake_content) try: data = self.file_cacher.get_file(self.digest) except Exception as error: self.fail("Error received: %r." % error) return received = data.read() data.close() if received != self.fake_content: if received == self.content: self.fail("Did not use the cache even if it could.") else: self.fail("Content differ.") # Check the size of the file. try: size = self.file_cacher.get_size(self.digest) except Exception as error: self.fail("Error received: %r." % error) return if size != self.size: self.fail("The size is wrong: %d instead of %d" % (size, self.size)) # Get file from FileCacher. os.unlink(self.cache_path) try: data = self.file_cacher.get_file(self.digest) except Exception as error: self.fail("Error received: %r." % error) return received = data.read() data.close() if received != self.content: self.fail("Content differ.") elif not os.path.exists(self.cache_path): self.fail("File not stored in local cache.") elif io.open(self.cache_path, "rb").read() != self.content: self.fail("Local cache's content differ " + "from original file.") # Delete the file through FS and tries to get it again through # FC. try: self.file_cacher.delete(digest=self.digest) except Exception as error: self.fail("Error received: %s." % error) return else: with self.assertRaises(Exception): self.file_cacher.get_file(self.digest) def test_fetch_missing_file(self): """Get unexisting file from FileCacher. """ with self.assertRaises(Exception): self.file_cacher.get_file(self.digest) def test_file_as_content(self): """Send a ~100B random binary file to the storage through FileCacher as a string. FC should cache the content locally. Then retrieve it as a string. """ self.content = b"".join(chr(random.randint(0, 255)) for unused_i in xrange(100)) try: data = self.file_cacher.put_file_content(self.content, u"Test #005") except Exception as error: self.fail("Error received: %r." % error) return if not os.path.exists(os.path.join(self.cache_base_path, data)): self.fail("File not stored in local cache.") elif io.open(os.path.join(self.cache_base_path, data), "rb").read() != self.content: self.fail("Local cache's content differ " "from original file.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data # Retrieve the file as a string. self.fake_content = "Fake content.\n" with io.open(self.cache_path, "wb") as cached_file: cached_file.write(self.fake_content) try: data = self.file_cacher.get_file_content(self.digest) except Exception as error: self.fail("Error received: %r." % error) return if data != self.fake_content: if data == self.content: self.fail("Did not use the cache even if it could.") else: self.fail("Content differ.") def test_big_file(self): """Put a ~10MB file into the storage (using a specially crafted file-like object). Then get it back. """ rand_file = RandomFile(10000000) try: data = self.file_cacher.put_file_from_fobj(rand_file, u"Test #007") except Exception as error: self.fail("Error received: %r." % error) return if rand_file.dim != 0: self.fail("The input file wasn't read completely.") my_digest = rand_file.digest rand_file.close() if not os.path.exists(os.path.join(self.cache_base_path, data)): self.fail("File not stored in local cache.") elif my_digest != data: self.fail("File received with wrong hash.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data # Get the ~100MB file from FileCacher. os.unlink(self.cache_path) hash_file = HashingFile() try: self.file_cacher.get_file_to_fobj(self.digest, hash_file) except Exception as error: self.fail("Error received: %r." % error) return my_digest = hash_file.digest hash_file.close() try: if self.digest != my_digest: self.fail("Content differs.") elif not os.path.exists(self.cache_path): self.fail("File not stored in local cache.") finally: self.file_cacher.delete(self.digest)
class TestFileCacher(TestService): """Service that performs automatically some tests for the FileCacher service. """ def __init__(self, shard): logger.initialize(ServiceCoord("TestFileCacher", shard)) TestService.__init__(self, shard, custom_logger=logger) # Assume we store the cache in "./cache/fs-cache-TestFileCacher-0/" self.cache_base_path = os.path.join(config.cache_dir, "fs-cache-TestFileCacher-0") self.cache_path = None self.content = None self.fake_content = None self.digest = None self.file_obj = None self.file_cacher = FileCacher(self) #self.file_cacher = FileCacher(self, path="fs-storage") def prepare(self): """Initialization for the test code - make sure that the cache is empty before testing. """ logger.info("Please delete directory %s before." % self.cache_base_path) ### TEST 000 ### def test_000(self): """Send a ~100B random binary file to the storage through FileCacher as a file-like object. FC should cache the content locally. """ self.size = 100 self.content = "".join( chr(random.randint(0, 255)) for unused_i in xrange(self.size)) logger.info(" I am sending the ~100B binary file to FileCacher") try: data = self.file_cacher.put_file_from_fobj(StringIO(self.content), u"Test #000") except Exception as error: self.test_end(False, "Error received: %r." % error) return if not os.path.exists(os.path.join(self.cache_base_path, data)): self.test_end(False, "File not stored in local cache.") elif open(os.path.join(self.cache_base_path, data), "rb").read() != \ self.content: self.test_end( False, "Local cache's content differ " "from original file.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data self.test_end(True, "Data sent and cached without error.") ### TEST 001 ### def test_001(self): """Retrieve the file. """ logger.info(" I am retrieving the ~100B binary file from FileCacher") self.fake_content = "Fake content.\n" with open(self.cache_path, "wb") as cached_file: cached_file.write(self.fake_content) try: data = self.file_cacher.get_file(self.digest) except Exception as error: self.test_end(False, "Error received: %r." % error) return received = data.read() data.close() if received != self.fake_content: if received == self.content: self.test_end(False, "Did not use the cache even if it could.") else: self.test_end(False, "Content differ.") else: self.test_end(True, "Data object received correctly.") ### TEST 002 ### def test_002(self): """Check the size of the file. """ logger.info(" I am checking the size of the ~100B binary file") try: size = self.file_cacher.get_size(self.digest) except Exception as error: self.test_end(False, "Error received: %r." % error) return if size == self.size: self.test_end(True, "The size is correct.") else: self.test_end( False, "The size is wrong: %d instead of %d" % (size, self.size)) ### TEST 003 ### def test_003(self): """Get file from FileCacher. """ logger.info(" I am retrieving the file from FileCacher " + "after deleting the cache.") os.unlink(self.cache_path) try: data = self.file_cacher.get_file(self.digest) except Exception as error: self.test_end(False, "Error received: %r." % error) return received = data.read() data.close() if received != self.content: self.test_end(False, "Content differ.") elif not os.path.exists(self.cache_path): self.test_end(False, "File not stored in local cache.") elif open(self.cache_path).read() != self.content: self.test_end( False, "Local cache's content differ " + "from original file.") else: self.test_end(True, "Content object received " + "and cached correctly.") ### TEST 004 ### def test_004(self): """Delete the file through FS and tries to get it again through FC. """ logger.info(" I am deleting the file from FileCacher.") try: self.file_cacher.delete(digest=self.digest) except Exception as error: self.test_end(False, "Error received: %s." % error) return else: logger.info(" File deleted correctly.") logger.info(" I am getting the file from FileCacher.") try: self.file_cacher.get_file(self.digest) except Exception as error: self.test_end(True, "Correctly received an error: %r." % error) else: self.test_end(False, "Did not receive error.") ### TEST 005 ### def test_005(self): """Get unexisting file from FileCacher. """ logger.info(" I am retrieving an unexisting file from FileCacher.") try: self.file_cacher.get_file(self.digest) except Exception as error: self.test_end(True, "Correctly received an error: %r." % error) else: self.test_end(False, "Did not receive error.") ### TEST 006 ### def test_006(self): """Send a ~100B random binary file to the storage through FileCacher as a string. FC should cache the content locally. """ self.content = "".join( chr(random.randint(0, 255)) for unused_i in xrange(100)) logger.info(" I am sending the ~100B binary file to FileCacher") try: data = self.file_cacher.put_file_content(self.content, u"Test #005") except Exception as error: self.test_end(False, "Error received: %r." % error) return if not os.path.exists(os.path.join(self.cache_base_path, data)): self.test_end(False, "File not stored in local cache.") elif open(os.path.join(self.cache_base_path, data), "rb").read() != self.content: self.test_end( False, "Local cache's content differ " "from original file.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data self.test_end(True, "Data sent and cached without error.") ### TEST 007 ### def test_007(self): """Retrieve the file as a string. """ logger.info(" I am retrieving the ~100B binary file from FileCacher " "using get_file_to_string()") self.fake_content = "Fake content.\n" with open(self.cache_path, "wb") as cached_file: cached_file.write(self.fake_content) try: data = self.file_cacher.get_file_content(self.digest) except Exception as error: self.test_end(False, "Error received: %r." % error) return if data != self.fake_content: if data == self.content: self.test_end(False, "Did not use the cache even if it could.") else: self.test_end(False, "Content differ.") else: self.test_end(True, "Data received correctly.") ### TEST 008 ### def test_008(self): """Put a ~100MB file into the storage (using a specially crafted file-like object). """ logger.info(" I am sending the ~100MB binary file to FileCacher") rand_file = RandomFile(100000000) try: data = self.file_cacher.put_file_from_fobj(rand_file, u"Test #007") except Exception as error: self.test_end(False, "Error received: %r." % error) return if rand_file.dim != 0: self.test_end(False, "The input file wasn't read completely.") my_digest = rand_file.digest rand_file.close() if not os.path.exists(os.path.join(self.cache_base_path, data)): self.test_end(False, "File not stored in local cache.") elif my_digest != data: self.test_end(False, "File received with wrong hash.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data self.test_end(True, "Data sent and cached without error.") ### TEST 009 ### def test_009(self): """Get the ~100MB file from FileCacher. """ logger.info(" I am retrieving the ~100MB file from FileCacher " + "after deleting the cache.") os.unlink(self.cache_path) hash_file = HashingFile() try: self.file_cacher.get_file_to_fobj(self.digest, hash_file) except Exception as error: self.test_end(False, "Error received: %r." % error) return my_digest = hash_file.digest hash_file.close() try: if self.digest != my_digest: self.test_end(False, "Content differs.") elif not os.path.exists(self.cache_path): self.test_end(False, "File not stored in local cache.") else: self.test_end( True, "Content object received " + "and cached correctly.") finally: self.file_cacher.delete(self.digest)