class TestFileCacher(TestService): """Service that performs automatically some tests for the FileCacher service. """ def __init__(self, shard): logger.initialize(ServiceCoord("TestFileCacher", shard)) TestService.__init__(self, shard, custom_logger=logger) # Assume we store the cache in "./cache/fs-cache-TestFileCacher-0/" self.cache_base_path = os.path.join(config.cache_dir, "fs-cache-TestFileCacher-0") self.cache_path = None self.content = None self.fake_content = None self.digest = None self.file_obj = None self.file_cacher = FileCacher(self) #self.file_cacher = FileCacher(self, path="fs-storage") def prepare(self): """Initialization for the test code - make sure that the cache is empty before testing. """ logger.info("Please delete directory %s before." % self.cache_base_path) ### TEST 000 ### def test_000(self): """Send a ~100B random binary file to the storage through FileCacher as a file-like object. FC should cache the content locally. """ self.size = 100 self.content = "".join(chr(random.randint(0, 255)) for unused_i in xrange(self.size)) logger.info(" I am sending the ~100B binary file to FileCacher") try: data = self.file_cacher.put_file_from_fobj(StringIO(self.content), u"Test #000") except Exception as error: self.test_end(False, "Error received: %r." % error) return if not os.path.exists(os.path.join(self.cache_base_path, data)): self.test_end(False, "File not stored in local cache.") elif open(os.path.join(self.cache_base_path, data), "rb").read() != \ self.content: self.test_end(False, "Local cache's content differ " "from original file.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data self.test_end(True, "Data sent and cached without error.") ### TEST 001 ### def test_001(self): """Retrieve the file. """ logger.info(" I am retrieving the ~100B binary file from FileCacher") self.fake_content = "Fake content.\n" with open(self.cache_path, "wb") as cached_file: cached_file.write(self.fake_content) try: data = self.file_cacher.get_file(self.digest) except Exception as error: self.test_end(False, "Error received: %r." % error) return received = data.read() data.close() if received != self.fake_content: if received == self.content: self.test_end(False, "Did not use the cache even if it could.") else: self.test_end(False, "Content differ.") else: self.test_end(True, "Data object received correctly.") ### TEST 002 ### def test_002(self): """Check the size of the file. """ logger.info(" I am checking the size of the ~100B binary file") try: size = self.file_cacher.get_size(self.digest) except Exception as error: self.test_end(False, "Error received: %r." % error) return if size == self.size: self.test_end(True, "The size is correct.") else: self.test_end(False, "The size is wrong: %d instead of %d" % (size, self.size)) ### TEST 003 ### def test_003(self): """Get file from FileCacher. """ logger.info(" I am retrieving the file from FileCacher " + "after deleting the cache.") os.unlink(self.cache_path) try: data = self.file_cacher.get_file(self.digest) except Exception as error: self.test_end(False, "Error received: %r." % error) return received = data.read() data.close() if received != self.content: self.test_end(False, "Content differ.") elif not os.path.exists(self.cache_path): self.test_end(False, "File not stored in local cache.") elif open(self.cache_path).read() != self.content: self.test_end(False, "Local cache's content differ " + "from original file.") else: self.test_end(True, "Content object received " + "and cached correctly.") ### TEST 004 ### def test_004(self): """Delete the file through FS and tries to get it again through FC. """ logger.info(" I am deleting the file from FileCacher.") try: self.file_cacher.delete(digest=self.digest) except Exception as error: self.test_end(False, "Error received: %s." % error) return else: logger.info(" File deleted correctly.") logger.info(" I am getting the file from FileCacher.") try: self.file_cacher.get_file(self.digest) except Exception as error: self.test_end(True, "Correctly received an error: %r." % error) else: self.test_end(False, "Did not receive error.") ### TEST 005 ### def test_005(self): """Get unexisting file from FileCacher. """ logger.info(" I am retrieving an unexisting file from FileCacher.") try: self.file_cacher.get_file(self.digest) except Exception as error: self.test_end(True, "Correctly received an error: %r." % error) else: self.test_end(False, "Did not receive error.") ### TEST 006 ### def test_006(self): """Send a ~100B random binary file to the storage through FileCacher as a string. FC should cache the content locally. """ self.content = "".join(chr(random.randint(0, 255)) for unused_i in xrange(100)) logger.info(" I am sending the ~100B binary file to FileCacher") try: data = self.file_cacher.put_file_content(self.content, u"Test #005") except Exception as error: self.test_end(False, "Error received: %r." % error) return if not os.path.exists(os.path.join(self.cache_base_path, data)): self.test_end(False, "File not stored in local cache.") elif open(os.path.join(self.cache_base_path, data), "rb").read() != self.content: self.test_end(False, "Local cache's content differ " "from original file.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data self.test_end(True, "Data sent and cached without error.") ### TEST 007 ### def test_007(self): """Retrieve the file as a string. """ logger.info(" I am retrieving the ~100B binary file from FileCacher " "using get_file_to_string()") self.fake_content = "Fake content.\n" with open(self.cache_path, "wb") as cached_file: cached_file.write(self.fake_content) try: data = self.file_cacher.get_file_content(self.digest) except Exception as error: self.test_end(False, "Error received: %r." % error) return if data != self.fake_content: if data == self.content: self.test_end(False, "Did not use the cache even if it could.") else: self.test_end(False, "Content differ.") else: self.test_end(True, "Data received correctly.") ### TEST 008 ### def test_008(self): """Put a ~100MB file into the storage (using a specially crafted file-like object). """ logger.info(" I am sending the ~100MB binary file to FileCacher") rand_file = RandomFile(100000000) try: data = self.file_cacher.put_file_from_fobj(rand_file, u"Test #007") except Exception as error: self.test_end(False, "Error received: %r." % error) return if rand_file.dim != 0: self.test_end(False, "The input file wasn't read completely.") my_digest = rand_file.digest rand_file.close() if not os.path.exists(os.path.join(self.cache_base_path, data)): self.test_end(False, "File not stored in local cache.") elif my_digest != data: self.test_end(False, "File received with wrong hash.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data self.test_end(True, "Data sent and cached without error.") ### TEST 009 ### def test_009(self): """Get the ~100MB file from FileCacher. """ logger.info(" I am retrieving the ~100MB file from FileCacher " + "after deleting the cache.") os.unlink(self.cache_path) hash_file = HashingFile() try: self.file_cacher.get_file_to_fobj(self.digest, hash_file) except Exception as error: self.test_end(False, "Error received: %r." % error) return my_digest = hash_file.digest hash_file.close() try: if self.digest != my_digest: self.test_end(False, "Content differs.") elif not os.path.exists(self.cache_path): self.test_end(False, "File not stored in local cache.") else: self.test_end(True, "Content object received " + "and cached correctly.") finally: self.file_cacher.delete(self.digest)
def main(): """Parse arguments and launch process. """ parser = argparse.ArgumentParser( description="Export CMS submissions to a folder.\n", formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("-c", "--contest-id", action="store", type=int, help="id of contest (default: all contests)") parser.add_argument("-t", "--task-id", action="store", type=int, help="id of task (default: all tasks)") parser.add_argument("-u", "--user-id", action="store", type=int, help="id of user (default: all users)") parser.add_argument("-s", "--submission-id", action="store", type=int, help="id of submission (default: all submissions)") parser.add_argument("--utf8", action="store_true", help="if set, the files will be encoded in utf8" " when possible") parser.add_argument("--add-info", action="store_true", help="if set, information on the submission will" " be added in the first lines of each file") parser.add_argument("--min-score", action="store", type=float, help="ignore submissions which scored strictly" " less than this (default: 0.0)", default=0.0) parser.add_argument("--filename", action="store", type=utf8_decoder, help="the filename format to use\n" "Variables:\n" " id: submission id\n" " file: filename without extension\n" " ext: filename extension\n" " time: submission timestamp\n" " user: username\n" " task: taskname\n" " score: raw score\n" " (default: {id}.{file}{ext})", default="{id}.{file}{ext}") parser.add_argument("output_dir", action="store", type=utf8_decoder, help="directory where to save the submissions") parser.add_argument("-y", "--yes", action="store_true", help="if set, confirmation will not be shown") group = parser.add_mutually_exclusive_group(required=False) group.add_argument("--unique", action="store_true", help="if set, only the earliest best submission" " will be exported for each (user, task)") group.add_argument("--best", action="store_true", help="if set, only the best submissions will be" " exported for each (user, task)") args = parser.parse_args() if not os.path.exists(args.output_dir): os.mkdir(args.output_dir) if not os.path.isdir(args.output_dir): logger.critical("The output-dir parameter must point to a directory") return 1 cacher = FileCacher() with SessionGen() as session: q = session.query(Submission)\ .join(Submission.task)\ .join(Submission.files)\ .join(Submission.results)\ .join(SubmissionResult.dataset)\ .join(Submission.participation)\ .join(Participation.user)\ .filter(Dataset.id == Task.active_dataset_id)\ .filter(SubmissionResult.score >= args.min_score)\ .with_entities(Submission.id, Submission.language, Submission.timestamp, SubmissionResult.score, File.filename, File.digest, User.id, User.username, User.first_name, User.last_name, Task.id, Task.name) if args.contest_id: q = q.filter(Participation.contest_id == args.contest_id) if args.task_id: q = q.filter(Submission.task_id == args.task_id) if args.user_id: q = q.filter(Participation.user_id == args.user_id) if args.submission_id: q = q.filter(Submission.id == args.submission_id) results = q.all() if args.unique or args.best: results = filter_top_scoring(results, args.unique) print("%s file(s) will be created." % len(results)) if not args.yes and (input("Continue? [Y/n] ").strip().lower() not in ["y", ""]): return 0 done = 0 for row in results: s_id, s_language, s_timestamp, sr_score, f_filename, f_digest, \ u_id, u_name, u_fname, u_lname, t_id, t_name = row timef = s_timestamp.strftime('%Y%m%dT%H%M%S') ext = languagemanager.get_language(s_language).source_extension \ if s_language else '.txt' filename_base, filename_ext = os.path.splitext( f_filename.replace('.%l', ext)) # "name" is a deprecated specifier with the same meaning as "file" filename = args.filename.format(id=s_id, file=filename_base, name=filename_base, ext=filename_ext, time=timef, user=u_name, task=t_name, score=sr_score) filename = os.path.join(args.output_dir, filename) if os.path.exists(filename): logger.warning("Skipping file '%s' because it already exists", filename) continue filedir = os.path.dirname(filename) if not os.path.exists(filedir): os.makedirs(filedir) if not os.path.isdir(filedir): logger.warning("%s is not a directory, skipped.", filedir) continue if not (args.utf8 or args.add_info): cacher.get_file_to_path(f_digest, filename) else: content_bytes = cacher.get_file_content(f_digest) if args.utf8: try: content = utf8_decoder(content_bytes) content_bytes = content.encode("utf-8") except TypeError: logger.critical( "Could not guess encoding of file " "'%s'. Aborting.", filename) return 1 if args.add_info: template_str = TEMPLATE[ext] % (u_name, u_fname, u_lname, t_name, sr_score, s_timestamp) template_bytes = template_str.encode("utf-8") content_bytes = template_bytes + content_bytes with io.open(filename, 'wb') as f_out: f_out.write(content_bytes) done += 1 print(done, "/", len(results)) return 0
class TestFileCacher(unittest.TestCase): """Service that performs automatically some tests for the FileCacher service. """ def setUp(self): self.file_cacher = FileCacher() #self.file_cacher = FileCacher(self, path="fs-storage") self.cache_base_path = self.file_cacher.file_dir self.cache_path = None self.content = None self.fake_content = None self.digest = None self.file_obj = None def tearDown(self): shutil.rmtree(self.cache_base_path, ignore_errors=True) def test_file_life(self): """Send a ~100B random binary file to the storage through FileCacher as a file-like object. FC should cache the content locally. Then retrieve it. Then check its size. Then get it back. Then delete it. """ self.size = 100 self.content = b"".join(chr(random.randint(0, 255)) for unused_i in xrange(self.size)) data = self.file_cacher.put_file_from_fobj(StringIO(self.content), u"Test #000") if not os.path.exists(os.path.join(self.cache_base_path, data)): self.fail("File not stored in local cache.") elif io.open(os.path.join(self.cache_base_path, data), "rb").read() != self.content: self.fail("Local cache's content differ " "from original file.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data # Retrieve the file. self.fake_content = "Fake content.\n" with io.open(self.cache_path, "wb") as cached_file: cached_file.write(self.fake_content) try: data = self.file_cacher.get_file(self.digest) except Exception as error: self.fail("Error received: %r." % error) return received = data.read() data.close() if received != self.fake_content: if received == self.content: self.fail("Did not use the cache even if it could.") else: self.fail("Content differ.") # Check the size of the file. try: size = self.file_cacher.get_size(self.digest) except Exception as error: self.fail("Error received: %r." % error) return if size != self.size: self.fail("The size is wrong: %d instead of %d" % (size, self.size)) # Get file from FileCacher. os.unlink(self.cache_path) try: data = self.file_cacher.get_file(self.digest) except Exception as error: self.fail("Error received: %r." % error) return received = data.read() data.close() if received != self.content: self.fail("Content differ.") elif not os.path.exists(self.cache_path): self.fail("File not stored in local cache.") elif io.open(self.cache_path, "rb").read() != self.content: self.fail("Local cache's content differ " + "from original file.") # Delete the file through FS and tries to get it again through # FC. try: self.file_cacher.delete(digest=self.digest) except Exception as error: self.fail("Error received: %s." % error) return else: with self.assertRaises(Exception): self.file_cacher.get_file(self.digest) def test_fetch_missing_file(self): """Get unexisting file from FileCacher. """ with self.assertRaises(Exception): self.file_cacher.get_file(self.digest) def test_file_as_content(self): """Send a ~100B random binary file to the storage through FileCacher as a string. FC should cache the content locally. Then retrieve it as a string. """ self.content = b"".join(chr(random.randint(0, 255)) for unused_i in xrange(100)) try: data = self.file_cacher.put_file_content(self.content, u"Test #005") except Exception as error: self.fail("Error received: %r." % error) return if not os.path.exists(os.path.join(self.cache_base_path, data)): self.fail("File not stored in local cache.") elif io.open(os.path.join(self.cache_base_path, data), "rb").read() != self.content: self.fail("Local cache's content differ " "from original file.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data # Retrieve the file as a string. self.fake_content = "Fake content.\n" with io.open(self.cache_path, "wb") as cached_file: cached_file.write(self.fake_content) try: data = self.file_cacher.get_file_content(self.digest) except Exception as error: self.fail("Error received: %r." % error) return if data != self.fake_content: if data == self.content: self.fail("Did not use the cache even if it could.") else: self.fail("Content differ.") def test_big_file(self): """Put a ~10MB file into the storage (using a specially crafted file-like object). Then get it back. """ rand_file = RandomFile(10000000) try: data = self.file_cacher.put_file_from_fobj(rand_file, u"Test #007") except Exception as error: self.fail("Error received: %r." % error) return if rand_file.dim != 0: self.fail("The input file wasn't read completely.") my_digest = rand_file.digest rand_file.close() if not os.path.exists(os.path.join(self.cache_base_path, data)): self.fail("File not stored in local cache.") elif my_digest != data: self.fail("File received with wrong hash.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data # Get the ~100MB file from FileCacher. os.unlink(self.cache_path) hash_file = HashingFile() try: self.file_cacher.get_file_to_fobj(self.digest, hash_file) except Exception as error: self.fail("Error received: %r." % error) return my_digest = hash_file.digest hash_file.close() try: if self.digest != my_digest: self.fail("Content differs.") elif not os.path.exists(self.cache_path): self.fail("File not stored in local cache.") finally: self.file_cacher.delete(self.digest)
class TestFileCacher(unittest.TestCase): """Service that performs automatically some tests for the FileCacher service. """ def setUp(self): self.file_cacher = FileCacher() #self.file_cacher = FileCacher(self, path="fs-storage") self.cache_base_path = self.file_cacher.file_dir self.cache_path = None self.content = None self.fake_content = None self.digest = None self.file_obj = None def tearDown(self): shutil.rmtree(self.cache_base_path, ignore_errors=True) def test_file_life(self): """Send a ~100B random binary file to the storage through FileCacher as a file-like object. FC should cache the content locally. Then retrieve it. Then check its size. Then get it back. Then delete it. """ self.size = 100 self.content = b"".join( chr(random.randint(0, 255)) for unused_i in xrange(self.size)) data = self.file_cacher.put_file_from_fobj(StringIO(self.content), u"Test #000") if not os.path.exists(os.path.join(self.cache_base_path, data)): self.fail("File not stored in local cache.") elif io.open(os.path.join(self.cache_base_path, data), "rb").read() != self.content: self.fail("Local cache's content differ " "from original file.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data # Retrieve the file. self.fake_content = "Fake content.\n" with io.open(self.cache_path, "wb") as cached_file: cached_file.write(self.fake_content) try: data = self.file_cacher.get_file(self.digest) except Exception as error: self.fail("Error received: %r." % error) return received = data.read() data.close() if received != self.fake_content: if received == self.content: self.fail("Did not use the cache even if it could.") else: self.fail("Content differ.") # Check the size of the file. try: size = self.file_cacher.get_size(self.digest) except Exception as error: self.fail("Error received: %r." % error) return if size != self.size: self.fail("The size is wrong: %d instead of %d" % (size, self.size)) # Get file from FileCacher. os.unlink(self.cache_path) try: data = self.file_cacher.get_file(self.digest) except Exception as error: self.fail("Error received: %r." % error) return received = data.read() data.close() if received != self.content: self.fail("Content differ.") elif not os.path.exists(self.cache_path): self.fail("File not stored in local cache.") elif io.open(self.cache_path, "rb").read() != self.content: self.fail("Local cache's content differ " + "from original file.") # Delete the file through FS and tries to get it again through # FC. try: self.file_cacher.delete(digest=self.digest) except Exception as error: self.fail("Error received: %s." % error) return else: with self.assertRaises(Exception): self.file_cacher.get_file(self.digest) def test_fetch_missing_file(self): """Get unexisting file from FileCacher. """ with self.assertRaises(Exception): self.file_cacher.get_file(self.digest) def test_file_as_content(self): """Send a ~100B random binary file to the storage through FileCacher as a string. FC should cache the content locally. Then retrieve it as a string. """ self.content = b"".join( chr(random.randint(0, 255)) for unused_i in xrange(100)) try: data = self.file_cacher.put_file_content(self.content, u"Test #005") except Exception as error: self.fail("Error received: %r." % error) return if not os.path.exists(os.path.join(self.cache_base_path, data)): self.fail("File not stored in local cache.") elif io.open(os.path.join(self.cache_base_path, data), "rb").read() != self.content: self.fail("Local cache's content differ " "from original file.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data # Retrieve the file as a string. self.fake_content = "Fake content.\n" with io.open(self.cache_path, "wb") as cached_file: cached_file.write(self.fake_content) try: data = self.file_cacher.get_file_content(self.digest) except Exception as error: self.fail("Error received: %r." % error) return if data != self.fake_content: if data == self.content: self.fail("Did not use the cache even if it could.") else: self.fail("Content differ.") def test_big_file(self): """Put a ~10MB file into the storage (using a specially crafted file-like object). Then get it back. """ rand_file = RandomFile(10000000) try: data = self.file_cacher.put_file_from_fobj(rand_file, u"Test #007") except Exception as error: self.fail("Error received: %r." % error) return if rand_file.dim != 0: self.fail("The input file wasn't read completely.") my_digest = rand_file.digest rand_file.close() if not os.path.exists(os.path.join(self.cache_base_path, data)): self.fail("File not stored in local cache.") elif my_digest != data: self.fail("File received with wrong hash.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data # Get the ~100MB file from FileCacher. os.unlink(self.cache_path) hash_file = HashingFile() try: self.file_cacher.get_file_to_fobj(self.digest, hash_file) except Exception as error: self.fail("Error received: %r." % error) return my_digest = hash_file.digest hash_file.close() try: if self.digest != my_digest: self.fail("Content differs.") elif not os.path.exists(self.cache_path): self.fail("File not stored in local cache.") finally: self.file_cacher.delete(self.digest)
class TestFileCacher(TestService): """Service that performs automatically some tests for the FileCacher service. """ def __init__(self, shard): logger.initialize(ServiceCoord("TestFileCacher", shard)) TestService.__init__(self, shard, custom_logger=logger) # Assume we store the cache in "./cache/fs-cache-TestFileCacher-0/" self.cache_base_path = os.path.join(config.cache_dir, "fs-cache-TestFileCacher-0") self.cache_path = None self.content = None self.fake_content = None self.digest = None self.file_obj = None self.file_cacher = FileCacher(self) #self.file_cacher = FileCacher(self, path="fs-storage") def prepare(self): """Initialization for the test code - make sure that the cache is empty before testing. """ logger.info("Please delete directory %s before." % self.cache_base_path) ### TEST 000 ### def test_000(self): """Send a ~100B random binary file to the storage through FileCacher as a file-like object. FC should cache the content locally. """ self.size = 100 self.content = "".join( chr(random.randint(0, 255)) for unused_i in xrange(self.size)) logger.info(" I am sending the ~100B binary file to FileCacher") try: data = self.file_cacher.put_file_from_fobj(StringIO(self.content), u"Test #000") except Exception as error: self.test_end(False, "Error received: %r." % error) return if not os.path.exists(os.path.join(self.cache_base_path, data)): self.test_end(False, "File not stored in local cache.") elif open(os.path.join(self.cache_base_path, data), "rb").read() != \ self.content: self.test_end( False, "Local cache's content differ " "from original file.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data self.test_end(True, "Data sent and cached without error.") ### TEST 001 ### def test_001(self): """Retrieve the file. """ logger.info(" I am retrieving the ~100B binary file from FileCacher") self.fake_content = "Fake content.\n" with open(self.cache_path, "wb") as cached_file: cached_file.write(self.fake_content) try: data = self.file_cacher.get_file(self.digest) except Exception as error: self.test_end(False, "Error received: %r." % error) return received = data.read() data.close() if received != self.fake_content: if received == self.content: self.test_end(False, "Did not use the cache even if it could.") else: self.test_end(False, "Content differ.") else: self.test_end(True, "Data object received correctly.") ### TEST 002 ### def test_002(self): """Check the size of the file. """ logger.info(" I am checking the size of the ~100B binary file") try: size = self.file_cacher.get_size(self.digest) except Exception as error: self.test_end(False, "Error received: %r." % error) return if size == self.size: self.test_end(True, "The size is correct.") else: self.test_end( False, "The size is wrong: %d instead of %d" % (size, self.size)) ### TEST 003 ### def test_003(self): """Get file from FileCacher. """ logger.info(" I am retrieving the file from FileCacher " + "after deleting the cache.") os.unlink(self.cache_path) try: data = self.file_cacher.get_file(self.digest) except Exception as error: self.test_end(False, "Error received: %r." % error) return received = data.read() data.close() if received != self.content: self.test_end(False, "Content differ.") elif not os.path.exists(self.cache_path): self.test_end(False, "File not stored in local cache.") elif open(self.cache_path).read() != self.content: self.test_end( False, "Local cache's content differ " + "from original file.") else: self.test_end(True, "Content object received " + "and cached correctly.") ### TEST 004 ### def test_004(self): """Delete the file through FS and tries to get it again through FC. """ logger.info(" I am deleting the file from FileCacher.") try: self.file_cacher.delete(digest=self.digest) except Exception as error: self.test_end(False, "Error received: %s." % error) return else: logger.info(" File deleted correctly.") logger.info(" I am getting the file from FileCacher.") try: self.file_cacher.get_file(self.digest) except Exception as error: self.test_end(True, "Correctly received an error: %r." % error) else: self.test_end(False, "Did not receive error.") ### TEST 005 ### def test_005(self): """Get unexisting file from FileCacher. """ logger.info(" I am retrieving an unexisting file from FileCacher.") try: self.file_cacher.get_file(self.digest) except Exception as error: self.test_end(True, "Correctly received an error: %r." % error) else: self.test_end(False, "Did not receive error.") ### TEST 006 ### def test_006(self): """Send a ~100B random binary file to the storage through FileCacher as a string. FC should cache the content locally. """ self.content = "".join( chr(random.randint(0, 255)) for unused_i in xrange(100)) logger.info(" I am sending the ~100B binary file to FileCacher") try: data = self.file_cacher.put_file_content(self.content, u"Test #005") except Exception as error: self.test_end(False, "Error received: %r." % error) return if not os.path.exists(os.path.join(self.cache_base_path, data)): self.test_end(False, "File not stored in local cache.") elif open(os.path.join(self.cache_base_path, data), "rb").read() != self.content: self.test_end( False, "Local cache's content differ " "from original file.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data self.test_end(True, "Data sent and cached without error.") ### TEST 007 ### def test_007(self): """Retrieve the file as a string. """ logger.info(" I am retrieving the ~100B binary file from FileCacher " "using get_file_to_string()") self.fake_content = "Fake content.\n" with open(self.cache_path, "wb") as cached_file: cached_file.write(self.fake_content) try: data = self.file_cacher.get_file_content(self.digest) except Exception as error: self.test_end(False, "Error received: %r." % error) return if data != self.fake_content: if data == self.content: self.test_end(False, "Did not use the cache even if it could.") else: self.test_end(False, "Content differ.") else: self.test_end(True, "Data received correctly.") ### TEST 008 ### def test_008(self): """Put a ~100MB file into the storage (using a specially crafted file-like object). """ logger.info(" I am sending the ~100MB binary file to FileCacher") rand_file = RandomFile(100000000) try: data = self.file_cacher.put_file_from_fobj(rand_file, u"Test #007") except Exception as error: self.test_end(False, "Error received: %r." % error) return if rand_file.dim != 0: self.test_end(False, "The input file wasn't read completely.") my_digest = rand_file.digest rand_file.close() if not os.path.exists(os.path.join(self.cache_base_path, data)): self.test_end(False, "File not stored in local cache.") elif my_digest != data: self.test_end(False, "File received with wrong hash.") else: self.cache_path = os.path.join(self.cache_base_path, data) self.digest = data self.test_end(True, "Data sent and cached without error.") ### TEST 009 ### def test_009(self): """Get the ~100MB file from FileCacher. """ logger.info(" I am retrieving the ~100MB file from FileCacher " + "after deleting the cache.") os.unlink(self.cache_path) hash_file = HashingFile() try: self.file_cacher.get_file_to_fobj(self.digest, hash_file) except Exception as error: self.test_end(False, "Error received: %r." % error) return my_digest = hash_file.digest hash_file.close() try: if self.digest != my_digest: self.test_end(False, "Content differs.") elif not os.path.exists(self.cache_path): self.test_end(False, "File not stored in local cache.") else: self.test_end( True, "Content object received " + "and cached correctly.") finally: self.file_cacher.delete(self.digest)