def clean_files(session, dry_run):
filecacher = FileCacher()
files = set(file[0] for file in filecacher.list())
logger.info("A total number of %d files are present in the file store",
len(files))
for cls in [
Attachment, Executable, File, Manager, PrintJob, Statement,
Testcase, UserTest, UserTestExecutable, UserTestFile,
UserTestManager, UserTestResult
]:
for col in ["input", "output", "digest"]:
if hasattr(cls, col):
found_digests = set()
digests = session.query(cls).all()
digests = [getattr(obj, col) for obj in digests]
found_digests |= set(digests)
found_digests.discard(FileCacher.TOMBSTONE_DIGEST)
logger.info("Found %d digests while scanning %s.%s",
len(found_digests), cls.__name__, col)
files -= found_digests
logger.info("%d digests are orphan.", len(files))
total_size = 0
for orphan in files:
total_size += filecacher.get_size(orphan)
logger.info("Orphan files take %s bytes of disk space",
"{:,}".format(total_size))
if not dry_run:
for count, orphan in enumerate(files):
filecacher.delete(orphan)
if count % 100 == 0:
logger.info("%d files deleted from the file store", count)
logger.info("All orphan files have been deleted")
def clean_files(session, dry_run):
filecacher = FileCacher()
files = set(file[0] for file in filecacher.list())
logger.info("A total number of %d files are present in the file store",
len(files))
for cls in [Attachment, Executable, File, Manager, PrintJob,
Statement, Testcase, UserTest, UserTestExecutable,
UserTestFile, UserTestManager, UserTestResult]:
for col in ["input", "output", "digest"]:
if hasattr(cls, col):
found_digests = set()
digests = session.query(cls).all()
digests = [getattr(obj, col) for obj in digests]
found_digests |= set(digests)
found_digests.discard(FileCacher.TOMBSTONE_DIGEST)
logger.info("Found %d digests while scanning %s.%s",
len(found_digests), cls.__name__, col)
files -= found_digests
logger.info("%d digests are orphan.", len(files))
total_size = 0
for orphan in files:
total_size += filecacher.get_size(orphan)
logger.info("Orphan files take %s bytes of disk space",
"{:,}".format(total_size))
if not dry_run:
for count, orphan in enumerate(files):
filecacher.delete(orphan)
if count % 100 == 0:
logger.info("%d files deleted from the file store", count)
logger.info("All orphan files have been deleted")
def clean_files(session, dry_run):
filecacher = FileCacher()
files = set(file[0] for file in filecacher.list())
logger.info("A total number of %d files are present in the file store",
len(files))
found_digests = enumerate_files(session)
logger.info("Found %d digests while scanning", len(found_digests))
files -= found_digests
logger.info("%d digests are orphan.", len(files))
total_size = 0
for orphan in files:
total_size += filecacher.get_size(orphan)
logger.info("Orphan files take %s bytes of disk space",
"{:,}".format(total_size))
if not dry_run:
for count, orphan in enumerate(files):
filecacher.delete(orphan)
if count % 100 == 0:
logger.info("%d files deleted from the file store", count)
logger.info("All orphan files have been deleted")
def clean_files(session, dry_run):
filecacher = FileCacher()
files = set(file[0] for file in filecacher.list())
logger.info("A total number of %d files are present in the file store",
len(files))
found_digests = enumerate_files(session)
logger.info("Found %d digests while scanning", len(found_digests))
files -= found_digests
logger.info("%d digests are orphan.", len(files))
total_size = 0
for orphan in files:
total_size += filecacher.get_size(orphan)
logger.info("Orphan files take %s bytes of disk space",
"{:,}".format(total_size))
if not dry_run:
for count, orphan in enumerate(files):
filecacher.delete(orphan)
if count % 100 == 0:
logger.info("%d files deleted from the file store", count)
logger.info("All orphan files have been deleted")
class TestFileCacher(TestService):
"""Service that performs automatically some tests for the
FileCacher service.
"""
def __init__(self, shard):
logger.initialize(ServiceCoord("TestFileCacher", shard))
TestService.__init__(self, shard, custom_logger=logger)
# Assume we store the cache in "./cache/fs-cache-TestFileCacher-0/"
self.cache_base_path = os.path.join(config.cache_dir,
"fs-cache-TestFileCacher-0")
self.cache_path = None
self.content = None
self.fake_content = None
self.digest = None
self.file_obj = None
self.file_cacher = FileCacher(self)
#self.file_cacher = FileCacher(self, path="fs-storage")
def prepare(self):
"""Initialization for the test code - make sure that the cache
is empty before testing.
"""
logger.info("Please delete directory %s before." %
self.cache_base_path)
### TEST 000 ###
def test_000(self):
"""Send a ~100B random binary file to the storage through
FileCacher as a file-like object. FC should cache the content
locally.
"""
self.size = 100
self.content = "".join(chr(random.randint(0, 255))
for unused_i in xrange(self.size))
logger.info(" I am sending the ~100B binary file to FileCacher")
try:
data = self.file_cacher.put_file_from_fobj(StringIO(self.content),
u"Test #000")
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
if not os.path.exists(os.path.join(self.cache_base_path, data)):
self.test_end(False, "File not stored in local cache.")
elif open(os.path.join(self.cache_base_path, data), "rb").read() != \
self.content:
self.test_end(False, "Local cache's content differ "
"from original file.")
else:
self.cache_path = os.path.join(self.cache_base_path, data)
self.digest = data
self.test_end(True, "Data sent and cached without error.")
### TEST 001 ###
def test_001(self):
"""Retrieve the file.
"""
logger.info(" I am retrieving the ~100B binary file from FileCacher")
self.fake_content = "Fake content.\n"
with open(self.cache_path, "wb") as cached_file:
cached_file.write(self.fake_content)
try:
data = self.file_cacher.get_file(self.digest)
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
received = data.read()
data.close()
if received != self.fake_content:
if received == self.content:
self.test_end(False,
"Did not use the cache even if it could.")
else:
self.test_end(False, "Content differ.")
else:
self.test_end(True, "Data object received correctly.")
### TEST 002 ###
def test_002(self):
"""Check the size of the file.
"""
logger.info(" I am checking the size of the ~100B binary file")
try:
size = self.file_cacher.get_size(self.digest)
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
if size == self.size:
self.test_end(True, "The size is correct.")
else:
self.test_end(False, "The size is wrong: %d instead of %d" %
(size, self.size))
### TEST 003 ###
def test_003(self):
"""Get file from FileCacher.
"""
logger.info(" I am retrieving the file from FileCacher " +
"after deleting the cache.")
os.unlink(self.cache_path)
try:
data = self.file_cacher.get_file(self.digest)
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
received = data.read()
data.close()
if received != self.content:
self.test_end(False, "Content differ.")
elif not os.path.exists(self.cache_path):
self.test_end(False, "File not stored in local cache.")
elif open(self.cache_path).read() != self.content:
self.test_end(False, "Local cache's content differ " +
"from original file.")
else:
self.test_end(True, "Content object received " +
"and cached correctly.")
### TEST 004 ###
def test_004(self):
"""Delete the file through FS and tries to get it again through FC.
"""
logger.info(" I am deleting the file from FileCacher.")
try:
self.file_cacher.delete(digest=self.digest)
except Exception as error:
self.test_end(False, "Error received: %s." % error)
return
else:
logger.info(" File deleted correctly.")
logger.info(" I am getting the file from FileCacher.")
try:
self.file_cacher.get_file(self.digest)
except Exception as error:
self.test_end(True, "Correctly received an error: %r." % error)
else:
self.test_end(False, "Did not receive error.")
### TEST 005 ###
def test_005(self):
"""Get unexisting file from FileCacher.
"""
logger.info(" I am retrieving an unexisting file from FileCacher.")
try:
self.file_cacher.get_file(self.digest)
except Exception as error:
self.test_end(True, "Correctly received an error: %r." % error)
else:
self.test_end(False, "Did not receive error.")
### TEST 006 ###
def test_006(self):
"""Send a ~100B random binary file to the storage through
FileCacher as a string. FC should cache the content locally.
"""
self.content = "".join(chr(random.randint(0, 255))
for unused_i in xrange(100))
logger.info(" I am sending the ~100B binary file to FileCacher")
try:
data = self.file_cacher.put_file_content(self.content,
u"Test #005")
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
if not os.path.exists(os.path.join(self.cache_base_path, data)):
self.test_end(False, "File not stored in local cache.")
elif open(os.path.join(self.cache_base_path, data),
"rb").read() != self.content:
self.test_end(False, "Local cache's content differ "
"from original file.")
else:
self.cache_path = os.path.join(self.cache_base_path, data)
self.digest = data
self.test_end(True, "Data sent and cached without error.")
### TEST 007 ###
def test_007(self):
"""Retrieve the file as a string.
"""
logger.info(" I am retrieving the ~100B binary file from FileCacher "
"using get_file_to_string()")
self.fake_content = "Fake content.\n"
with open(self.cache_path, "wb") as cached_file:
cached_file.write(self.fake_content)
try:
data = self.file_cacher.get_file_content(self.digest)
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
if data != self.fake_content:
if data == self.content:
self.test_end(False,
"Did not use the cache even if it could.")
else:
self.test_end(False, "Content differ.")
else:
self.test_end(True, "Data received correctly.")
### TEST 008 ###
def test_008(self):
"""Put a ~100MB file into the storage (using a specially
crafted file-like object).
"""
logger.info(" I am sending the ~100MB binary file to FileCacher")
rand_file = RandomFile(100000000)
try:
data = self.file_cacher.put_file_from_fobj(rand_file, u"Test #007")
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
if rand_file.dim != 0:
self.test_end(False, "The input file wasn't read completely.")
my_digest = rand_file.digest
rand_file.close()
if not os.path.exists(os.path.join(self.cache_base_path, data)):
self.test_end(False, "File not stored in local cache.")
elif my_digest != data:
self.test_end(False, "File received with wrong hash.")
else:
self.cache_path = os.path.join(self.cache_base_path, data)
self.digest = data
self.test_end(True, "Data sent and cached without error.")
### TEST 009 ###
def test_009(self):
"""Get the ~100MB file from FileCacher.
"""
logger.info(" I am retrieving the ~100MB file from FileCacher " +
"after deleting the cache.")
os.unlink(self.cache_path)
hash_file = HashingFile()
try:
self.file_cacher.get_file_to_fobj(self.digest, hash_file)
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
my_digest = hash_file.digest
hash_file.close()
try:
if self.digest != my_digest:
self.test_end(False, "Content differs.")
elif not os.path.exists(self.cache_path):
self.test_end(False, "File not stored in local cache.")
else:
self.test_end(True, "Content object received " +
"and cached correctly.")
finally:
self.file_cacher.delete(self.digest)
class TestFileCacher(unittest.TestCase):
"""Service that performs automatically some tests for the
FileCacher service.
"""
def setUp(self):
self.file_cacher = FileCacher()
#self.file_cacher = FileCacher(self, path="fs-storage")
self.cache_base_path = self.file_cacher.file_dir
self.cache_path = None
self.content = None
self.fake_content = None
self.digest = None
self.file_obj = None
def tearDown(self):
shutil.rmtree(self.cache_base_path, ignore_errors=True)
def test_file_life(self):
"""Send a ~100B random binary file to the storage through
FileCacher as a file-like object. FC should cache the content
locally.
Then retrieve it.
Then check its size.
Then get it back.
Then delete it.
"""
self.size = 100
self.content = b"".join(
chr(random.randint(0, 255)) for unused_i in xrange(self.size))
data = self.file_cacher.put_file_from_fobj(StringIO(self.content),
u"Test #000")
if not os.path.exists(os.path.join(self.cache_base_path, data)):
self.fail("File not stored in local cache.")
elif io.open(os.path.join(self.cache_base_path, data),
"rb").read() != self.content:
self.fail("Local cache's content differ " "from original file.")
else:
self.cache_path = os.path.join(self.cache_base_path, data)
self.digest = data
# Retrieve the file.
self.fake_content = "Fake content.\n"
with io.open(self.cache_path, "wb") as cached_file:
cached_file.write(self.fake_content)
try:
data = self.file_cacher.get_file(self.digest)
except Exception as error:
self.fail("Error received: %r." % error)
return
received = data.read()
data.close()
if received != self.fake_content:
if received == self.content:
self.fail("Did not use the cache even if it could.")
else:
self.fail("Content differ.")
# Check the size of the file.
try:
size = self.file_cacher.get_size(self.digest)
except Exception as error:
self.fail("Error received: %r." % error)
return
if size != self.size:
self.fail("The size is wrong: %d instead of %d" %
(size, self.size))
# Get file from FileCacher.
os.unlink(self.cache_path)
try:
data = self.file_cacher.get_file(self.digest)
except Exception as error:
self.fail("Error received: %r." % error)
return
received = data.read()
data.close()
if received != self.content:
self.fail("Content differ.")
elif not os.path.exists(self.cache_path):
self.fail("File not stored in local cache.")
elif io.open(self.cache_path, "rb").read() != self.content:
self.fail("Local cache's content differ " + "from original file.")
# Delete the file through FS and tries to get it again through
# FC.
try:
self.file_cacher.delete(digest=self.digest)
except Exception as error:
self.fail("Error received: %s." % error)
return
else:
with self.assertRaises(Exception):
self.file_cacher.get_file(self.digest)
def test_fetch_missing_file(self):
"""Get unexisting file from FileCacher.
"""
with self.assertRaises(Exception):
self.file_cacher.get_file(self.digest)
def test_file_as_content(self):
"""Send a ~100B random binary file to the storage through
FileCacher as a string. FC should cache the content locally.
Then retrieve it as a string.
"""
self.content = b"".join(
chr(random.randint(0, 255)) for unused_i in xrange(100))
try:
data = self.file_cacher.put_file_content(self.content,
u"Test #005")
except Exception as error:
self.fail("Error received: %r." % error)
return
if not os.path.exists(os.path.join(self.cache_base_path, data)):
self.fail("File not stored in local cache.")
elif io.open(os.path.join(self.cache_base_path, data),
"rb").read() != self.content:
self.fail("Local cache's content differ " "from original file.")
else:
self.cache_path = os.path.join(self.cache_base_path, data)
self.digest = data
# Retrieve the file as a string.
self.fake_content = "Fake content.\n"
with io.open(self.cache_path, "wb") as cached_file:
cached_file.write(self.fake_content)
try:
data = self.file_cacher.get_file_content(self.digest)
except Exception as error:
self.fail("Error received: %r." % error)
return
if data != self.fake_content:
if data == self.content:
self.fail("Did not use the cache even if it could.")
else:
self.fail("Content differ.")
def test_big_file(self):
"""Put a ~10MB file into the storage (using a specially
crafted file-like object).
Then get it back.
"""
rand_file = RandomFile(10000000)
try:
data = self.file_cacher.put_file_from_fobj(rand_file, u"Test #007")
except Exception as error:
self.fail("Error received: %r." % error)
return
if rand_file.dim != 0:
self.fail("The input file wasn't read completely.")
my_digest = rand_file.digest
rand_file.close()
if not os.path.exists(os.path.join(self.cache_base_path, data)):
self.fail("File not stored in local cache.")
elif my_digest != data:
self.fail("File received with wrong hash.")
else:
self.cache_path = os.path.join(self.cache_base_path, data)
self.digest = data
# Get the ~100MB file from FileCacher.
os.unlink(self.cache_path)
hash_file = HashingFile()
try:
self.file_cacher.get_file_to_fobj(self.digest, hash_file)
except Exception as error:
self.fail("Error received: %r." % error)
return
my_digest = hash_file.digest
hash_file.close()
try:
if self.digest != my_digest:
self.fail("Content differs.")
elif not os.path.exists(self.cache_path):
self.fail("File not stored in local cache.")
finally:
self.file_cacher.delete(self.digest)
class TestFileCacher(unittest.TestCase):
"""Service that performs automatically some tests for the
FileCacher service.
"""
def setUp(self):
self.file_cacher = FileCacher()
#self.file_cacher = FileCacher(self, path="fs-storage")
self.cache_base_path = self.file_cacher.file_dir
self.cache_path = None
self.content = None
self.fake_content = None
self.digest = None
self.file_obj = None
def tearDown(self):
shutil.rmtree(self.cache_base_path, ignore_errors=True)
def test_file_life(self):
"""Send a ~100B random binary file to the storage through
FileCacher as a file-like object. FC should cache the content
locally.
Then retrieve it.
Then check its size.
Then get it back.
Then delete it.
"""
self.size = 100
self.content = b"".join(chr(random.randint(0, 255))
for unused_i in xrange(self.size))
data = self.file_cacher.put_file_from_fobj(StringIO(self.content),
u"Test #000")
if not os.path.exists(os.path.join(self.cache_base_path, data)):
self.fail("File not stored in local cache.")
elif io.open(os.path.join(self.cache_base_path, data),
"rb").read() != self.content:
self.fail("Local cache's content differ "
"from original file.")
else:
self.cache_path = os.path.join(self.cache_base_path, data)
self.digest = data
# Retrieve the file.
self.fake_content = "Fake content.\n"
with io.open(self.cache_path, "wb") as cached_file:
cached_file.write(self.fake_content)
try:
data = self.file_cacher.get_file(self.digest)
except Exception as error:
self.fail("Error received: %r." % error)
return
received = data.read()
data.close()
if received != self.fake_content:
if received == self.content:
self.fail("Did not use the cache even if it could.")
else:
self.fail("Content differ.")
# Check the size of the file.
try:
size = self.file_cacher.get_size(self.digest)
except Exception as error:
self.fail("Error received: %r." % error)
return
if size != self.size:
self.fail("The size is wrong: %d instead of %d" %
(size, self.size))
# Get file from FileCacher.
os.unlink(self.cache_path)
try:
data = self.file_cacher.get_file(self.digest)
except Exception as error:
self.fail("Error received: %r." % error)
return
received = data.read()
data.close()
if received != self.content:
self.fail("Content differ.")
elif not os.path.exists(self.cache_path):
self.fail("File not stored in local cache.")
elif io.open(self.cache_path, "rb").read() != self.content:
self.fail("Local cache's content differ " +
"from original file.")
# Delete the file through FS and tries to get it again through
# FC.
try:
self.file_cacher.delete(digest=self.digest)
except Exception as error:
self.fail("Error received: %s." % error)
return
else:
with self.assertRaises(Exception):
self.file_cacher.get_file(self.digest)
def test_fetch_missing_file(self):
"""Get unexisting file from FileCacher.
"""
with self.assertRaises(Exception):
self.file_cacher.get_file(self.digest)
def test_file_as_content(self):
"""Send a ~100B random binary file to the storage through
FileCacher as a string. FC should cache the content locally.
Then retrieve it as a string.
"""
self.content = b"".join(chr(random.randint(0, 255))
for unused_i in xrange(100))
try:
data = self.file_cacher.put_file_content(self.content,
u"Test #005")
except Exception as error:
self.fail("Error received: %r." % error)
return
if not os.path.exists(os.path.join(self.cache_base_path, data)):
self.fail("File not stored in local cache.")
elif io.open(os.path.join(self.cache_base_path, data),
"rb").read() != self.content:
self.fail("Local cache's content differ "
"from original file.")
else:
self.cache_path = os.path.join(self.cache_base_path, data)
self.digest = data
# Retrieve the file as a string.
self.fake_content = "Fake content.\n"
with io.open(self.cache_path, "wb") as cached_file:
cached_file.write(self.fake_content)
try:
data = self.file_cacher.get_file_content(self.digest)
except Exception as error:
self.fail("Error received: %r." % error)
return
if data != self.fake_content:
if data == self.content:
self.fail("Did not use the cache even if it could.")
else:
self.fail("Content differ.")
def test_big_file(self):
"""Put a ~10MB file into the storage (using a specially
crafted file-like object).
Then get it back.
"""
rand_file = RandomFile(10000000)
try:
data = self.file_cacher.put_file_from_fobj(rand_file, u"Test #007")
except Exception as error:
self.fail("Error received: %r." % error)
return
if rand_file.dim != 0:
self.fail("The input file wasn't read completely.")
my_digest = rand_file.digest
rand_file.close()
if not os.path.exists(os.path.join(self.cache_base_path, data)):
self.fail("File not stored in local cache.")
elif my_digest != data:
self.fail("File received with wrong hash.")
else:
self.cache_path = os.path.join(self.cache_base_path, data)
self.digest = data
# Get the ~100MB file from FileCacher.
os.unlink(self.cache_path)
hash_file = HashingFile()
try:
self.file_cacher.get_file_to_fobj(self.digest, hash_file)
except Exception as error:
self.fail("Error received: %r." % error)
return
my_digest = hash_file.digest
hash_file.close()
try:
if self.digest != my_digest:
self.fail("Content differs.")
elif not os.path.exists(self.cache_path):
self.fail("File not stored in local cache.")
finally:
self.file_cacher.delete(self.digest)
class TestFileCacher(TestService):
"""Service that performs automatically some tests for the
FileCacher service.
"""
def __init__(self, shard):
logger.initialize(ServiceCoord("TestFileCacher", shard))
TestService.__init__(self, shard, custom_logger=logger)
# Assume we store the cache in "./cache/fs-cache-TestFileCacher-0/"
self.cache_base_path = os.path.join(config.cache_dir,
"fs-cache-TestFileCacher-0")
self.cache_path = None
self.content = None
self.fake_content = None
self.digest = None
self.file_obj = None
self.file_cacher = FileCacher(self)
#self.file_cacher = FileCacher(self, path="fs-storage")
def prepare(self):
"""Initialization for the test code - make sure that the cache
is empty before testing.
"""
logger.info("Please delete directory %s before." %
self.cache_base_path)
### TEST 000 ###
def test_000(self):
"""Send a ~100B random binary file to the storage through
FileCacher as a file-like object. FC should cache the content
locally.
"""
self.size = 100
self.content = "".join(
chr(random.randint(0, 255)) for unused_i in xrange(self.size))
logger.info(" I am sending the ~100B binary file to FileCacher")
try:
data = self.file_cacher.put_file_from_fobj(StringIO(self.content),
u"Test #000")
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
if not os.path.exists(os.path.join(self.cache_base_path, data)):
self.test_end(False, "File not stored in local cache.")
elif open(os.path.join(self.cache_base_path, data), "rb").read() != \
self.content:
self.test_end(
False, "Local cache's content differ "
"from original file.")
else:
self.cache_path = os.path.join(self.cache_base_path, data)
self.digest = data
self.test_end(True, "Data sent and cached without error.")
### TEST 001 ###
def test_001(self):
"""Retrieve the file.
"""
logger.info(" I am retrieving the ~100B binary file from FileCacher")
self.fake_content = "Fake content.\n"
with open(self.cache_path, "wb") as cached_file:
cached_file.write(self.fake_content)
try:
data = self.file_cacher.get_file(self.digest)
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
received = data.read()
data.close()
if received != self.fake_content:
if received == self.content:
self.test_end(False, "Did not use the cache even if it could.")
else:
self.test_end(False, "Content differ.")
else:
self.test_end(True, "Data object received correctly.")
### TEST 002 ###
def test_002(self):
"""Check the size of the file.
"""
logger.info(" I am checking the size of the ~100B binary file")
try:
size = self.file_cacher.get_size(self.digest)
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
if size == self.size:
self.test_end(True, "The size is correct.")
else:
self.test_end(
False,
"The size is wrong: %d instead of %d" % (size, self.size))
### TEST 003 ###
def test_003(self):
"""Get file from FileCacher.
"""
logger.info(" I am retrieving the file from FileCacher " +
"after deleting the cache.")
os.unlink(self.cache_path)
try:
data = self.file_cacher.get_file(self.digest)
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
received = data.read()
data.close()
if received != self.content:
self.test_end(False, "Content differ.")
elif not os.path.exists(self.cache_path):
self.test_end(False, "File not stored in local cache.")
elif open(self.cache_path).read() != self.content:
self.test_end(
False, "Local cache's content differ " + "from original file.")
else:
self.test_end(True,
"Content object received " + "and cached correctly.")
### TEST 004 ###
def test_004(self):
"""Delete the file through FS and tries to get it again through FC.
"""
logger.info(" I am deleting the file from FileCacher.")
try:
self.file_cacher.delete(digest=self.digest)
except Exception as error:
self.test_end(False, "Error received: %s." % error)
return
else:
logger.info(" File deleted correctly.")
logger.info(" I am getting the file from FileCacher.")
try:
self.file_cacher.get_file(self.digest)
except Exception as error:
self.test_end(True, "Correctly received an error: %r." % error)
else:
self.test_end(False, "Did not receive error.")
### TEST 005 ###
def test_005(self):
"""Get unexisting file from FileCacher.
"""
logger.info(" I am retrieving an unexisting file from FileCacher.")
try:
self.file_cacher.get_file(self.digest)
except Exception as error:
self.test_end(True, "Correctly received an error: %r." % error)
else:
self.test_end(False, "Did not receive error.")
### TEST 006 ###
def test_006(self):
"""Send a ~100B random binary file to the storage through
FileCacher as a string. FC should cache the content locally.
"""
self.content = "".join(
chr(random.randint(0, 255)) for unused_i in xrange(100))
logger.info(" I am sending the ~100B binary file to FileCacher")
try:
data = self.file_cacher.put_file_content(self.content,
u"Test #005")
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
if not os.path.exists(os.path.join(self.cache_base_path, data)):
self.test_end(False, "File not stored in local cache.")
elif open(os.path.join(self.cache_base_path, data),
"rb").read() != self.content:
self.test_end(
False, "Local cache's content differ "
"from original file.")
else:
self.cache_path = os.path.join(self.cache_base_path, data)
self.digest = data
self.test_end(True, "Data sent and cached without error.")
### TEST 007 ###
def test_007(self):
"""Retrieve the file as a string.
"""
logger.info(" I am retrieving the ~100B binary file from FileCacher "
"using get_file_to_string()")
self.fake_content = "Fake content.\n"
with open(self.cache_path, "wb") as cached_file:
cached_file.write(self.fake_content)
try:
data = self.file_cacher.get_file_content(self.digest)
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
if data != self.fake_content:
if data == self.content:
self.test_end(False, "Did not use the cache even if it could.")
else:
self.test_end(False, "Content differ.")
else:
self.test_end(True, "Data received correctly.")
### TEST 008 ###
def test_008(self):
"""Put a ~100MB file into the storage (using a specially
crafted file-like object).
"""
logger.info(" I am sending the ~100MB binary file to FileCacher")
rand_file = RandomFile(100000000)
try:
data = self.file_cacher.put_file_from_fobj(rand_file, u"Test #007")
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
if rand_file.dim != 0:
self.test_end(False, "The input file wasn't read completely.")
my_digest = rand_file.digest
rand_file.close()
if not os.path.exists(os.path.join(self.cache_base_path, data)):
self.test_end(False, "File not stored in local cache.")
elif my_digest != data:
self.test_end(False, "File received with wrong hash.")
else:
self.cache_path = os.path.join(self.cache_base_path, data)
self.digest = data
self.test_end(True, "Data sent and cached without error.")
### TEST 009 ###
def test_009(self):
"""Get the ~100MB file from FileCacher.
"""
logger.info(" I am retrieving the ~100MB file from FileCacher " +
"after deleting the cache.")
os.unlink(self.cache_path)
hash_file = HashingFile()
try:
self.file_cacher.get_file_to_fobj(self.digest, hash_file)
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
my_digest = hash_file.digest
hash_file.close()
try:
if self.digest != my_digest:
self.test_end(False, "Content differs.")
elif not os.path.exists(self.cache_path):
self.test_end(False, "File not stored in local cache.")
else:
self.test_end(
True, "Content object received " + "and cached correctly.")
finally:
self.file_cacher.delete(self.digest)