class TestFileCacher(TestService):
"""Service that performs automatically some tests for the
FileCacher service.
"""
def __init__(self, shard):
logger.initialize(ServiceCoord("TestFileCacher", shard))
TestService.__init__(self, shard, custom_logger=logger)
# Assume we store the cache in "./cache/fs-cache-TestFileCacher-0/"
self.cache_base_path = os.path.join(config.cache_dir,
"fs-cache-TestFileCacher-0")
self.cache_path = None
self.content = None
self.fake_content = None
self.digest = None
self.file_obj = None
self.file_cacher = FileCacher(self)
#self.file_cacher = FileCacher(self, path="fs-storage")
def prepare(self):
"""Initialization for the test code - make sure that the cache
is empty before testing.
"""
logger.info("Please delete directory %s before." %
self.cache_base_path)
### TEST 000 ###
def test_000(self):
"""Send a ~100B random binary file to the storage through
FileCacher as a file-like object. FC should cache the content
locally.
"""
self.size = 100
self.content = "".join(chr(random.randint(0, 255))
for unused_i in xrange(self.size))
logger.info(" I am sending the ~100B binary file to FileCacher")
try:
data = self.file_cacher.put_file_from_fobj(StringIO(self.content),
u"Test #000")
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
if not os.path.exists(os.path.join(self.cache_base_path, data)):
self.test_end(False, "File not stored in local cache.")
elif open(os.path.join(self.cache_base_path, data), "rb").read() != \
self.content:
self.test_end(False, "Local cache's content differ "
"from original file.")
else:
self.cache_path = os.path.join(self.cache_base_path, data)
self.digest = data
self.test_end(True, "Data sent and cached without error.")
### TEST 001 ###
def test_001(self):
"""Retrieve the file.
"""
logger.info(" I am retrieving the ~100B binary file from FileCacher")
self.fake_content = "Fake content.\n"
with open(self.cache_path, "wb") as cached_file:
cached_file.write(self.fake_content)
try:
data = self.file_cacher.get_file(self.digest)
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
received = data.read()
data.close()
if received != self.fake_content:
if received == self.content:
self.test_end(False,
"Did not use the cache even if it could.")
else:
self.test_end(False, "Content differ.")
else:
self.test_end(True, "Data object received correctly.")
### TEST 002 ###
def test_002(self):
"""Check the size of the file.
"""
logger.info(" I am checking the size of the ~100B binary file")
try:
size = self.file_cacher.get_size(self.digest)
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
if size == self.size:
self.test_end(True, "The size is correct.")
else:
self.test_end(False, "The size is wrong: %d instead of %d" %
(size, self.size))
### TEST 003 ###
def test_003(self):
"""Get file from FileCacher.
"""
logger.info(" I am retrieving the file from FileCacher " +
"after deleting the cache.")
os.unlink(self.cache_path)
try:
data = self.file_cacher.get_file(self.digest)
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
received = data.read()
data.close()
if received != self.content:
self.test_end(False, "Content differ.")
elif not os.path.exists(self.cache_path):
self.test_end(False, "File not stored in local cache.")
elif open(self.cache_path).read() != self.content:
self.test_end(False, "Local cache's content differ " +
"from original file.")
else:
self.test_end(True, "Content object received " +
"and cached correctly.")
### TEST 004 ###
def test_004(self):
"""Delete the file through FS and tries to get it again through FC.
"""
logger.info(" I am deleting the file from FileCacher.")
try:
self.file_cacher.delete(digest=self.digest)
except Exception as error:
self.test_end(False, "Error received: %s." % error)
return
else:
logger.info(" File deleted correctly.")
logger.info(" I am getting the file from FileCacher.")
try:
self.file_cacher.get_file(self.digest)
except Exception as error:
self.test_end(True, "Correctly received an error: %r." % error)
else:
self.test_end(False, "Did not receive error.")
### TEST 005 ###
def test_005(self):
"""Get unexisting file from FileCacher.
"""
logger.info(" I am retrieving an unexisting file from FileCacher.")
try:
self.file_cacher.get_file(self.digest)
except Exception as error:
self.test_end(True, "Correctly received an error: %r." % error)
else:
self.test_end(False, "Did not receive error.")
### TEST 006 ###
def test_006(self):
"""Send a ~100B random binary file to the storage through
FileCacher as a string. FC should cache the content locally.
"""
self.content = "".join(chr(random.randint(0, 255))
for unused_i in xrange(100))
logger.info(" I am sending the ~100B binary file to FileCacher")
try:
data = self.file_cacher.put_file_content(self.content,
u"Test #005")
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
if not os.path.exists(os.path.join(self.cache_base_path, data)):
self.test_end(False, "File not stored in local cache.")
elif open(os.path.join(self.cache_base_path, data),
"rb").read() != self.content:
self.test_end(False, "Local cache's content differ "
"from original file.")
else:
self.cache_path = os.path.join(self.cache_base_path, data)
self.digest = data
self.test_end(True, "Data sent and cached without error.")
### TEST 007 ###
def test_007(self):
"""Retrieve the file as a string.
"""
logger.info(" I am retrieving the ~100B binary file from FileCacher "
"using get_file_to_string()")
self.fake_content = "Fake content.\n"
with open(self.cache_path, "wb") as cached_file:
cached_file.write(self.fake_content)
try:
data = self.file_cacher.get_file_content(self.digest)
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
if data != self.fake_content:
if data == self.content:
self.test_end(False,
"Did not use the cache even if it could.")
else:
self.test_end(False, "Content differ.")
else:
self.test_end(True, "Data received correctly.")
### TEST 008 ###
def test_008(self):
"""Put a ~100MB file into the storage (using a specially
crafted file-like object).
"""
logger.info(" I am sending the ~100MB binary file to FileCacher")
rand_file = RandomFile(100000000)
try:
data = self.file_cacher.put_file_from_fobj(rand_file, u"Test #007")
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
if rand_file.dim != 0:
self.test_end(False, "The input file wasn't read completely.")
my_digest = rand_file.digest
rand_file.close()
if not os.path.exists(os.path.join(self.cache_base_path, data)):
self.test_end(False, "File not stored in local cache.")
elif my_digest != data:
self.test_end(False, "File received with wrong hash.")
else:
self.cache_path = os.path.join(self.cache_base_path, data)
self.digest = data
self.test_end(True, "Data sent and cached without error.")
### TEST 009 ###
def test_009(self):
"""Get the ~100MB file from FileCacher.
"""
logger.info(" I am retrieving the ~100MB file from FileCacher " +
"after deleting the cache.")
os.unlink(self.cache_path)
hash_file = HashingFile()
try:
self.file_cacher.get_file_to_fobj(self.digest, hash_file)
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
my_digest = hash_file.digest
hash_file.close()
try:
if self.digest != my_digest:
self.test_end(False, "Content differs.")
elif not os.path.exists(self.cache_path):
self.test_end(False, "File not stored in local cache.")
else:
self.test_end(True, "Content object received " +
"and cached correctly.")
finally:
self.file_cacher.delete(self.digest)
def main():
"""Parse arguments and launch process.
"""
parser = argparse.ArgumentParser(
description="Export CMS submissions to a folder.\n",
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument("-c",
"--contest-id",
action="store",
type=int,
help="id of contest (default: all contests)")
parser.add_argument("-t",
"--task-id",
action="store",
type=int,
help="id of task (default: all tasks)")
parser.add_argument("-u",
"--user-id",
action="store",
type=int,
help="id of user (default: all users)")
parser.add_argument("-s",
"--submission-id",
action="store",
type=int,
help="id of submission (default: all submissions)")
parser.add_argument("--utf8",
action="store_true",
help="if set, the files will be encoded in utf8"
" when possible")
parser.add_argument("--add-info",
action="store_true",
help="if set, information on the submission will"
" be added in the first lines of each file")
parser.add_argument("--min-score",
action="store",
type=float,
help="ignore submissions which scored strictly"
" less than this (default: 0.0)",
default=0.0)
parser.add_argument("--filename",
action="store",
type=utf8_decoder,
help="the filename format to use\n"
"Variables:\n"
" id: submission id\n"
" file: filename without extension\n"
" ext: filename extension\n"
" time: submission timestamp\n"
" user: username\n"
" task: taskname\n"
" score: raw score\n"
" (default: {id}.{file}{ext})",
default="{id}.{file}{ext}")
parser.add_argument("output_dir",
action="store",
type=utf8_decoder,
help="directory where to save the submissions")
parser.add_argument("-y",
"--yes",
action="store_true",
help="if set, confirmation will not be shown")
group = parser.add_mutually_exclusive_group(required=False)
group.add_argument("--unique",
action="store_true",
help="if set, only the earliest best submission"
" will be exported for each (user, task)")
group.add_argument("--best",
action="store_true",
help="if set, only the best submissions will be"
" exported for each (user, task)")
args = parser.parse_args()
if not os.path.exists(args.output_dir):
os.mkdir(args.output_dir)
if not os.path.isdir(args.output_dir):
logger.critical("The output-dir parameter must point to a directory")
return 1
cacher = FileCacher()
with SessionGen() as session:
q = session.query(Submission)\
.join(Submission.task)\
.join(Submission.files)\
.join(Submission.results)\
.join(SubmissionResult.dataset)\
.join(Submission.participation)\
.join(Participation.user)\
.filter(Dataset.id == Task.active_dataset_id)\
.filter(SubmissionResult.score >= args.min_score)\
.with_entities(Submission.id, Submission.language,
Submission.timestamp,
SubmissionResult.score,
File.filename, File.digest,
User.id, User.username, User.first_name,
User.last_name,
Task.id, Task.name)
if args.contest_id:
q = q.filter(Participation.contest_id == args.contest_id)
if args.task_id:
q = q.filter(Submission.task_id == args.task_id)
if args.user_id:
q = q.filter(Participation.user_id == args.user_id)
if args.submission_id:
q = q.filter(Submission.id == args.submission_id)
results = q.all()
if args.unique or args.best:
results = filter_top_scoring(results, args.unique)
print("%s file(s) will be created." % len(results))
if not args.yes and (input("Continue? [Y/n] ").strip().lower()
not in ["y", ""]):
return 0
done = 0
for row in results:
s_id, s_language, s_timestamp, sr_score, f_filename, f_digest, \
u_id, u_name, u_fname, u_lname, t_id, t_name = row
timef = s_timestamp.strftime('%Y%m%dT%H%M%S')
ext = languagemanager.get_language(s_language).source_extension \
if s_language else '.txt'
filename_base, filename_ext = os.path.splitext(
f_filename.replace('.%l', ext))
# "name" is a deprecated specifier with the same meaning as "file"
filename = args.filename.format(id=s_id,
file=filename_base,
name=filename_base,
ext=filename_ext,
time=timef,
user=u_name,
task=t_name,
score=sr_score)
filename = os.path.join(args.output_dir, filename)
if os.path.exists(filename):
logger.warning("Skipping file '%s' because it already exists",
filename)
continue
filedir = os.path.dirname(filename)
if not os.path.exists(filedir):
os.makedirs(filedir)
if not os.path.isdir(filedir):
logger.warning("%s is not a directory, skipped.", filedir)
continue
if not (args.utf8 or args.add_info):
cacher.get_file_to_path(f_digest, filename)
else:
content_bytes = cacher.get_file_content(f_digest)
if args.utf8:
try:
content = utf8_decoder(content_bytes)
content_bytes = content.encode("utf-8")
except TypeError:
logger.critical(
"Could not guess encoding of file "
"'%s'. Aborting.", filename)
return 1
if args.add_info:
template_str = TEMPLATE[ext] % (u_name, u_fname, u_lname,
t_name, sr_score,
s_timestamp)
template_bytes = template_str.encode("utf-8")
content_bytes = template_bytes + content_bytes
with io.open(filename, 'wb') as f_out:
f_out.write(content_bytes)
done += 1
print(done, "/", len(results))
return 0
class TestFileCacher(unittest.TestCase):
"""Service that performs automatically some tests for the
FileCacher service.
"""
def setUp(self):
self.file_cacher = FileCacher()
#self.file_cacher = FileCacher(self, path="fs-storage")
self.cache_base_path = self.file_cacher.file_dir
self.cache_path = None
self.content = None
self.fake_content = None
self.digest = None
self.file_obj = None
def tearDown(self):
shutil.rmtree(self.cache_base_path, ignore_errors=True)
def test_file_life(self):
"""Send a ~100B random binary file to the storage through
FileCacher as a file-like object. FC should cache the content
locally.
Then retrieve it.
Then check its size.
Then get it back.
Then delete it.
"""
self.size = 100
self.content = b"".join(chr(random.randint(0, 255))
for unused_i in xrange(self.size))
data = self.file_cacher.put_file_from_fobj(StringIO(self.content),
u"Test #000")
if not os.path.exists(os.path.join(self.cache_base_path, data)):
self.fail("File not stored in local cache.")
elif io.open(os.path.join(self.cache_base_path, data),
"rb").read() != self.content:
self.fail("Local cache's content differ "
"from original file.")
else:
self.cache_path = os.path.join(self.cache_base_path, data)
self.digest = data
# Retrieve the file.
self.fake_content = "Fake content.\n"
with io.open(self.cache_path, "wb") as cached_file:
cached_file.write(self.fake_content)
try:
data = self.file_cacher.get_file(self.digest)
except Exception as error:
self.fail("Error received: %r." % error)
return
received = data.read()
data.close()
if received != self.fake_content:
if received == self.content:
self.fail("Did not use the cache even if it could.")
else:
self.fail("Content differ.")
# Check the size of the file.
try:
size = self.file_cacher.get_size(self.digest)
except Exception as error:
self.fail("Error received: %r." % error)
return
if size != self.size:
self.fail("The size is wrong: %d instead of %d" %
(size, self.size))
# Get file from FileCacher.
os.unlink(self.cache_path)
try:
data = self.file_cacher.get_file(self.digest)
except Exception as error:
self.fail("Error received: %r." % error)
return
received = data.read()
data.close()
if received != self.content:
self.fail("Content differ.")
elif not os.path.exists(self.cache_path):
self.fail("File not stored in local cache.")
elif io.open(self.cache_path, "rb").read() != self.content:
self.fail("Local cache's content differ " +
"from original file.")
# Delete the file through FS and tries to get it again through
# FC.
try:
self.file_cacher.delete(digest=self.digest)
except Exception as error:
self.fail("Error received: %s." % error)
return
else:
with self.assertRaises(Exception):
self.file_cacher.get_file(self.digest)
def test_fetch_missing_file(self):
"""Get unexisting file from FileCacher.
"""
with self.assertRaises(Exception):
self.file_cacher.get_file(self.digest)
def test_file_as_content(self):
"""Send a ~100B random binary file to the storage through
FileCacher as a string. FC should cache the content locally.
Then retrieve it as a string.
"""
self.content = b"".join(chr(random.randint(0, 255))
for unused_i in xrange(100))
try:
data = self.file_cacher.put_file_content(self.content,
u"Test #005")
except Exception as error:
self.fail("Error received: %r." % error)
return
if not os.path.exists(os.path.join(self.cache_base_path, data)):
self.fail("File not stored in local cache.")
elif io.open(os.path.join(self.cache_base_path, data),
"rb").read() != self.content:
self.fail("Local cache's content differ "
"from original file.")
else:
self.cache_path = os.path.join(self.cache_base_path, data)
self.digest = data
# Retrieve the file as a string.
self.fake_content = "Fake content.\n"
with io.open(self.cache_path, "wb") as cached_file:
cached_file.write(self.fake_content)
try:
data = self.file_cacher.get_file_content(self.digest)
except Exception as error:
self.fail("Error received: %r." % error)
return
if data != self.fake_content:
if data == self.content:
self.fail("Did not use the cache even if it could.")
else:
self.fail("Content differ.")
def test_big_file(self):
"""Put a ~10MB file into the storage (using a specially
crafted file-like object).
Then get it back.
"""
rand_file = RandomFile(10000000)
try:
data = self.file_cacher.put_file_from_fobj(rand_file, u"Test #007")
except Exception as error:
self.fail("Error received: %r." % error)
return
if rand_file.dim != 0:
self.fail("The input file wasn't read completely.")
my_digest = rand_file.digest
rand_file.close()
if not os.path.exists(os.path.join(self.cache_base_path, data)):
self.fail("File not stored in local cache.")
elif my_digest != data:
self.fail("File received with wrong hash.")
else:
self.cache_path = os.path.join(self.cache_base_path, data)
self.digest = data
# Get the ~100MB file from FileCacher.
os.unlink(self.cache_path)
hash_file = HashingFile()
try:
self.file_cacher.get_file_to_fobj(self.digest, hash_file)
except Exception as error:
self.fail("Error received: %r." % error)
return
my_digest = hash_file.digest
hash_file.close()
try:
if self.digest != my_digest:
self.fail("Content differs.")
elif not os.path.exists(self.cache_path):
self.fail("File not stored in local cache.")
finally:
self.file_cacher.delete(self.digest)
class TestFileCacher(unittest.TestCase):
"""Service that performs automatically some tests for the
FileCacher service.
"""
def setUp(self):
self.file_cacher = FileCacher()
#self.file_cacher = FileCacher(self, path="fs-storage")
self.cache_base_path = self.file_cacher.file_dir
self.cache_path = None
self.content = None
self.fake_content = None
self.digest = None
self.file_obj = None
def tearDown(self):
shutil.rmtree(self.cache_base_path, ignore_errors=True)
def test_file_life(self):
"""Send a ~100B random binary file to the storage through
FileCacher as a file-like object. FC should cache the content
locally.
Then retrieve it.
Then check its size.
Then get it back.
Then delete it.
"""
self.size = 100
self.content = b"".join(
chr(random.randint(0, 255)) for unused_i in xrange(self.size))
data = self.file_cacher.put_file_from_fobj(StringIO(self.content),
u"Test #000")
if not os.path.exists(os.path.join(self.cache_base_path, data)):
self.fail("File not stored in local cache.")
elif io.open(os.path.join(self.cache_base_path, data),
"rb").read() != self.content:
self.fail("Local cache's content differ " "from original file.")
else:
self.cache_path = os.path.join(self.cache_base_path, data)
self.digest = data
# Retrieve the file.
self.fake_content = "Fake content.\n"
with io.open(self.cache_path, "wb") as cached_file:
cached_file.write(self.fake_content)
try:
data = self.file_cacher.get_file(self.digest)
except Exception as error:
self.fail("Error received: %r." % error)
return
received = data.read()
data.close()
if received != self.fake_content:
if received == self.content:
self.fail("Did not use the cache even if it could.")
else:
self.fail("Content differ.")
# Check the size of the file.
try:
size = self.file_cacher.get_size(self.digest)
except Exception as error:
self.fail("Error received: %r." % error)
return
if size != self.size:
self.fail("The size is wrong: %d instead of %d" %
(size, self.size))
# Get file from FileCacher.
os.unlink(self.cache_path)
try:
data = self.file_cacher.get_file(self.digest)
except Exception as error:
self.fail("Error received: %r." % error)
return
received = data.read()
data.close()
if received != self.content:
self.fail("Content differ.")
elif not os.path.exists(self.cache_path):
self.fail("File not stored in local cache.")
elif io.open(self.cache_path, "rb").read() != self.content:
self.fail("Local cache's content differ " + "from original file.")
# Delete the file through FS and tries to get it again through
# FC.
try:
self.file_cacher.delete(digest=self.digest)
except Exception as error:
self.fail("Error received: %s." % error)
return
else:
with self.assertRaises(Exception):
self.file_cacher.get_file(self.digest)
def test_fetch_missing_file(self):
"""Get unexisting file from FileCacher.
"""
with self.assertRaises(Exception):
self.file_cacher.get_file(self.digest)
def test_file_as_content(self):
"""Send a ~100B random binary file to the storage through
FileCacher as a string. FC should cache the content locally.
Then retrieve it as a string.
"""
self.content = b"".join(
chr(random.randint(0, 255)) for unused_i in xrange(100))
try:
data = self.file_cacher.put_file_content(self.content,
u"Test #005")
except Exception as error:
self.fail("Error received: %r." % error)
return
if not os.path.exists(os.path.join(self.cache_base_path, data)):
self.fail("File not stored in local cache.")
elif io.open(os.path.join(self.cache_base_path, data),
"rb").read() != self.content:
self.fail("Local cache's content differ " "from original file.")
else:
self.cache_path = os.path.join(self.cache_base_path, data)
self.digest = data
# Retrieve the file as a string.
self.fake_content = "Fake content.\n"
with io.open(self.cache_path, "wb") as cached_file:
cached_file.write(self.fake_content)
try:
data = self.file_cacher.get_file_content(self.digest)
except Exception as error:
self.fail("Error received: %r." % error)
return
if data != self.fake_content:
if data == self.content:
self.fail("Did not use the cache even if it could.")
else:
self.fail("Content differ.")
def test_big_file(self):
"""Put a ~10MB file into the storage (using a specially
crafted file-like object).
Then get it back.
"""
rand_file = RandomFile(10000000)
try:
data = self.file_cacher.put_file_from_fobj(rand_file, u"Test #007")
except Exception as error:
self.fail("Error received: %r." % error)
return
if rand_file.dim != 0:
self.fail("The input file wasn't read completely.")
my_digest = rand_file.digest
rand_file.close()
if not os.path.exists(os.path.join(self.cache_base_path, data)):
self.fail("File not stored in local cache.")
elif my_digest != data:
self.fail("File received with wrong hash.")
else:
self.cache_path = os.path.join(self.cache_base_path, data)
self.digest = data
# Get the ~100MB file from FileCacher.
os.unlink(self.cache_path)
hash_file = HashingFile()
try:
self.file_cacher.get_file_to_fobj(self.digest, hash_file)
except Exception as error:
self.fail("Error received: %r." % error)
return
my_digest = hash_file.digest
hash_file.close()
try:
if self.digest != my_digest:
self.fail("Content differs.")
elif not os.path.exists(self.cache_path):
self.fail("File not stored in local cache.")
finally:
self.file_cacher.delete(self.digest)
class TestFileCacher(TestService):
"""Service that performs automatically some tests for the
FileCacher service.
"""
def __init__(self, shard):
logger.initialize(ServiceCoord("TestFileCacher", shard))
TestService.__init__(self, shard, custom_logger=logger)
# Assume we store the cache in "./cache/fs-cache-TestFileCacher-0/"
self.cache_base_path = os.path.join(config.cache_dir,
"fs-cache-TestFileCacher-0")
self.cache_path = None
self.content = None
self.fake_content = None
self.digest = None
self.file_obj = None
self.file_cacher = FileCacher(self)
#self.file_cacher = FileCacher(self, path="fs-storage")
def prepare(self):
"""Initialization for the test code - make sure that the cache
is empty before testing.
"""
logger.info("Please delete directory %s before." %
self.cache_base_path)
### TEST 000 ###
def test_000(self):
"""Send a ~100B random binary file to the storage through
FileCacher as a file-like object. FC should cache the content
locally.
"""
self.size = 100
self.content = "".join(
chr(random.randint(0, 255)) for unused_i in xrange(self.size))
logger.info(" I am sending the ~100B binary file to FileCacher")
try:
data = self.file_cacher.put_file_from_fobj(StringIO(self.content),
u"Test #000")
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
if not os.path.exists(os.path.join(self.cache_base_path, data)):
self.test_end(False, "File not stored in local cache.")
elif open(os.path.join(self.cache_base_path, data), "rb").read() != \
self.content:
self.test_end(
False, "Local cache's content differ "
"from original file.")
else:
self.cache_path = os.path.join(self.cache_base_path, data)
self.digest = data
self.test_end(True, "Data sent and cached without error.")
### TEST 001 ###
def test_001(self):
"""Retrieve the file.
"""
logger.info(" I am retrieving the ~100B binary file from FileCacher")
self.fake_content = "Fake content.\n"
with open(self.cache_path, "wb") as cached_file:
cached_file.write(self.fake_content)
try:
data = self.file_cacher.get_file(self.digest)
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
received = data.read()
data.close()
if received != self.fake_content:
if received == self.content:
self.test_end(False, "Did not use the cache even if it could.")
else:
self.test_end(False, "Content differ.")
else:
self.test_end(True, "Data object received correctly.")
### TEST 002 ###
def test_002(self):
"""Check the size of the file.
"""
logger.info(" I am checking the size of the ~100B binary file")
try:
size = self.file_cacher.get_size(self.digest)
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
if size == self.size:
self.test_end(True, "The size is correct.")
else:
self.test_end(
False,
"The size is wrong: %d instead of %d" % (size, self.size))
### TEST 003 ###
def test_003(self):
"""Get file from FileCacher.
"""
logger.info(" I am retrieving the file from FileCacher " +
"after deleting the cache.")
os.unlink(self.cache_path)
try:
data = self.file_cacher.get_file(self.digest)
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
received = data.read()
data.close()
if received != self.content:
self.test_end(False, "Content differ.")
elif not os.path.exists(self.cache_path):
self.test_end(False, "File not stored in local cache.")
elif open(self.cache_path).read() != self.content:
self.test_end(
False, "Local cache's content differ " + "from original file.")
else:
self.test_end(True,
"Content object received " + "and cached correctly.")
### TEST 004 ###
def test_004(self):
"""Delete the file through FS and tries to get it again through FC.
"""
logger.info(" I am deleting the file from FileCacher.")
try:
self.file_cacher.delete(digest=self.digest)
except Exception as error:
self.test_end(False, "Error received: %s." % error)
return
else:
logger.info(" File deleted correctly.")
logger.info(" I am getting the file from FileCacher.")
try:
self.file_cacher.get_file(self.digest)
except Exception as error:
self.test_end(True, "Correctly received an error: %r." % error)
else:
self.test_end(False, "Did not receive error.")
### TEST 005 ###
def test_005(self):
"""Get unexisting file from FileCacher.
"""
logger.info(" I am retrieving an unexisting file from FileCacher.")
try:
self.file_cacher.get_file(self.digest)
except Exception as error:
self.test_end(True, "Correctly received an error: %r." % error)
else:
self.test_end(False, "Did not receive error.")
### TEST 006 ###
def test_006(self):
"""Send a ~100B random binary file to the storage through
FileCacher as a string. FC should cache the content locally.
"""
self.content = "".join(
chr(random.randint(0, 255)) for unused_i in xrange(100))
logger.info(" I am sending the ~100B binary file to FileCacher")
try:
data = self.file_cacher.put_file_content(self.content,
u"Test #005")
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
if not os.path.exists(os.path.join(self.cache_base_path, data)):
self.test_end(False, "File not stored in local cache.")
elif open(os.path.join(self.cache_base_path, data),
"rb").read() != self.content:
self.test_end(
False, "Local cache's content differ "
"from original file.")
else:
self.cache_path = os.path.join(self.cache_base_path, data)
self.digest = data
self.test_end(True, "Data sent and cached without error.")
### TEST 007 ###
def test_007(self):
"""Retrieve the file as a string.
"""
logger.info(" I am retrieving the ~100B binary file from FileCacher "
"using get_file_to_string()")
self.fake_content = "Fake content.\n"
with open(self.cache_path, "wb") as cached_file:
cached_file.write(self.fake_content)
try:
data = self.file_cacher.get_file_content(self.digest)
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
if data != self.fake_content:
if data == self.content:
self.test_end(False, "Did not use the cache even if it could.")
else:
self.test_end(False, "Content differ.")
else:
self.test_end(True, "Data received correctly.")
### TEST 008 ###
def test_008(self):
"""Put a ~100MB file into the storage (using a specially
crafted file-like object).
"""
logger.info(" I am sending the ~100MB binary file to FileCacher")
rand_file = RandomFile(100000000)
try:
data = self.file_cacher.put_file_from_fobj(rand_file, u"Test #007")
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
if rand_file.dim != 0:
self.test_end(False, "The input file wasn't read completely.")
my_digest = rand_file.digest
rand_file.close()
if not os.path.exists(os.path.join(self.cache_base_path, data)):
self.test_end(False, "File not stored in local cache.")
elif my_digest != data:
self.test_end(False, "File received with wrong hash.")
else:
self.cache_path = os.path.join(self.cache_base_path, data)
self.digest = data
self.test_end(True, "Data sent and cached without error.")
### TEST 009 ###
def test_009(self):
"""Get the ~100MB file from FileCacher.
"""
logger.info(" I am retrieving the ~100MB file from FileCacher " +
"after deleting the cache.")
os.unlink(self.cache_path)
hash_file = HashingFile()
try:
self.file_cacher.get_file_to_fobj(self.digest, hash_file)
except Exception as error:
self.test_end(False, "Error received: %r." % error)
return
my_digest = hash_file.digest
hash_file.close()
try:
if self.digest != my_digest:
self.test_end(False, "Content differs.")
elif not os.path.exists(self.cache_path):
self.test_end(False, "File not stored in local cache.")
else:
self.test_end(
True, "Content object received " + "and cached correctly.")
finally:
self.file_cacher.delete(self.digest)