def setUp(self):
pydb_base_dir = tempfile.mkdtemp('test_file_server')
pydbserver = pydb.maindb.build(os.path.join(pydb_base_dir, 'db'),
pydb.testing.guess_schema_dir())
self.file_server = pydb.fileserver.build(os.path.join(pydb_base_dir, 'filestore'), pydbserver)
print "filestore is in ", os.path.join(pydb_base_dir, 'filestore')
time.sleep(0.5)
self.test_epub_path = get_book_path('pg1661.epub')
self.test_txt_path = get_book_path('pg1661.txt')
def setUp(self):
pydb_base_dir = tempfile.mkdtemp('test_file_server')
pydbserver = pydb.maindb.build(os.path.join(pydb_base_dir, 'db'),
pydb.testing.guess_schema_dir())
self.file_server = pydb.fileserver.build(
os.path.join(pydb_base_dir, 'filestore'), pydbserver)
print "filestore is in ", os.path.join(pydb_base_dir, 'filestore')
time.sleep(0.5)
self.test_epub_path = get_book_path('pg1661.epub')
self.test_txt_path = get_book_path('pg1661.txt')
def test_disconnect_detection_during_file_providing_works(self):
session_friend_id = 4
hash_of_file_to_request = '1234'
job_id = 11
main_db_mock = mock.MagicMock()
main_db_mock.get_friend_last_query_dates.return_value = (0, 0)
com_service_mock = mock.MagicMock()
com_service_mock.register_job.return_value = job_id
file_server_mock = mock.MagicMock()
file_server_mock.get_local_file_path.return_value = get_book_path(
'pg1661.epub')
session_mock = mock.MagicMock()
session_controller = pydb.com.server.SessionController(
main_db_mock, com_service_mock, file_server_mock)
session_controller.set_lower_layer(session_mock)
session_controller.session_established(session_friend_id)
com_service_mock.register_job.assert_called_once_with(
'fetch_updates', session_friend_id)
metadata_requester = session_controller._communication_strategy._metadata_requester
metadata_requester.command_update_modification_date_received(0, 0)
provider = session_controller._communication_strategy._provider
self.assertIsNotNone(
provider.lower_layer) # check that the provider has been activated
provider.command_request_file_received(hash_of_file_to_request)
provider.session_lost('Disconnect (Test)')
com_service_mock.unregister_job.assert_called_once_with(job_id)
def add_sample_tome(db, author_id, upload_a_file=False, tag_values=None):
"""
:param tag_values: if not none, iterable of tag values to add
"""
all_tag_values = ['one tag', 'two tag']
if tag_values:
all_tag_values += tag_values
tome_id = db.add_tome(title='it is also on file!',
principal_language='en',
author_ids=[author_id],
tags_values=all_tag_values)
tome = db.get_tome(tome_id)
if upload_a_file:
file_path = test_data.get_book_path('pg1661.epub')
(file_id, file_hash,
size) = pydb.pyrosetup.fileserver().add_file_from_local_disk(
file_path, 'epub', move_file=False)
db.link_tome_to_file(tome_id,
file_hash,
size,
file_extension='epub',
file_type=FileType.Content,
fidelity=80)
return tome
def test_disconnect_detection_during_file_providing_works(self):
session_friend_id = 4
hash_of_file_to_request = '1234'
job_id = 11
main_db_mock = mock.MagicMock()
main_db_mock.get_friend_last_query_dates.return_value = (0, 0)
com_service_mock = mock.MagicMock()
com_service_mock.register_job.return_value = job_id
file_server_mock = mock.MagicMock()
file_server_mock.get_local_file_path.return_value = get_book_path('pg1661.epub')
session_mock = mock.MagicMock()
session_controller = pydb.com.server.SessionController(main_db_mock, com_service_mock, file_server_mock)
session_controller.set_lower_layer(session_mock)
session_controller.session_established(session_friend_id)
com_service_mock.register_job.assert_called_once_with('fetch_updates', session_friend_id)
metadata_requester = session_controller._communication_strategy._metadata_requester
metadata_requester.command_update_modification_date_received(0, 0)
provider = session_controller._communication_strategy._provider
self.assertIsNotNone(provider.lower_layer) # check that the provider has been activated
provider.command_request_file_received(hash_of_file_to_request)
provider.session_lost('Disconnect (Test)')
com_service_mock.unregister_job.assert_called_once_with(job_id)
def test_timeline_with_a_tome_having_a_file_returns_list_and_view_renders(
self):
author_id = self.pdb.add_author('filemaster')
tome_id = self.pdb.add_tome(title='it is on file!',
principal_language='en',
author_ids=[author_id])
file_path = test_data.get_book_path('pg1661.epub')
(file_id, file_hash,
size) = pydb.pyrosetup.fileserver().add_file_from_local_disk(
file_path, 'epub', move_file=False)
self.pdb.link_tome_to_file(tome_id,
file_hash,
size,
file_extension='epub',
file_type=FileType.Content,
fidelity=80)
res = self.timeline.execute()
self.assertIn('tome_info', res)
tome_info = res['tome_info']
self.assertGreaterEqual(len(tome_info), 1)
html = self.timeline.render_result()
self.assertIn('Timeline', html)
def test_multi_strip(self):
with open(get_book_path('pg1661.pdf'), 'rb') as f:
original_data = f.read()
print "Original hash : {}, size {}".format(
hashlib.sha256(original_data).hexdigest(), len(original_data))
# hash once for reference
test_data, reference_hash = strip_and_hash(original_data)
if test_data is None:
print "Hashing not possible, aborting"
return
print "Hash after stripping : {}, size {}".format(
reference_hash, len(test_data))
dump_file(0, test_data)
for i in range(10):
stripped_data, file_hash = strip_and_hash(test_data)
print "hash after iteration {}: {}, size {}".format(
i, file_hash, len(stripped_data))
dump_file(i, stripped_data)
self.assertEqual(
file_hash, reference_hash,
"Hash compare at iteration {} should match".format(i))
test_data = stripped_data
def test_after_re_adding_an_existing_file_without_extension_no_new_entry_is_created(
self):
txt_file_without_extension = get_book_path('pg1661_txt')
first_id, first_hash, first_size = self.file_server.add_file_from_local_disk(
self.test_txt_path, 'txt')
second_id, second_hash, second_size = self.file_server.add_file_from_local_disk(
txt_file_without_extension, '')
self.assertEqual(first_id, second_id)
def setUp(self):
self.session_mock = mock.MagicMock()
self.report_progress_mock = mock.MagicMock()
self.reactor_mock = Clock()
self.file_server_mock = mock.MagicMock()
self.file_server_mock.get_local_file_path.return_value = get_book_path(
'pg1661.epub')
self.send_queue = pydb.com.file_send_queue.FileSendQueue(
self.session_mock, self.file_server_mock,
self.report_progress_mock, self.reactor_mock)
def setUp(self):
self.session_mock = mock.MagicMock()
self.report_progress_mock = mock.MagicMock()
self.reactor_mock = Clock()
self.file_server_mock = mock.MagicMock()
self.file_server_mock.get_local_file_path.return_value = get_book_path('pg1661.epub')
self.send_queue = pydb.com.file_send_queue.FileSendQueue(self.session_mock,
self.file_server_mock,
self.report_progress_mock,
self.reactor_mock)
def setUp(self):
self.test_file_contents = open(get_book_path('pg1661.epub'), "rb").read()
self.file_stream = StringIO(self.test_file_contents)
self.received_file_contents = ""
# noinspection PyUnusedLocal
def file_received(file_hash, extension, content, more_parts_follow):
self.received_file_contents += content
self.client_control = mock.MagicMock()
self.client_control.command_deliver_file_received.side_effect = file_received
self.server_control = mock.MagicMock()
def test_timeline_with_a_tome_having_a_file_returns_list_and_view_renders(self):
author_id = self.pdb.add_author('filemaster')
tome_id = self.pdb.add_tome(title='it is on file!', principal_language='en', author_ids=[author_id])
file_path = test_data.get_book_path('pg1661.epub')
(file_id, file_hash, size) = pydb.pyrosetup.fileserver().add_file_from_local_disk(file_path, 'epub',
move_file=False)
self.pdb.link_tome_to_file(tome_id, file_hash, size, file_extension='epub', file_type=FileType.Content,
fidelity=80)
res = self.timeline.execute()
self.assertIn('tome_info', res)
tome_info = res['tome_info']
self.assertGreaterEqual(len(tome_info), 1)
html = self.timeline.render_result()
self.assertIn('Timeline', html)
def test_add_an_epub(self):
for i in xrange(10):
pydb_base_dir = tempfile.mkdtemp('pydb_file_add_performance')
pydbserver = mock.MagicMock()
self.file_server = pydb.fileserver.build(os.path.join(pydb_base_dir, 'filestore'), pydbserver)
time.sleep(0.5)
file_path = get_book_path('pg1661.epub')
# read the file to fill the os cache
test_file_contents = open(file_path, "rb").read()
epub_size = len(test_file_contents)
start_time = time.clock()
self.file_server.add_file_from_local_disk(file_path, 'epub',
only_allowed_hash=None, move_file=False, strip_file=True)
stop_time = time.clock()
duration = stop_time - start_time
print_results('epub', epub_size, duration)
def add_sample_tome(db, author_id, upload_a_file=False, tag_values=None):
"""
:param tag_values: if not none, iterable of tag values to add
"""
all_tag_values = ['one tag', 'two tag']
if tag_values:
all_tag_values += tag_values
tome_id = db.add_tome(title='it is also on file!', principal_language='en', author_ids=[author_id],
tags_values=all_tag_values)
tome = db.get_tome(tome_id)
if upload_a_file:
file_path = test_data.get_book_path('pg1661.epub')
(file_id, file_hash, size) = pydb.pyrosetup.fileserver().add_file_from_local_disk(file_path, 'epub',
move_file=False)
db.link_tome_to_file(tome_id, file_hash, size, file_extension='epub', file_type=FileType.Content,
fidelity=80)
return tome
def test_multi_strip(self):
with open(get_book_path('pg1661.pdf'), 'rb') as f:
original_data = f.read()
print "Original hash : {}, size {}".format(hashlib.sha256(original_data).hexdigest(),
len(original_data))
# hash once for reference
test_data, reference_hash = strip_and_hash(original_data)
if test_data is None:
print "Hashing not possible, aborting"
return
print "Hash after stripping : {}, size {}".format(reference_hash, len(test_data))
dump_file(0, test_data)
for i in range(10):
stripped_data, file_hash = strip_and_hash(test_data)
print "hash after iteration {}: {}, size {}".format(i, file_hash, len(stripped_data))
dump_file(i, stripped_data)
self.assertEqual(file_hash, reference_hash, "Hash compare at iteration {} should match".format(i))
test_data = stripped_data
def test_add_an_epub(self):
for i in xrange(10):
pydb_base_dir = tempfile.mkdtemp('pydb_file_add_performance')
pydbserver = mock.MagicMock()
self.file_server = pydb.fileserver.build(
os.path.join(pydb_base_dir, 'filestore'), pydbserver)
time.sleep(0.5)
file_path = get_book_path('pg1661.epub')
# read the file to fill the os cache
test_file_contents = open(file_path, "rb").read()
epub_size = len(test_file_contents)
start_time = time.clock()
self.file_server.add_file_from_local_disk(file_path,
'epub',
only_allowed_hash=None,
move_file=False,
strip_file=True)
stop_time = time.clock()
duration = stop_time - start_time
print_results('epub', epub_size, duration)
def dump_file(index, data):
do_dumps = False
if do_dumps:
return
with open(get_book_path('out_{}.pdf'.format(index)), 'wb') as f:
f.write(data)
def test_after_re_adding_an_existing_file_without_extension_no_new_entry_is_created(self):
txt_file_without_extension = get_book_path('pg1661_txt')
first_id, first_hash, first_size = self.file_server.add_file_from_local_disk(self.test_txt_path, 'txt')
second_id, second_hash, second_size = self.file_server.add_file_from_local_disk(txt_file_without_extension, '')
self.assertEqual(first_id, second_id)
# coding=utf-8
import unittest
import os
import pydb.ebook_metadata_tools as etools
from pydb.testing.test_data import get_book_path
script_path = os.path.dirname(__file__)
epub_file_path = get_book_path('pg1661.epub')
txt_file_path = get_book_path('pg1661.txt')
def extract_cover_from_file(file_path, extension):
with open(file_path, 'rb') as source_stream:
return etools.get_cover_image(source_stream, extension)
class TestCoverExtraction(unittest.TestCase):
def test_epub_without_cover_leads_to_none(self):
result = extract_cover_from_file(txt_file_path, 'txt')
self.assertIsNone(result)
if __name__ == '__main__':
unittest.main()
def dump_file(index, data):
do_dumps = False
if do_dumps:
return
with open(get_book_path('out_{}.pdf'.format(index)), 'wb') as f:
f.write(data)