def setUp(self):
self.azure_client_mock = Mock()
self.email_server_client_mock = Mock()
self.sync = AzureSync(
container='container',
email_server_client=self.email_server_client_mock,
account_key='mock',
account_name='mock',
azure_client=self.azure_client_mock,
serializer=JsonSerializer())
def setUp(self):
self._root_folder = mkdtemp()
self.email_server_client_mock = Mock()
self._container = 'compressedpackages'
self.sync = AzureSync(container=self._container,
email_server_client=self.email_server_client_mock,
account_key='mock',
account_name=self._root_folder,
provider='LOCAL',
compression='gz',
serializer=JsonSerializer())
self._content_root = join(self._root_folder, self._container)
mkdir(self._content_root)
def email_sync(self):
if AppConfig.TESTING:
email_server_client = LocalEmailServerClient()
else:
if AppConfig.EMAIL_SERVER_ENDPOINT:
endpoint = AppConfig.EMAIL_SERVER_ENDPOINT
elif AppConfig.EMAIL_SERVER_HOSTNAME:
endpoint = 'https://{}'.format(AppConfig.EMAIL_SERVER_HOSTNAME)
else:
endpoint = None
email_server_client = HttpEmailServerClient(
compression=AppConfig.COMPRESSION,
endpoint=endpoint,
client_id=AppConfig.CLIENT_ID,
)
serializer = JsonSerializer()
return AzureSync(
compression=AppConfig.COMPRESSION,
account_name=AppConfig.STORAGE_ACCOUNT_NAME,
account_key=AppConfig.STORAGE_ACCOUNT_KEY,
account_host=AppConfig.STORAGE_ACCOUNT_HOST,
account_secure=AppConfig.STORAGE_ACCOUNT_SECURE,
email_server_client=email_server_client,
container=AppConfig.STORAGE_CONTAINER,
provider=AppConfig.STORAGE_PROVIDER,
serializer=serializer,
)
def setUp(self):
self._root_folder = mkdtemp()
self.email_server_client_mock = Mock()
self._container = 'compressedpackages'
self.sync = AzureSync(
container=self._container,
email_server_client=self.email_server_client_mock,
account_key='mock',
account_name=self._root_folder,
provider='LOCAL',
compression='gz',
serializer=JsonSerializer())
self._content_root = join(self._root_folder, self._container)
mkdir(self._content_root)
class Ioc(object):
serializer = JsonSerializer()
email_server_client = HttpEmailServerClient(
read_api=AppConfig.EMAIL_SERVER_READ_API_HOSTNAME,
write_api=AppConfig.EMAIL_SERVER_WRITE_API_HOSTNAME,
client_id=AppConfig.CLIENT_ID)
email_store = SqliteEmailStore(
database_path=AppConfig.LOCAL_EMAIL_STORE)
email_sync = AzureSync(
account_name=AppConfig.STORAGE_ACCOUNT_NAME,
account_key=AppConfig.STORAGE_ACCOUNT_KEY,
email_server_client=email_server_client,
container=AppConfig.STORAGE_CONTAINER,
serializer=serializer)
attachment_encoder = Base64AttachmentEncoder()
attachments_session = AttachmentsStore(
email_store=email_store,
attachment_encoder=attachment_encoder)
class AzureSyncTests(TestCase):
_test_compressions = ('gz', 'zstd')
# noinspection PyTypeChecker
def setUp(self):
self._root_folder = mkdtemp()
self.email_server_client_mock = Mock()
self._container = 'compressedpackages'
self.sync = AzureSync(
container=self._container,
email_server_client=self.email_server_client_mock,
account_key='mock',
account_name=self._root_folder,
account_host=None,
account_secure=True,
provider='LOCAL',
compression='gz',
serializer=JsonSerializer())
self._content_root = join(self._root_folder, self._container)
mkdir(self._content_root)
def tearDown(self):
rmtree(self._root_folder)
def assertUploadIs(self,
expected: Dict[str, bytes],
compression: str = ''):
uploaded = glob(
join(self._content_root, '*.{}'.format(compression
or self.sync._compression)))
self.assertEqual(len(uploaded), 1, 'Expected exactly one upload')
infos = (Download(name=name, optional=False, type_='')
for name in expected)
with open(uploaded[0], 'rb') as buffer:
with self.sync._open(buffer.name, 'r') as archive:
for download, fobj in self.sync._get_file_from_download(
archive, infos):
self.assertEqual(expected[download.name], fobj.read())
def assertNoUpload(self):
uploaded = glob(join(self._content_root, '*'))
self.assertEqual(len(uploaded), 0, 'Expected no uploads')
def given_download(self, payload: Dict[str, bytes], compression: str):
resource_id = '{}.tar.{}'.format(uuid4(), compression)
download_filename = join(self._content_root, resource_id)
with self.sync._open(download_filename, 'w') as archive:
for filename, content in payload.items():
tarinfo = TarInfo(filename)
tarinfo.size = len(content)
archive.addfile(tarinfo, BytesIO(content))
self.email_server_client_mock.download.return_value = resource_id
def given_download_exception(self):
self.email_server_client_mock.download.return_value = 'unknown'
def test_upload(self):
for compression in self._test_compressions:
with self.subTest(compression=compression):
self.sync._compression = compression
self.sync.upload(items=[{'foo': 'bar'}], users=[])
self.assertUploadIs(
{self.sync._emails_file: b'{"foo":"bar"}\n'}, compression)
self.assertTrue(self.email_server_client_mock.upload.called)
def test_upload_excludes_null_values(self):
self.sync.upload(items=[{'foo': 0, 'bar': None}], users=[])
self.assertUploadIs({self.sync._emails_file: b'{"foo":0}\n'})
def test_upload_excludes_internal_values(self):
self.sync.upload(items=[{
'foo':
0,
'read':
True,
'attachments': [{
'_uid': '1',
'filename': 'foo.txt'
}]
}],
users=[])
self.assertUploadIs({
self.sync._emails_file:
b'{"attachments":[{"filename":"foo.txt"}]'
b',"foo":0}\n'
})
def test_upload_with_no_content_does_not_hit_network(self):
self.sync.upload(items=[], users=[])
self.assertNoUpload()
self.assertFalse(self.email_server_client_mock.upload.called)
def test_download(self):
for compression in self._test_compressions:
with self.subTest(compression=compression):
self.given_download(
{self.sync._emails_file: b'{"foo":"bar"}\n{"baz":1}'},
compression)
downloaded = list(self.sync.download())
self.assertTrue(self.email_server_client_mock.download.called)
self.assertEqual(len(downloaded), 2)
self.assertIn({'foo': 'bar', '_type': 'email'}, downloaded)
self.assertIn({'baz': 1, '_type': 'email'}, downloaded)
def test_download_with_attachments(self):
for compression in self._test_compressions:
with self.subTest(compression=compression):
self.given_download(
{
self.sync._emails_file: b'{"foo":"bar"}\n{"baz":1}',
self.sync._attachments_file: b'{"x":"y"}\n{"z":1}'
}, compression)
downloaded = list(self.sync.download())
self.assertTrue(self.email_server_client_mock.download.called)
self.assertEqual(len(downloaded), 4)
self.assertIn({'foo': 'bar', '_type': 'email'}, downloaded)
self.assertIn({'baz': 1, '_type': 'email'}, downloaded)
self.assertIn({'x': 'y', '_type': 'attachment'}, downloaded)
self.assertIn({'z': 1, '_type': 'attachment'}, downloaded)
def test_download_missing_resource(self):
self.given_download_exception()
downloaded = list(self.sync.download())
self.assertEqual(downloaded, [])
class AzureSyncTests(TestCase):
_test_compressions = ('gz', 'zstd')
# noinspection PyTypeChecker
def setUp(self):
self._root_folder = mkdtemp()
self.email_server_client_mock = Mock()
self._container = 'compressedpackages'
self.sync = AzureSync(
container=self._container,
email_server_client=self.email_server_client_mock,
account_key='mock',
account_name=self._root_folder,
provider='LOCAL',
compression='gz',
serializer=JsonSerializer())
self._content_root = join(self._root_folder, self._container)
mkdir(self._content_root)
def tearDown(self):
rmtree(self._root_folder)
def assertUploadIs(self, expected: Dict[str, bytes],
compression: str = ''):
uploaded = glob(join(
self._content_root,
'*.{}'.format(compression or self.sync._compression)))
self.assertEqual(len(uploaded), 1, 'Expected exactly one upload')
infos = (Download(name=name, optional=False, type_='')
for name in expected)
with open(uploaded[0], 'rb') as buffer:
with self.sync._open(buffer.name, 'r') as archive:
for download, fobj in self.sync._get_file_from_download(
archive, infos):
self.assertEqual(expected[download.name], fobj.read())
def assertNoUpload(self):
uploaded = glob(join(self._content_root, '*'))
self.assertEqual(len(uploaded), 0, 'Expected no uploads')
def given_download(self, payload: Dict[str, bytes], compression: str):
resource_id = '{}.tar.{}'.format(uuid4(), compression)
download_filename = join(self._content_root, resource_id)
with self.sync._open(download_filename, 'w') as archive:
for filename, content in payload.items():
tarinfo = TarInfo(filename)
tarinfo.size = len(content)
archive.addfile(tarinfo, BytesIO(content))
self.email_server_client_mock.download.return_value = resource_id
def given_download_exception(self):
self.email_server_client_mock.download.return_value = 'unknown'
def test_upload(self):
for compression in self._test_compressions:
with self.subTest(compression=compression):
self.sync._compression = compression
self.sync.upload(items=[{'foo': 'bar'}])
self.assertUploadIs(
{self.sync._emails_file: b'{"foo":"bar"}\n'},
compression)
self.assertTrue(self.email_server_client_mock.upload.called)
def test_upload_excludes_null_values(self):
self.sync.upload(items=[{'foo': 0, 'bar': None}])
self.assertUploadIs({self.sync._emails_file: b'{"foo":0}\n'})
def test_upload_excludes_internal_values(self):
self.sync.upload(items=[{'foo': 0, 'read': True, 'attachments': [
{'_uid': '1', 'filename': 'foo.txt'}
]}])
self.assertUploadIs(
{self.sync._emails_file: b'{"attachments":[{"filename":"foo.txt"}]'
b',"foo":0}\n'})
def test_upload_with_no_content_does_not_hit_network(self):
self.sync.upload(items=[])
self.assertNoUpload()
self.assertFalse(self.email_server_client_mock.upload.called)
def test_download(self):
for compression in self._test_compressions:
with self.subTest(compression=compression):
self.given_download(
{self.sync._emails_file: b'{"foo":"bar"}\n{"baz":1}'},
compression)
downloaded = list(self.sync.download())
self.assertTrue(self.email_server_client_mock.download.called)
self.assertEqual(len(downloaded), 2)
self.assertIn({'foo': 'bar', '_type': 'email'}, downloaded)
self.assertIn({'baz': 1, '_type': 'email'}, downloaded)
def test_download_with_attachments(self):
for compression in self._test_compressions:
with self.subTest(compression=compression):
self.given_download(
{self.sync._emails_file: b'{"foo":"bar"}\n{"baz":1}',
self.sync._attachments_file: b'{"x":"y"}\n{"z":1}'},
compression)
downloaded = list(self.sync.download())
self.assertTrue(self.email_server_client_mock.download.called)
self.assertEqual(len(downloaded), 4)
self.assertIn({'foo': 'bar', '_type': 'email'}, downloaded)
self.assertIn({'baz': 1, '_type': 'email'}, downloaded)
self.assertIn({'x': 'y', '_type': 'attachment'}, downloaded)
self.assertIn({'z': 1, '_type': 'attachment'}, downloaded)
def test_download_missing_resource(self):
self.given_download_exception()
downloaded = list(self.sync.download())
self.assertEqual(downloaded, [])
class AzureSyncTests(TestCase):
# noinspection PyTypeChecker
def setUp(self):
self.azure_client_mock = Mock()
self.email_server_client_mock = Mock()
self.sync = AzureSync(
container='container',
email_server_client=self.email_server_client_mock,
account_key='mock',
account_name='mock',
azure_client=self.azure_client_mock,
serializer=JsonSerializer())
def assertUploadIs(self, actual: BytesIO, expected: bytes):
with self.sync._open(actual) as uploaded:
self.assertEqual(expected, uploaded.read())
def given_upload(self) -> BytesIO:
buffer = BytesIO()
# noinspection PyUnusedLocal
def side_effect(container, blobname, stream):
copyfileobj(stream, buffer)
buffer.seek(0)
self.azure_client_mock.create_blob_from_stream.side_effect = side_effect
return buffer
def given_download(self, payload: bytes):
buffer = BytesIO()
with self.sync._open(buffer, 'wb') as fobj:
fobj.write(payload)
buffer.seek(0)
# noinspection PyUnusedLocal
def side_effect(container, blobname, stream):
copyfileobj(buffer, stream)
self.email_server_client_mock.download.return_value = ('id', 'folder')
self.azure_client_mock.get_blob_to_stream.side_effect = side_effect
def given_download_exception(self):
error = AzureMissingResourceHttpError('injected error', 404)
self.email_server_client_mock.download.return_value = ('id', 'folder')
self.azure_client_mock.get_blob_to_stream.side_effect = error
def test_upload(self):
uploaded = self.given_upload()
self.sync.upload(items=[{'foo': 'bar'}])
self.assertUploadIs(uploaded, b'{"foo":"bar"}\n')
self.assertTrue(self.email_server_client_mock.upload.called)
def test_upload_excludes_null_values(self):
uploaded = self.given_upload()
self.sync.upload(items=[{'foo': 0, 'bar': None}])
self.assertUploadIs(uploaded, b'{"foo":0}\n')
def test_upload_with_no_content_does_not_hit_network(self):
self.sync.upload(items=[])
self.assertFalse(self.azure_client_mock.create_blob_from_stream.called)
self.assertFalse(self.email_server_client_mock.upload.called)
def test_download(self):
self.given_download(b'{"foo":"bar"}\n{"baz":1}')
downloaded = list(self.sync.download())
self.assertTrue(self.email_server_client_mock.download.called)
self.assertEqual(len(downloaded), 2)
self.assertIn({'foo': 'bar'}, downloaded)
self.assertIn({'baz': 1}, downloaded)
def test_download_missing_resource(self):
self.given_download_exception()
downloaded = list(self.sync.download())
self.assertEqual(downloaded, [])