def test_chunkify(self):
# Test tuple, list, set, range, map, chain and generator
seq = [1, 2, 3, 4, 5]
self.assertEqual(list(chunkify(seq, chunksize=2)),
[[1, 2], [3, 4], [5]])
seq = (1, 2, 3, 4, 6, 7, 9)
self.assertEqual(list(chunkify(seq, chunksize=3)), [(1, 2, 3),
(4, 6, 7), (9, )])
seq = {1, 2, 3, 4, 5}
self.assertEqual(list(chunkify(seq, chunksize=2)),
[[1, 2], [3, 4], [5]])
seq = range(5)
self.assertEqual(
list(chunkify(seq, chunksize=2)),
[range(0, 2), range(2, 4), range(4, 5)])
seq = map(int, range(5))
self.assertEqual(list(chunkify(seq, chunksize=2)),
[[0, 1], [2, 3], [4]])
seq = chain(*[[i] for i in range(5)])
self.assertEqual(list(chunkify(seq, chunksize=2)),
[[0, 1], [2, 3], [4]])
seq = (i for i in range(5))
self.assertEqual(list(chunkify(seq, chunksize=2)),
[[0, 1], [2, 3], [4]])
def _export_folder_subset(self, folder, start_dt=None, end_dt=None):
""" Export a time-subset of an exchange folder
:param folder: The exchange folder to export
:start_dt: The start date to export from, default 1900-01-01
:end_dt: The end date to export to, default 2100-01-01
:return: Number of attachments in folder
"""
try:
logger.info('Export subset: {} to {}'.format(start_dt, end_dt))
attachments = 0
if start_dt is None:
start_dt = EWSDate(1900, 1, 1)
if end_dt is None:
end_dt = EWSDate(2100, 1, 1)
items = folder.all()
start_dt = UTC.localize(
EWSDateTime(start_dt.year, start_dt.month, start_dt.day, 0, 0))
end_dt = UTC.localize(
EWSDateTime(end_dt.year, end_dt.month, end_dt.day, 0, 0))
items = items.filter(datetime_received__range=(start_dt, end_dt))
for chunk in chunkify(items, 10):
for item in chunk:
self.actual_exported_mails += 1
logger.error(
str(item.datetime_created) + ':' + str(item.subject))
skip_list = self.export_item_body(item)
attachments += self.export_attachments(item, skip_list)
self.update_amqp(only_mails=True)
except ErrorMimeContentConversionFailed:
msg = '{}: ErrorMimeContentConversionFailed, giving up sub-folder'
msg += ' Attachment value: {}'
logger.warning(msg.format(self.export_path, attachments))
except ErrorInternalServerError:
# Possibly happens on p7m files?
msg = '{}: ErrorInternalServerError, giving up sub-folder'
msg += ' Attachment value: {}'
logger.warning(msg.format(self.export_path, attachments))
except ErrorInvalidOperation:
msg = '{}: ErrorInvalidOperation, giving up sub-folder'
msg += ' Attachment value: {}'
logger.warning(msg.format(self.export_path, attachments))
except ErrorTimeoutExpired:
attachments = -1
time.sleep(30)
logger.warning('{}: ErrorTimeoutExpired'.format(self.export_path))
except ErrorInternalServerTransientError:
attachments = -1
time.sleep(30)
warning = '{}, {}: ErrorInternalServerTransientError'
logger.warning(warning.format(self.export_path, folder))
return attachments
def _export_folder_subset(self, folder, start_dt=None, end_dt=None):
""" Export a time-subset of an exchange folder
:param folder: The exchange folder to export
:start_dt: The start date to export from, default 1900-01-01
:end_dt: The end date to export to, default 2100-01-01
:return: Number of attachments in folder
"""
try:
logger.info('Export subset: {} to {}'.format(start_dt, end_dt))
attachments = 0
if start_dt is None:
start_dt = EWSDate(1900, 1, 1)
if end_dt is None:
end_dt = EWSDate(2100, 1, 1)
items = folder.all()
start_dt = UTC.localize(EWSDateTime(start_dt.year, start_dt.month,
start_dt.day, 0, 0))
end_dt = UTC.localize(EWSDateTime(end_dt.year, end_dt.month,
end_dt.day, 0, 0))
items = items.filter(datetime_received__range=(start_dt, end_dt))
for chunk in chunkify(items, 10):
for item in chunk:
self.actual_exported_mails += 1
logger.error(str(item.datetime_created) + ':' + str(item.subject))
skip_list = self.export_item_body(item)
attachments += self.export_attachments(item, skip_list)
self.update_amqp(only_mails=True)
except ErrorMimeContentConversionFailed:
msg = '{}: ErrorMimeContentConversionFailed, giving up sub-folder'
msg += ' Attachment value: {}'
logger.warning(msg.format(self.export_path, attachments))
except ErrorInternalServerError:
# Possibly happens on p7m files?
msg = '{}: ErrorInternalServerError, giving up sub-folder'
msg += ' Attachment value: {}'
logger.warning(msg.format(self.export_path, attachments))
except ErrorInvalidOperation:
msg = '{}: ErrorInvalidOperation, giving up sub-folder'
msg += ' Attachment value: {}'
logger.warning(msg.format(self.export_path, attachments))
except ErrorTimeoutExpired:
attachments = -1
time.sleep(30)
logger.warning('{}: ErrorTimeoutExpired'.format(self.export_path))
except ErrorInternalServerTransientError:
attachments = -1
time.sleep(30)
warning = '{}, {}: ErrorInternalServerTransientError'
logger.warning(warning.format(self.export_path, folder))
return attachments
def test_streaming_file_attachments(self):
item = self.get_test_item(folder=self.test_folder)
large_binary_file_content = get_random_string(2 ** 10).encode("utf-8")
large_att = FileAttachment(name="my_large_file.txt", content=large_binary_file_content)
item.attach(large_att)
item.save()
# Test streaming file content
fresh_item = self.get_item_by_id(item)
with fresh_item.attachments[0].fp as fp:
self.assertEqual(fp.read(), large_binary_file_content)
# Test partial reads of streaming file content
fresh_item = self.get_item_by_id(item)
with fresh_item.attachments[0].fp as fp:
chunked_reads = []
buffer = fp.read(7)
while buffer:
chunked_reads.append(buffer)
buffer = fp.read(7)
self.assertListEqual(chunked_reads, list(chunkify(large_binary_file_content, 7)))
def test_chunkify(self):
# Test list, tuple, set, range, map and generator
seq = [1, 2, 3, 4, 5]
self.assertEqual(list(chunkify(seq, chunksize=2)), [[1, 2], [3, 4], [5]])
seq = (1, 2, 3, 4, 6, 7, 9)
self.assertEqual(list(chunkify(seq, chunksize=3)), [(1, 2, 3), (4, 6, 7), (9,)])
seq = {1, 2, 3, 4, 5}
self.assertEqual(list(chunkify(seq, chunksize=2)), [[1, 2], [3, 4], [5,]])
seq = range(5)
self.assertEqual(list(chunkify(seq, chunksize=2)), [range(0, 2), range(2, 4), range(4, 5)])
seq = map(int, range(5))
self.assertEqual(list(chunkify(seq, chunksize=2)), [[0, 1], [2, 3], [4]])
seq = (i for i in range(5))
self.assertEqual(list(chunkify(seq, chunksize=2)), [[0, 1], [2, 3], [4]])
