def test_chunkify(self): # Test tuple, list, set, range, map, chain and generator seq = [1, 2, 3, 4, 5] self.assertEqual(list(chunkify(seq, chunksize=2)), [[1, 2], [3, 4], [5]]) seq = (1, 2, 3, 4, 6, 7, 9) self.assertEqual(list(chunkify(seq, chunksize=3)), [(1, 2, 3), (4, 6, 7), (9, )]) seq = {1, 2, 3, 4, 5} self.assertEqual(list(chunkify(seq, chunksize=2)), [[1, 2], [3, 4], [5]]) seq = range(5) self.assertEqual( list(chunkify(seq, chunksize=2)), [range(0, 2), range(2, 4), range(4, 5)]) seq = map(int, range(5)) self.assertEqual(list(chunkify(seq, chunksize=2)), [[0, 1], [2, 3], [4]]) seq = chain(*[[i] for i in range(5)]) self.assertEqual(list(chunkify(seq, chunksize=2)), [[0, 1], [2, 3], [4]]) seq = (i for i in range(5)) self.assertEqual(list(chunkify(seq, chunksize=2)), [[0, 1], [2, 3], [4]])
def _export_folder_subset(self, folder, start_dt=None, end_dt=None): """ Export a time-subset of an exchange folder :param folder: The exchange folder to export :start_dt: The start date to export from, default 1900-01-01 :end_dt: The end date to export to, default 2100-01-01 :return: Number of attachments in folder """ try: logger.info('Export subset: {} to {}'.format(start_dt, end_dt)) attachments = 0 if start_dt is None: start_dt = EWSDate(1900, 1, 1) if end_dt is None: end_dt = EWSDate(2100, 1, 1) items = folder.all() start_dt = UTC.localize( EWSDateTime(start_dt.year, start_dt.month, start_dt.day, 0, 0)) end_dt = UTC.localize( EWSDateTime(end_dt.year, end_dt.month, end_dt.day, 0, 0)) items = items.filter(datetime_received__range=(start_dt, end_dt)) for chunk in chunkify(items, 10): for item in chunk: self.actual_exported_mails += 1 logger.error( str(item.datetime_created) + ':' + str(item.subject)) skip_list = self.export_item_body(item) attachments += self.export_attachments(item, skip_list) self.update_amqp(only_mails=True) except ErrorMimeContentConversionFailed: msg = '{}: ErrorMimeContentConversionFailed, giving up sub-folder' msg += ' Attachment value: {}' logger.warning(msg.format(self.export_path, attachments)) except ErrorInternalServerError: # Possibly happens on p7m files? msg = '{}: ErrorInternalServerError, giving up sub-folder' msg += ' Attachment value: {}' logger.warning(msg.format(self.export_path, attachments)) except ErrorInvalidOperation: msg = '{}: ErrorInvalidOperation, giving up sub-folder' msg += ' Attachment value: {}' logger.warning(msg.format(self.export_path, attachments)) except ErrorTimeoutExpired: attachments = -1 time.sleep(30) logger.warning('{}: ErrorTimeoutExpired'.format(self.export_path)) except ErrorInternalServerTransientError: attachments = -1 time.sleep(30) warning = '{}, {}: ErrorInternalServerTransientError' logger.warning(warning.format(self.export_path, folder)) return attachments
def _export_folder_subset(self, folder, start_dt=None, end_dt=None): """ Export a time-subset of an exchange folder :param folder: The exchange folder to export :start_dt: The start date to export from, default 1900-01-01 :end_dt: The end date to export to, default 2100-01-01 :return: Number of attachments in folder """ try: logger.info('Export subset: {} to {}'.format(start_dt, end_dt)) attachments = 0 if start_dt is None: start_dt = EWSDate(1900, 1, 1) if end_dt is None: end_dt = EWSDate(2100, 1, 1) items = folder.all() start_dt = UTC.localize(EWSDateTime(start_dt.year, start_dt.month, start_dt.day, 0, 0)) end_dt = UTC.localize(EWSDateTime(end_dt.year, end_dt.month, end_dt.day, 0, 0)) items = items.filter(datetime_received__range=(start_dt, end_dt)) for chunk in chunkify(items, 10): for item in chunk: self.actual_exported_mails += 1 logger.error(str(item.datetime_created) + ':' + str(item.subject)) skip_list = self.export_item_body(item) attachments += self.export_attachments(item, skip_list) self.update_amqp(only_mails=True) except ErrorMimeContentConversionFailed: msg = '{}: ErrorMimeContentConversionFailed, giving up sub-folder' msg += ' Attachment value: {}' logger.warning(msg.format(self.export_path, attachments)) except ErrorInternalServerError: # Possibly happens on p7m files? msg = '{}: ErrorInternalServerError, giving up sub-folder' msg += ' Attachment value: {}' logger.warning(msg.format(self.export_path, attachments)) except ErrorInvalidOperation: msg = '{}: ErrorInvalidOperation, giving up sub-folder' msg += ' Attachment value: {}' logger.warning(msg.format(self.export_path, attachments)) except ErrorTimeoutExpired: attachments = -1 time.sleep(30) logger.warning('{}: ErrorTimeoutExpired'.format(self.export_path)) except ErrorInternalServerTransientError: attachments = -1 time.sleep(30) warning = '{}, {}: ErrorInternalServerTransientError' logger.warning(warning.format(self.export_path, folder)) return attachments
def test_streaming_file_attachments(self): item = self.get_test_item(folder=self.test_folder) large_binary_file_content = get_random_string(2 ** 10).encode("utf-8") large_att = FileAttachment(name="my_large_file.txt", content=large_binary_file_content) item.attach(large_att) item.save() # Test streaming file content fresh_item = self.get_item_by_id(item) with fresh_item.attachments[0].fp as fp: self.assertEqual(fp.read(), large_binary_file_content) # Test partial reads of streaming file content fresh_item = self.get_item_by_id(item) with fresh_item.attachments[0].fp as fp: chunked_reads = [] buffer = fp.read(7) while buffer: chunked_reads.append(buffer) buffer = fp.read(7) self.assertListEqual(chunked_reads, list(chunkify(large_binary_file_content, 7)))
def test_chunkify(self): # Test list, tuple, set, range, map and generator seq = [1, 2, 3, 4, 5] self.assertEqual(list(chunkify(seq, chunksize=2)), [[1, 2], [3, 4], [5]]) seq = (1, 2, 3, 4, 6, 7, 9) self.assertEqual(list(chunkify(seq, chunksize=3)), [(1, 2, 3), (4, 6, 7), (9,)]) seq = {1, 2, 3, 4, 5} self.assertEqual(list(chunkify(seq, chunksize=2)), [[1, 2], [3, 4], [5,]]) seq = range(5) self.assertEqual(list(chunkify(seq, chunksize=2)), [range(0, 2), range(2, 4), range(4, 5)]) seq = map(int, range(5)) self.assertEqual(list(chunkify(seq, chunksize=2)), [[0, 1], [2, 3], [4]]) seq = (i for i in range(5)) self.assertEqual(list(chunkify(seq, chunksize=2)), [[0, 1], [2, 3], [4]])