def test_parse_incident_from_item_with_attachments():
"""
Given:
- Message item with attachment that contains email attachments
When:
- Parsing incident from item
Verify:
- Parsing runs successfully
"""
content = b'ARC-Seal: i=1; a=rsa-sha256; s=arcselector9901;' \
b' d=microsoft.com; cv=none;b=ES/YXpFlV19rlN1iV+ORg5RzID8GPSQL' \
b'nUT26MNdeTzcQSwK679doIz5Avpv8Ps2H/aBkBamwRNOCJBkl7iCHyy+04yRj3ghikw3u/ufIFHi0sQ7QG95mO1PVPLibv9A=='
message = Message(
datetime_created=EWSDate(year=2021, month=1, day=25),
to_recipients=[],
attachments=[
ItemAttachment(
item=Item(mime_content=content, headers=[]),
attachment_id=AttachmentId(),
last_modified_time=EWSDate(year=2021, month=1, day=25),
),
],
)
incident = parse_incident_from_item(message)
assert incident['attachment']
def test_parse_incident_from_item():
"""
Given:
- Message item with attachment that contains non UTF-8 encoded char
When:
- Parsing incident from item
Verify:
- Parsing runs successfully
- Incidnet attachment is not empty
"""
message = Message(
datetime_created=EWSDate(year=2021, month=1, day=25),
to_recipients=[],
attachments=[
ItemAttachment(
item=Item(mime_content=b'\xc400'),
attachment_id=AttachmentId(),
last_modified_time=EWSDate(year=2021, month=1, day=25),
),
],
)
incident = parse_incident_from_item(message)
assert incident['attachment']
def _export_folder_subset(self, folder, start_dt=None, end_dt=None):
""" Export a time-subset of an exchange folder
:param folder: The exchange folder to export
:start_dt: The start date to export from, default 1900-01-01
:end_dt: The end date to export to, default 2100-01-01
:return: Number of attachments in folder
"""
try:
logger.info('Export subset: {} to {}'.format(start_dt, end_dt))
attachments = 0
if start_dt is None:
start_dt = EWSDate(1900, 1, 1)
if end_dt is None:
end_dt = EWSDate(2100, 1, 1)
items = folder.all()
start_dt = UTC.localize(
EWSDateTime(start_dt.year, start_dt.month, start_dt.day, 0, 0))
end_dt = UTC.localize(
EWSDateTime(end_dt.year, end_dt.month, end_dt.day, 0, 0))
items = items.filter(datetime_received__range=(start_dt, end_dt))
for chunk in chunkify(items, 10):
for item in chunk:
self.actual_exported_mails += 1
logger.error(
str(item.datetime_created) + ':' + str(item.subject))
skip_list = self.export_item_body(item)
attachments += self.export_attachments(item, skip_list)
self.update_amqp(only_mails=True)
except ErrorMimeContentConversionFailed:
msg = '{}: ErrorMimeContentConversionFailed, giving up sub-folder'
msg += ' Attachment value: {}'
logger.warning(msg.format(self.export_path, attachments))
except ErrorInternalServerError:
# Possibly happens on p7m files?
msg = '{}: ErrorInternalServerError, giving up sub-folder'
msg += ' Attachment value: {}'
logger.warning(msg.format(self.export_path, attachments))
except ErrorInvalidOperation:
msg = '{}: ErrorInvalidOperation, giving up sub-folder'
msg += ' Attachment value: {}'
logger.warning(msg.format(self.export_path, attachments))
except ErrorTimeoutExpired:
attachments = -1
time.sleep(30)
logger.warning('{}: ErrorTimeoutExpired'.format(self.export_path))
except ErrorInternalServerTransientError:
attachments = -1
time.sleep(30)
warning = '{}, {}: ErrorInternalServerTransientError'
logger.warning(warning.format(self.export_path, folder))
return attachments
def test_validation(self):
p = DailyPattern(interval=3)
d_start = EWSDate(2017, 9, 1)
d_end = EWSDate(2017, 9, 7)
with self.assertRaises(ValueError):
Recurrence(pattern=p, boundary='foo', start='bar') # Specify *either* boundary *or* start, end and number
with self.assertRaises(ValueError):
Recurrence(pattern=p, start='foo', end='bar', number='baz') # number is invalid when end is present
with self.assertRaises(ValueError):
Recurrence(pattern=p, end='bar', number='baz') # Must have start
r = Recurrence(pattern=p, start=d_start)
self.assertEqual(r.boundary, NoEndPattern(start=d_start))
r = Recurrence(pattern=p, start=d_start, end=d_end)
self.assertEqual(r.boundary, EndDatePattern(start=d_start, end=d_end))
r = Recurrence(pattern=p, start=d_start, number=1)
self.assertEqual(r.boundary, NumberedPattern(start=d_start, number=1))
def export_folder(self, folder):
""" Export a given folder
:param folder: The folder to export
:return: The number of exported attachments
"""
folder_name = folder.name.replace(' ', '_').replace('/', '_')
self.current_path = self.export_path.joinpath(folder_name)
if self.export_path.joinpath(folder_name + '_done').exists():
logger.info('Already done: {}'.format(self.current_path))
return folder.total_count # Already scanned
if self.current_path.exists():
logger.info('Clean up: {}'.format(self.current_path))
shutil.rmtree(str(self.current_path))
self.current_path.mkdir()
attachments = 0
if self.start_date is None:
start_dt = EWSDate(2010, 1, 1)
# First, export everything before 2010
attachments += self._attempt_export(folder, end_dt=start_dt)
else:
start_dt = self.start_date
end_dt = start_dt + timedelta(days=10)
while end_dt < (EWSDate.today() + timedelta(days=10)):
msg = 'Export folder, currently from {} to {}'
logger.debug(msg.format(start_dt, end_dt))
attachments += self._attempt_export(folder,
start_dt=start_dt,
end_dt=end_dt)
start_dt = end_dt
end_dt = start_dt + timedelta(days=10)
# Finally, export everything later than today (hopefully nothing)
attachments += self._attempt_export(folder, start_dt=end_dt)
try:
self.current_path.rename(str(self.current_path) + '_done')
except FileNotFoundError:
# This can happen if a user mistakenly is scanned twice at
# the same time
# For now we will log and do no more. The offending folder
# will still contain the export, but will lose the information
# that it is already scanned and thus will be re-scanned on
# next run.
logger.error('Rename error from {}'.format(self.current_path))
return attachments
def __init__(self, scan_id):
multiprocessing.Process.__init__(self)
print('Program started')
self.scan_id = scan_id
django.setup()
from os2webscanner.models.scan_model import Scan
scan_object = Scan.objects.get(pk=self.scan_id)
valid_domains = scan_object.domains.filter(
validation_status=Domain.VALID
)
"""Making scan dir if it does not exists"""
if not os.path.exists(scan_object.scan_dir):
print('Creating scan dir {}'.format(scan_object.scan_dir))
os.makedirs(scan_object.scan_dir)
scan_dir = scan_object.scan_dir + '/'
"""Handling last scannings date"""
last_scannings_date = None
if scan_object.do_last_modified_check:
last_scannings_date = scan_object.exchangescan.last_scannings_date
if last_scannings_date:
last_scannings_date = EWSDate.from_date(
last_scannings_date)
"""Foreach domain x number of mail processors are started."""
for domain in valid_domains:
credentials = (domain.authentication.username,
domain.authentication.get_password())
self.user_queue = multiprocessing.Queue()
read_users(self.user_queue,
domain.exchangedomain.get_userlist_file_path())
self.done_queue = multiprocessing.Queue()
mail_ending = domain.url
scanners = {}
for i in range(0, NUMBER_OF_EMAIL_THREADS):
scanners[i] = ExchangeServerScan(credentials,
self.user_queue,
self.done_queue,
scan_dir,
mail_ending,
start_date=
last_scannings_date)
scanners[i].start()
print('Started scanner {}'.format(i))
time.sleep(1)
self.scanners = scanners
print('Scanners started...')
def __init__(self, scan_id):
multiprocessing.Process.__init__(self)
print('Program started')
self.scan_id = scan_id
django.setup()
from os2webscanner.models.scan_model import Scan
scan_object = Scan.objects.get(pk=self.scan_id)
valid_domains = scan_object.domains.filter(
validation_status=Domain.VALID)
"""Making scan dir if it does not exists"""
if not os.path.exists(scan_object.scan_dir):
print('Creating scan dir {}'.format(scan_object.scan_dir))
os.makedirs(scan_object.scan_dir)
scan_dir = scan_object.scan_dir + '/'
"""Handling last scannings date"""
last_scannings_date = None
if scan_object.do_last_modified_check:
last_scannings_date = scan_object.exchangescan.last_scannings_date
if last_scannings_date:
last_scannings_date = EWSDate.from_date(last_scannings_date)
"""Foreach domain x number of mail processors are started."""
for domain in valid_domains:
credentials = (domain.authentication.username,
domain.authentication.get_password())
self.user_queue = multiprocessing.Queue()
read_users(self.user_queue,
domain.exchangedomain.get_userlist_file_path())
self.done_queue = multiprocessing.Queue()
mail_ending = domain.url
scanners = {}
for i in range(0, NUMBER_OF_EMAIL_THREADS):
scanners[i] = ExchangeServerScan(
credentials,
self.user_queue,
self.done_queue,
scan_dir,
mail_ending,
start_date=last_scannings_date)
scanners[i].start()
print('Started scanner {}'.format(i))
time.sleep(1)
self.scanners = scanners
print('Scanners started...')
def export_folder(self, folder):
""" Export a given folder
:param folder: The folder to export
:return: The number of exported attachments
"""
folder_name = folder.name.replace(' ', '_').replace('/', '_')
self.current_path = self.export_path.joinpath(folder_name)
if self.export_path.joinpath(folder_name + '_done').exists():
logger.info('Already done: {}'.format(self.current_path))
return folder.total_count # Already scanned
if self.current_path.exists():
logger.info('Clean up: {}'.format(self.current_path))
shutil.rmtree(str(self.current_path))
self.current_path.mkdir()
attachments = 0
if self.start_date is None:
start_dt = EWSDate(2010, 1, 1)
# First, export everything before 2010
attachments += self._attempt_export(folder, end_dt=start_dt)
else:
start_dt = self.start_date
end_dt = start_dt + timedelta(days=10)
while end_dt < (EWSDate.today() + timedelta(days=10)):
msg = 'Export folder, currently from {} to {}'
logger.debug(msg.format(start_dt, end_dt))
attachments += self._attempt_export(folder, start_dt=start_dt,
end_dt=end_dt)
start_dt = end_dt
end_dt = start_dt + timedelta(days=10)
# Finally, export everything later than today (hopefully nothing)
attachments += self._attempt_export(folder, start_dt=end_dt)
try:
self.current_path.rename(str(self.current_path) + '_done')
except FileNotFoundError:
# This can happen if a user mistakenly is scanned twice at
# the same time
# For now we will log and do no more. The offending folder
# will still contain the export, but will lose the information
# that it is already scanned and thus will be re-scanned on
# next run.
logger.error('Rename error from {}'.format(self.current_path))
return attachments
def test_oof_settings(self):
# First, ensure a common starting point
self.account.oof_settings = OofSettings(state=OofSettings.DISABLED)
oof = OofSettings(
state=OofSettings.ENABLED,
external_audience='None',
internal_reply="I'm on holidays. See ya guys!",
external_reply='Dear Sir, your email has now been deleted.',
)
self.account.oof_settings = oof
self.assertEqual(self.account.oof_settings, oof)
oof = OofSettings(
state=OofSettings.ENABLED,
external_audience='Known',
internal_reply='XXX',
external_reply='YYY',
)
self.account.oof_settings = oof
self.assertEqual(self.account.oof_settings, oof)
# Scheduled duration must not be in the past
start, end = get_random_datetime_range(start_date=EWSDate.today())
oof = OofSettings(
state=OofSettings.SCHEDULED,
external_audience='Known',
internal_reply="I'm in the pub. See ya guys!",
external_reply="I'm having a business dinner in town",
start=start,
end=end,
)
self.account.oof_settings = oof
self.assertEqual(self.account.oof_settings, oof)
oof = OofSettings(
state=OofSettings.DISABLED,
)
self.account.oof_settings = oof
self.assertEqual(self.account.oof_settings, oof)
def test_ewsdate(self):
self.assertEqual(EWSDate(2000, 1, 1).ewsformat(), '2000-01-01')
self.assertEqual(EWSDate.from_string('2000-01-01'),
EWSDate(2000, 1, 1))
self.assertEqual(EWSDate.from_string('2000-01-01Z'),
EWSDate(2000, 1, 1))
self.assertEqual(EWSDate.from_string('2000-01-01+01:00'),
EWSDate(2000, 1, 1))
self.assertEqual(EWSDate.from_string('2000-01-01-01:00'),
EWSDate(2000, 1, 1))
self.assertIsInstance(
EWSDate(2000, 1, 2) - EWSDate(2000, 1, 1), datetime.timedelta)
self.assertIsInstance(
EWSDate(2000, 1, 2) + datetime.timedelta(days=1), EWSDate)
self.assertIsInstance(
EWSDate(2000, 1, 2) - datetime.timedelta(days=1), EWSDate)
# Test in-place add and subtract
dt = EWSDate(2000, 1, 2)
dt += datetime.timedelta(days=1)
self.assertIsInstance(dt, EWSDate)
self.assertEqual(dt, EWSDate(2000, 1, 3))
dt = EWSDate(2000, 1, 2)
dt -= datetime.timedelta(days=1)
self.assertIsInstance(dt, EWSDate)
self.assertEqual(dt, EWSDate(2000, 1, 1))
with self.assertRaises(ValueError):
EWSDate.from_date(EWSDate(2000, 1, 2))
print("password: "******"timezone: ", timezone)
print()
# Set up account:
creds = Credentials(username=username, password=password)
config = Configuration(server=server, credentials=creds)
account = Account(primary_smtp_address=emailAddress,
autodiscover=False,
config=config,
access_type=DELEGATE)
# Start and end date to fetch calendar items for:
today = EWSDate.today() + timedelta(days=deltaDay)
startDate = account.default_timezone.localize(
EWSDateTime(today.year, today.month, today.day, 0, 0, 0))
endDate = account.default_timezone.localize(
EWSDateTime(today.year, today.month, today.day, 23, 59, 59))
# Desired timezone:
if timezone == '':
tz = EWSTimeZone.localzone()
else:
tz = EWSTimeZone.timezone(timezone)
# Start output:
print()
print(
colored('Appointments for ' + startDate.strftime('%A %d %B %Y') + ':',
if len(user.strip()) == 0:
users.remove(user)
for user in users:
user_queue.put(user)
if __name__ == '__main__':
import settings_local as settings
import password
amqp = True
credentials = ('mailscan', password.password)
number_of_threads = int(sys.argv[1])
try:
start_arg = datetime.strptime(sys.argv[2], '%Y-%m-%d')
start_date = EWSDate.from_date(start_arg)
except IndexError:
start_date = None # Scan from beginning of time
# Populate user queue
user_queue = Queue()
done_queue = Queue()
read_users(user_queue, settings.user_path)
# Stats should be run before the scanners to allow stats to mke the
# correct initial-value measurements
stats = Stats(user_queue, log_data=True)
for i in range(0, number_of_threads):
scanner = ExchangeServerExport(credentials, user_queue, done_queue,
settings.export_path,
tz = EWSTimeZone.timezone('Europe/Copenhagen')
# You can also get the local timezone defined in your operating system
tz = EWSTimeZone.localzone()
# EWSDate and EWSDateTime work just like datetime.datetime and datetime.date. Always create
# timezone-aware datetimes with EWSTimeZone.localize():
localized_dt = tz.localize(EWSDateTime(2017, 9, 5, 8, 30))
right_now = tz.localize(EWSDateTime.now())
# Datetime math works transparently
two_hours_later = localized_dt + timedelta(hours=2)
two_hours = two_hours_later - localized_dt
two_hours_later += timedelta(hours=2)
# Dates
my_date = EWSDate(2017, 9, 5)
today = EWSDate.today()
also_today = right_now.date()
also_today += timedelta(days=10)
# UTC helpers. 'UTC' is the UTC timezone as an EWSTimeZone instance.
# 'UTC_NOW' returns a timezone-aware UTC timestamp of current time.
from exchangelib import UTC, UTC_NOW
right_now_in_utc = UTC.localize(EWSDateTime.now())
right_now_in_utc = UTC_NOW()
# Already have a Python datetime object you want to use? Make sure it's localized. Then pass
# it to from_datetime().
pytz_tz = pytz.timezone('Europe/Copenhagen')
py_dt = pytz_tz.localize(datetime(2017, 12, 11, 10, 9, 8))