def test_send_warnings_after_disable(self):
# Test that required number of warnings are sent after the delivery is
# disabled.
self._mlist.bounce_notify_owner_on_disable = False
self._mlist.bounce_you_are_disabled_warnings = 1
self._mlist.bounce_you_are_disabled_warnings_interval = timedelta(
days=1)
self._mlist.bounce_score_threshold = 3
self._mlist.send_welcome_message = False
member = self._subscribe_and_add_bounce_event('*****@*****.**')
member.bounce_score = 3
member.last_bounce_received = now() - timedelta(days=2)
# We will process all events now.
self._process_pending_events()
self.assertEqual(member.preferences.delivery_status,
DeliveryStatus.by_bounces)
self.assertEqual(member.bounce_score, 4)
self._processor._send_warnings()
self.assertEqual(member.last_warning_sent.day, now().day)
self.assertEqual(member.total_warnings_sent, 1)
msgs = get_queue_messages('virgin', expected_count=1)
msg = msgs[0].msg
self.assertEqual(
str(msg['Subject']), 'Your subscription for Test mailing list has'
' been disabled')
def test_member_changes_preferred_address(self):
with transaction():
anne = self._usermanager.create_user('*****@*****.**')
preferred = list(anne.addresses)[0]
preferred.verified_on = now()
anne.preferred_address = preferred
self._mlist.subscribe(anne)
# Take a look at Anne's current membership.
content, response = call_api('http://*****:*****@example.com')
self.assertEqual(
entry_0['address'],
'http://*****:*****@example.com')
# Anne registers a new address and makes it her preferred address.
# There are no changes to her membership.
with transaction():
new_preferred = anne.register('*****@*****.**')
new_preferred.verified_on = now()
anne.preferred_address = new_preferred
# Take another look at Anne's current membership.
content, response = call_api('http://*****:*****@example.com')
self.assertEqual(
entry_0['address'],
'http://*****:*****@example.com')
def test_cannot_set_address_with_preferred_address_subscription(self):
# A user is subscribed to a mailing list with their preferred address.
# You cannot set the `address` attribute on such IMembers.
anne = self._usermanager.create_user("*****@*****.**")
preferred = list(anne.addresses)[0]
preferred.verified_on = now()
anne.preferred_address = preferred
# Subscribe with the IUser object, not the address. This makes Anne a
# member via her preferred address.
member = self._mlist.subscribe(anne)
new_address = anne.register("*****@*****.**")
new_address.verified_on = now()
self.assertRaises(MembershipError, setattr, member, "address", new_address)
def test_cannot_set_address_with_preferred_address_subscription(self):
# A user is subscribed to a mailing list with their preferred address.
# You cannot set the `address` attribute on such IMembers.
anne = self._usermanager.create_user('*****@*****.**')
preferred = list(anne.addresses)[0]
preferred.verified_on = now()
anne.preferred_address = preferred
# Subscribe with the IUser object, not the address. This makes Anne a
# member via her preferred address.
member = self._mlist.subscribe(anne)
new_address = anne.register('*****@*****.**')
new_address.verified_on = now()
self.assertRaises(MembershipError, setattr, member, 'address',
new_address)
def test_archive_runner_with_dated_message(self):
# Date headers don't throw off the archiver runner.
self._msg['Date'] = now(strip_tzinfo=False).strftime(RFC822_DATE_FMT)
self._archiveq.enqueue(self._msg, {},
listid=self._mlist.list_id,
received_time=now())
self._runner.run()
# There should now be a copy of the message in the file system.
filename = os.path.join(config.MESSAGES_DIR,
'4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB')
with open(filename) as fp:
archived = message_from_file(fp)
self.assertEqual(archived['message-id'], '<first>')
self.assertEqual(archived['date'], 'Mon, 01 Aug 2005 07:49:23 +0000')
def ensure_current_suffix_list():
# Read and parse the organizational domain suffix list. First look in the
# cached directory to see if we already have a valid copy of it.
cached_copy_path = os.path.join(config.VAR_DIR, LOCAL_FILE_NAME)
lifetime = as_timedelta(config.dmarc.cache_lifetime)
download = False
try:
mtime = os.stat(cached_copy_path).st_mtime
except FileNotFoundError:
vlog.info('No cached copy of the public suffix list found')
download = True
cache_found = False
else:
cache_found = True
# Is the cached copy out-of-date? Note that when we write a new cache
# version we explicitly set its mtime to the time in the future when
# the cache will expire.
if mtime < now().timestamp():
download = True
vlog.info('Cached copy of public suffix list is out of date')
if download:
try:
content = get(config.dmarc.org_domain_data_url)
except (URLError, HTTPError) as error:
elog.error('Unable to retrieve public suffix list from %s: %s',
config.dmarc.org_domain_data_url,
getattr(error, 'reason', str(error)))
if cache_found:
vlog.info('Using out of date public suffix list')
content = None
else:
# We couldn't access the URL and didn't even have an out of
# date suffix list cached. Use the shipped version.
content = resource_bytes('mailman.rules.data', LOCAL_FILE_NAME)
if content is not None:
# Content is either a string or UTF-8 encoded bytes.
if isinstance(content, bytes):
content = content.decode('utf-8')
# Write the cache atomically.
new_path = cached_copy_path + '.new'
with open(new_path, 'w', encoding='utf-8') as fp:
fp.write(content)
# Set the expiry time to the future.
mtime = (now() + lifetime).timestamp()
os.utime(new_path, (mtime, mtime))
# Flip the new file into the cached location. This does not
# modify the mtime.
os.rename(new_path, cached_copy_path)
return cached_copy_path
def test_archive_runner_with_dated_message(self):
# Date headers don't throw off the archiver runner.
self._msg['Date'] = now(strip_tzinfo=False).strftime(RFC822_DATE_FMT)
self._archiveq.enqueue(
self._msg, {},
listid=self._mlist.list_id,
received_time=now())
self._runner.run()
# There should now be a copy of the message in the file system.
filename = os.path.join(
config.MESSAGES_DIR, '4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB')
with open(filename) as fp:
archived = message_from_file(fp)
self.assertEqual(archived['message-id'], '<first>')
self.assertEqual(archived['date'], 'Mon, 01 Aug 2005 07:49:23 +0000')
def test_clobber_date_never(self):
# Even if the Date header is insanely off from the received time of
# the message, if clobber_date is 'never', the header is not clobbered.
self._msg['Date'] = now(strip_tzinfo=False).strftime(RFC822_DATE_FMT)
self._archiveq.enqueue(self._msg, {},
listid=self._mlist.list_id,
received_time=now())
self._runner.run()
# There should now be a copy of the message in the file system.
filename = os.path.join(config.MESSAGES_DIR,
'4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB')
with open(filename) as fp:
archived = message_from_file(fp)
self.assertEqual(archived['message-id'], '<first>')
self.assertEqual(archived['date'], 'Mon, 01 Aug 2005 07:49:23 +0000')
def test_update_display_name(self):
verified_on = now()
with transaction():
anne = getUtility(IUserManager).create_address('*****@*****.**')
anne.verified_on = verified_on
json, response = call_api(
'http://*****:*****@example.com')
# By default, there won't be any display_name since we didn't specify
# any above when creating the address.
self.assertFalse('display_name' in json)
json, response = call_api(
'http://*****:*****@example.com',
method='PATCH',
data={'display_name': 'Anne Person'})
self.assertEqual(response.status_code, 204)
# Now the address should have a display_name set.
json, response = call_api(
'http://*****:*****@example.com')
self.assertEqual(json['display_name'], 'Anne Person')
# This name can also be updated.
json, response = call_api(
'http://*****:*****@example.com',
method='PATCH',
data={'display_name': 'Anne P. Person'})
self.assertEqual(response.status_code, 204)
# Now the address should have a display_name updated to new one.
json, response = call_api(
'http://*****:*****@example.com')
self.assertEqual(json['display_name'], 'Anne P. Person')
def setUp(self):
self._mlist = create_list('*****@*****.**')
self._now = now()
# Enable just the dummy archiver.
config.push('dummy', """
[archiver.dummy]
class: mailman.runners.tests.test_archiver.DummyArchiver
enable: no
[archiver.prototype]
enable: no
[archiver.mhonarc]
enable: no
[archiver.mail_archive]
enable: no
""")
self._archiveq = config.switchboards['archive']
self._msg = mfs("""\
From: [email protected]
To: [email protected]
Subject: My first post
Message-ID: <first>
X-Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB
First post!
""")
self._runner = make_testable_runner(ArchiveRunner)
IListArchiverSet(self._mlist).get('dummy').is_enabled = True
def test_clobber_date_never(self):
# Even if the Date header is insanely off from the received time of
# the message, if clobber_date is 'never', the header is not clobbered.
self._msg['Date'] = now(strip_tzinfo=False).strftime(RFC822_DATE_FMT)
self._archiveq.enqueue(
self._msg, {},
listid=self._mlist.list_id,
received_time=now())
self._runner.run()
# There should now be a copy of the message in the file system.
filename = os.path.join(
config.MESSAGES_DIR, '4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB')
with open(filename) as fp:
archived = message_from_file(fp)
self.assertEqual(archived['message-id'], '<first>')
self.assertEqual(archived['date'], 'Mon, 01 Aug 2005 07:49:23 +0000')
def test_events_membership_removal_no_warnings(self):
self._mlist.bounce_notify_owner_on_removal = True
self._mlist.bounce_you_are_disabled_warnings = 0
self._mlist.bounce_you_are_disabled_warnings_interval = timedelta(
days=2)
anne = self._mlist.members.get_member(self._anne.email)
anne.preferences.delivery_status = DeliveryStatus.by_bounces
anne.total_warnings_sent = 0
# Remove immediately.
anne.last_warning_sent = now()
self._runner.run()
items = get_queue_messages('virgin', expected_count=2)
if items[0].msg['to'] == '*****@*****.**':
owner_notif, user_notif = items
else:
user_notif, owner_notif = items
self.assertEqual(user_notif.msg['to'], '*****@*****.**')
self.assertEqual(
user_notif.msg['subject'],
'You have been unsubscribed from the Test mailing list')
self.assertEqual(
str(owner_notif.msg['subject']),
'[email protected] unsubscribed from Test mailing '
'list due to bounces')
# The membership should no longer exist.
self.assertIsNone(self._mlist.members.get_member(self._anne.email))
def test_transaction_abort_after_failing_subcommand(self):
with transaction():
mlist = create_list('*****@*****.**')
mlist.volume = 5
mlist.next_digest_number = 3
mlist.digest_last_sent_at = now() - timedelta(days=60)
testargs = [
'mailman', 'digests', '-b', '-l', '*****@*****.**', '--send'
]
output = StringIO()
with ExitStack() as resources:
enter = resources.enter_context
enter(patch('sys.argv', testargs))
enter(patch('sys.stdout', output))
# Force an exception in the subcommand.
enter(
patch('mailman.commands.cli_digests.maybe_send_digest_now',
side_effect=RuntimeError))
# Everything is already initialized.
enter(patch('mailman.bin.mailman.initialize'))
with self.assertRaises(RuntimeError):
main()
# Clear the current transaction to force a database reload.
config.db.abort()
# The volume and number haven't changed.
self.assertEqual(mlist.volume, 5)
self.assertEqual(mlist.next_digest_number, 3)
def __init__(self, email, display_name):
super(Address, self).__init__()
lower_case = email.lower()
self.email = lower_case
self.display_name = display_name
self._original = (None if lower_case == email else email)
self.registered_on = now()
def memberships_pending_warning(self, store):
"""See `IMembershipManager`."""
from mailman.model.mailinglist import MailingList
from mailman.model.preferences import Preferences
# maxking: We don't care so much about the bounce score here since it
# could have been reset due to bounce info getting stale. We will send
# warnings to people who have been disabled already, regardless of
# their bounce score. Same is true below for removal.
query = store.query(Member).join(
MailingList, Member.list_id ==
MailingList._list_id).join(Member.preferences).filter(
and_(
MailingList.process_bounces == True, # noqa: E712
Member.total_warnings_sent <
MailingList.bounce_you_are_disabled_warnings, # noqa: E501
Preferences.delivery_status == DeliveryStatus.by_bounces))
# XXX(maxking): This is IMO a query that *should* work, but I haven't
# been able to get it to work in my tests. It could be due to lack of
# native datetime type in SQLite, which we use for tests. Hence, we are
# going to do some filtering in Python, which is inefficient, but
# works. We do filter as much as possible in SQL, so hopefully the
# output of the above query won't be *too* many.
# TODO: Port the Python loop below to SQLAlchemy.
# (func.DATETIME(Member.last_warning_sent) +
# func.DATETIME(MailingList.bounce_you_are_disabled_warnings_interval))
# < func.DATETIME(now())))
for member in query.all():
if (member.last_warning_sent + member.mailing_list.
bounce_you_are_disabled_warnings_interval
) <= now(): # noqa: E501
yield member
def setUp(self):
self._manager = getUtility(IUserManager)
self._mlist = create_list("*****@*****.**")
self._anne = self._manager.create_user("*****@*****.**", "Anne Person")
preferred = list(self._anne.addresses)[0]
preferred.verified_on = now()
self._anne.preferred_address = preferred
def _generate_lmtp_file(self, fp):
# The format for Postfix's LMTP transport map is defined here:
# http://www.postfix.org/transport.5.html
#
# Sort all existing mailing list names first by domain, then by
# local part. For Postfix we need a dummy entry for the domain.
list_manager = getUtility(IListManager)
utility = getUtility(IMailTransportAgentAliases)
by_domain = {}
sort_key = attrgetter('list_name')
for list_name, mail_host in list_manager.name_components:
mlist = _FakeList(list_name, mail_host)
by_domain.setdefault(mlist.mail_host, []).append(mlist)
print("""\
# AUTOMATICALLY GENERATED BY MAILMAN ON {0}
#
# This file is generated by Mailman, and is kept in sync with the binary hash
# file. YOU SHOULD NOT MANUALLY EDIT THIS FILE unless you know what you're
# doing, and can keep the two files properly in sync. If you screw it up,
# you're on your own.
""".format(now().replace(microsecond=0)), file=fp)
for domain in sorted(by_domain):
print("""\
# Aliases which are visible only in the @{0} domain.""".format(domain),
file=fp)
for mlist in sorted(by_domain[domain], key=sort_key):
aliases = list(utility.aliases(mlist))
width = max(len(alias) for alias in aliases) + 3
print(ALIASTMPL.format(aliases.pop(0), config, width), file=fp)
for alias in aliases:
print(ALIASTMPL.format(alias, config, width), file=fp)
print(file=fp)
def test_transaction_abort_after_failing_subcommand(self):
with transaction():
mlist = create_list('*****@*****.**')
mlist.volume = 5
mlist.next_digest_number = 3
mlist.digest_last_sent_at = now() - timedelta(days=60)
testargs = ['mailman', 'digests', '-b', '-l', '*****@*****.**',
'--send']
output = StringIO()
with ExitStack() as resources:
enter = resources.enter_context
enter(patch('sys.argv', testargs))
enter(patch('sys.stdout', output))
# Force an exception in the subcommand.
enter(patch('mailman.commands.cli_digests.maybe_send_digest_now',
side_effect=RuntimeError))
# Everything is already initialized.
enter(patch('mailman.bin.mailman.initialize'))
with self.assertRaises(RuntimeError):
main()
# Clear the current transaction to force a database reload.
config.db.abort()
# The volume and number haven't changed.
self.assertEqual(mlist.volume, 5)
self.assertEqual(mlist.next_digest_number, 3)
def test_member_subscribed_address_has_no_display_name(self):
address = getUtility(IUserManager).create_address("*****@*****.**")
address.verified_on = now()
self._mlist.subscribe(address)
items = get_queue_messages("virgin", expected_count=1)
message = items[0].msg
self.assertEqual(message["to"], "*****@*****.**")
def test_disable_delivery_already_disabled(self):
# Attempting to disable delivery for an already disabled member does
# nothing.
self._mlist.send_welcome_message = False
member = self._subscribe_and_add_bounce_event('*****@*****.**')
events = list(self._processor.events)
self.assertEqual(len(events), 1)
member.total_warnings_sent = 3
member.last_warning_sent = now() - timedelta(days=2)
member.preferences.delivery_status = DeliveryStatus.by_bounces
mark = LogFileMark('mailman.bounce')
self._processor._disable_delivery(self._mlist, member, events[0])
self.assertEqual(mark.read(), '')
self.assertEqual(member.total_warnings_sent, 3)
self.assertEqual(member.last_warning_sent, now() - timedelta(days=2))
get_queue_messages('virgin', expected_count=0)
def add(self, store, pendable, lifetime=None):
verifyObject(IPendable, pendable)
# Calculate the token and the lifetime.
if lifetime is None:
lifetime = as_timedelta(config.mailman.pending_request_life)
# Calculate a unique token. Algorithm vetted by the Timbot. time()
# has high resolution on Linux, clock() on Windows. random gives us
# about 45 bits in Python 2.2, 53 bits on Python 2.3. The time and
# clock values basically help obscure the random number generator, as
# does the hash calculation. The integral parts of the time values
# are discarded because they're the most predictable bits.
for attempts in range(3):
right_now = time.time()
x = random.random() + right_now % 1.0 + time.clock() % 1.0
# Use sha1 because it produces shorter strings.
token = hashlib.sha1(repr(x).encode("utf-8")).hexdigest()
# In practice, we'll never get a duplicate, but we'll be anal
# about checking anyway.
if store.query(Pended).filter_by(token=token).count() == 0:
break
else:
raise RuntimeError("Could not find a valid pendings token")
# Create the record, and then the individual key/value pairs.
pending = Pended(token=token, expiration_date=now() + lifetime)
for key, value in pendable.items():
# Both keys and values must be strings.
if isinstance(key, bytes):
key = key.decode("utf-8")
if isinstance(value, bytes):
# Make sure we can turn this back into a bytes.
value = dict(__encoding__="utf-8", value=value.decode("utf-8"))
keyval = PendedKeyValue(key=key, value=json.dumps(value))
pending.key_values.append(keyval)
store.add(pending)
return token
def test_delete_nonmembers_on_adding_member(self):
# GL: #237 - When a new address is subscribed, any existing nonmember
# subscriptions for this address; or any addresses also controlled by
# this user, are deleted.
anne_nonmember = add_member(
self._mlist,
RequestRecord('*****@*****.**', 'Anne Person',
DeliveryMode.regular,
system_preferences.preferred_language),
MemberRole.nonmember)
# Add a few other validated addresses to this user, and subscribe them
# as nonmembers.
for email in ('*****@*****.**', '*****@*****.**'):
address = anne_nonmember.user.register(email)
address.verified_on = now()
self._mlist.subscribe(address, MemberRole.nonmember)
# There are now three nonmembers.
self.assertEqual(
{address.email for address in self._mlist.nonmembers.addresses},
{'*****@*****.**',
'*****@*****.**',
'*****@*****.**',
})
# Let's now add one of Anne's addresses as a member. This deletes all
# of Anne's nonmember memberships.
anne_member = add_member(
self._mlist,
RequestRecord('*****@*****.**', 'Anne Person',
DeliveryMode.regular,
system_preferences.preferred_language),
MemberRole.member)
self.assertEqual(self._mlist.nonmembers.member_count, 0)
members = list(self._mlist.members.members)
self.assertEqual(len(members), 1)
self.assertEqual(members[0], anne_member)
def add(self, store, pendable, lifetime=None):
verifyObject(IPendable, pendable)
# Calculate the token and the lifetime.
if lifetime is None:
lifetime = as_timedelta(config.mailman.pending_request_life)
for attempts in range(3):
token = token_factory.new()
# In practice, we'll never get a duplicate, but we'll be anal
# about checking anyway.
if store.query(Pended).filter_by(token=token).count() == 0:
break
else:
raise RuntimeError('Could not find a valid pendings token')
# Create the record, and then the individual key/value pairs.
pending = Pended(token=token, expiration_date=now() + lifetime)
pendable_type = pendable.get('type', pendable.PEND_TYPE)
pending.key_values.append(
PendedKeyValue(key='type', value=pendable_type))
for key, value in pendable.items():
# The type has been handled above.
if key == 'type':
continue
# Both keys and values must be strings.
if isinstance(key, bytes):
key = key.decode('utf-8')
if isinstance(value, bytes):
# Make sure we can turn this back into a bytes.
value = dict(__encoding__='utf-8', value=value.decode('utf-8'))
keyval = PendedKeyValue(key=key, value=json.dumps(value))
pending.key_values.append(keyval)
store.add(pending)
return token
def test_events_disable_delivery(self):
self._mlist.bounce_score_threshold = 3
anne = self._subscribe_and_add_bounce_event('*****@*****.**',
subscribe=False,
create=False)
anne.bounce_score = 2
anne.last_bounce_received = now() - timedelta(days=2)
self._runner.run()
self.assertEqual(anne.bounce_score, 3)
self.assertEqual(anne.preferences.delivery_status,
DeliveryStatus.by_bounces)
# There should also be a pending notification for the the list
# administrator.
items = get_queue_messages('virgin', expected_count=2)
if items[0].msg['to'] == '*****@*****.**':
owner_notif, disable_notice = items
else:
disable_notice, owner_notif = items
self.assertEqual(owner_notif.msg['Subject'],
"*****@*****.**'s subscription disabled on Test")
self.assertEqual(disable_notice.msg['to'], '*****@*****.**')
self.assertEqual(
str(disable_notice.msg['subject']),
'Your subscription for Test mailing list has been disabled')
def hold_subscription(mlist, address, display_name, password, mode, language):
data = dict(when=now().isoformat(),
address=address,
display_name=display_name,
password=password,
delivery_mode=mode.name,
language=language)
# Now hold this request. We'll use the address as the key.
requestsdb = IListRequests(mlist)
request_id = requestsdb.hold_request(
RequestType.subscription, address, data)
vlog.info('%s: held subscription request from %s',
mlist.fqdn_listname, address)
# Possibly notify the administrator in default list language
if mlist.admin_immed_notify:
subject = _(
'New subscription request to $mlist.display_name from $address')
text = make('subauth.txt',
mailing_list=mlist,
username=address,
listname=mlist.fqdn_listname,
admindb_url=mlist.script_url('admindb'),
)
# This message should appear to come from the <list>-owner so as
# to avoid any useless bounce processing.
msg = UserNotification(
mlist.owner_address, mlist.owner_address,
subject, text, mlist.preferred_language)
msg.send(mlist, tomoderators=True)
return request_id
def setUp(self):
self._mlist = create_list('*****@*****.**')
self._anne = getUtility(IUserManager).create_user(
'*****@*****.**', 'Anne Person')
preferred = list(self._anne.addresses)[0]
preferred.verified_on = now()
self._anne.preferred_address = preferred
def _generate_lmtp_file(self, fp):
# The format for Postfix's LMTP transport map is defined here:
# http://www.postfix.org/transport.5.html
#
# Sort all existing mailing list names first by domain, then by
# local part. For Postfix we need a dummy entry for the domain.
list_manager = getUtility(IListManager)
utility = getUtility(IMailTransportAgentAliases)
by_domain = {}
sort_key = attrgetter('list_name')
for list_name, mail_host in list_manager.name_components:
mlist = _FakeList(list_name, mail_host)
by_domain.setdefault(mlist.mail_host, []).append(mlist)
print("""\
# AUTOMATICALLY GENERATED BY MAILMAN ON {0}
#
# This file is generated by Mailman, and is kept in sync with the binary hash
# file. YOU SHOULD NOT MANUALLY EDIT THIS FILE unless you know what you're
# doing, and can keep the two files properly in sync. If you screw it up,
# you're on your own.
""".format(now().replace(microsecond=0)), file=fp)
for domain in sorted(by_domain):
print("""\
# Aliases which are visible only in the @{0} domain.""".format(domain),
file=fp)
for mlist in sorted(by_domain[domain], key=sort_key):
aliases = list(utility.aliases(mlist))
width = max(len(alias) for alias in aliases) + 3
print(ALIASTMPL.format(aliases.pop(0), config, width), file=fp)
for alias in aliases:
print(ALIASTMPL.format(alias, config, width), file=fp)
print(file=fp)
def add(self, store, pendable, lifetime=None):
verifyObject(IPendable, pendable)
# Calculate the token and the lifetime.
if lifetime is None:
lifetime = as_timedelta(config.mailman.pending_request_life)
for attempts in range(3):
token = token_factory.new()
# In practice, we'll never get a duplicate, but we'll be anal
# about checking anyway.
if store.query(Pended).filter_by(token=token).count() == 0:
break
else:
raise RuntimeError('Could not find a valid pendings token')
# Create the record, and then the individual key/value pairs.
pending = Pended(
token=token,
expiration_date=now() + lifetime)
pendable_type = pendable.get('type', pendable.PEND_TYPE)
pending.key_values.append(
PendedKeyValue(key='type', value=pendable_type))
for key, value in pendable.items():
# The type has been handled above.
if key == 'type':
continue
# Both keys and values must be strings.
if isinstance(key, bytes):
key = key.decode('utf-8')
if isinstance(value, bytes):
# Make sure we can turn this back into a bytes.
value = dict(__encoding__='utf-8',
value=value.decode('utf-8'))
keyval = PendedKeyValue(key=key, value=json.dumps(value))
pending.key_values.append(keyval)
store.add(pending)
return token
def set_preferred(user):
# Avoid circular imports.
from mailman.utilities.datetime import now
preferred = list(user.addresses)[0]
preferred.verified_on = now()
user.preferred_address = preferred
return preferred
def hold_subscription(mlist, address, display_name, password, mode, language):
data = dict(when=now().isoformat(),
address=address,
display_name=display_name,
password=password,
delivery_mode=mode.name,
language=language)
# Now hold this request. We'll use the address as the key.
requestsdb = IListRequests(mlist)
request_id = requestsdb.hold_request(RequestType.subscription, address,
data)
vlog.info('%s: held subscription request from %s', mlist.fqdn_listname,
address)
# Possibly notify the administrator in default list language
if mlist.admin_immed_notify:
subject = _(
'New subscription request to $mlist.display_name from $address')
text = make(
'subauth.txt',
mailing_list=mlist,
username=address,
listname=mlist.fqdn_listname,
admindb_url=mlist.script_url('admindb'),
)
# This message should appear to come from the <list>-owner so as
# to avoid any useless bounce processing.
msg = UserNotification(mlist.owner_address, mlist.owner_address,
subject, text, mlist.preferred_language)
msg.send(mlist, tomoderators=True)
return request_id
def test_member_changes_preferred_address(self):
with transaction():
anne = self._usermanager.create_user('*****@*****.**')
_set_preferred(anne)
self._mlist.subscribe(anne)
# Take a look at Anne's current membership.
content, response = call_api('http://*****:*****@example.com')
self.assertEqual(
entry_0['address'],
'http://*****:*****@example.com')
# Anne registers a new address and makes it her preferred address.
# There are no changes to her membership.
with transaction():
new_preferred = anne.register('*****@*****.**')
new_preferred.verified_on = now()
anne.preferred_address = new_preferred
# Take another look at Anne's current membership.
content, response = call_api('http://*****:*****@example.com')
self.assertEqual(
entry_0['address'],
'http://*****:*****@example.com')
def test_confirm_because_confirm_then_moderation(self):
# We have a subscription request which requires the user to confirm
# (because she does not have a verified address) and the moderator to
# approve. Running the workflow gives us a token. Confirming the
# token runs the workflow a little farther, but still gives us a
# token. Confirming again subscribes the user.
self._mlist.subscription_policy = (
SubscriptionPolicy.confirm_then_moderate)
self._anne.verified_on = now()
# Runs until subscription confirmation.
token, token_owner, rmember = self._registrar.register(self._anne)
self.assertIsNotNone(token)
self.assertEqual(token_owner, TokenOwner.subscriber)
self.assertIsNone(rmember)
member = self._mlist.regular_members.get_member('*****@*****.**')
self.assertIsNone(member)
# Now confirm the subscription, and wait for the moderator to approve
# the subscription. She is still not subscribed.
new_token, token_owner, rmember = self._registrar.confirm(token)
# The new token, used for the moderator to approve the message, is not
# the same as the old token.
self.assertNotEqual(new_token, token)
self.assertIsNotNone(new_token)
self.assertEqual(token_owner, TokenOwner.moderator)
self.assertIsNone(rmember)
member = self._mlist.regular_members.get_member('*****@*****.**')
self.assertIsNone(member)
# Confirm once more, this time as the moderator approving the
# subscription. Now she's a member.
token, token_owner, rmember = self._registrar.confirm(new_token)
self.assertIsNone(token)
self.assertEqual(token_owner, TokenOwner.no_one)
member = self._mlist.regular_members.get_member('*****@*****.**')
self.assertEqual(rmember, member)
self.assertEqual(member.address, self._anne)
def _set_preferred(user):
# Avoid circular imports.
from mailman.utilities.datetime import now
preferred = list(user.addresses)[0]
preferred.verified_on = now()
user.preferred_address = preferred
return preferred
def __init__(self, list_name, email, msg, context=None):
self.list_name = list_name
self.email = email
self.timestamp = now()
self.message_id = msg["message-id"]
self.context = BounceContext.normal if context is None else context
self.processed = False
def __init__(self, list_id, email, msg, context=None):
self.list_id = list_id
self.email = email
self.timestamp = now()
self.message_id = msg['message-id']
self.context = (BounceContext.normal if context is None else context)
self.processed = False
def _step_do_confirm_verify(self):
# Restore a little extra state that can't be stored in the database
# (because the order of setattr() on restore is indeterminate), then
# continue with the confirmation/verification step.
if self.which is WhichSubscriber.address:
self.subscriber = self.address
else:
assert self.which is WhichSubscriber.user
self.subscriber = self.user
# Reset the token so it can't be used in a replay attack.
self._set_token(TokenOwner.no_one)
# The user has confirmed their subscription request, and also verified
# their email address if necessary. This latter needs to be set on the
# IAddress, but there's nothing more to do about the confirmation step.
# We just continue along with the workflow.
if self.address.verified_on is None:
self.address.verified_on = now()
# The next step depends on the mailing list's subscription policy.
next_step = ('moderation_checks'
if self.mlist.subscription_policy in (
SubscriptionPolicy.moderate,
SubscriptionPolicy.confirm_then_moderate,
)
else 'do_subscription')
self.push(next_step)
def on_post(self, request, response):
# We don't care about the POST data, just do the action.
if self._action == 'verify' and self._address.verified_on is None:
self._address.verified_on = now()
elif self._action == 'unverify':
self._address.verified_on = None
no_content(response)
def verify(self, request):
# We don't care about the POST data, just do the action.
if self._action == 'verify' and self._address.verified_on is None:
self._address.verified_on = now()
elif self._action == 'unverify':
self._address.verified_on = None
return no_content()
def test_confirm_because_confirm_then_moderation(self):
# We have a subscription request which requires the user to confirm
# (because she does not have a verified address) and the moderator to
# approve. Running the workflow gives us a token. Confirming the
# token runs the workflow a little farther, but still gives us a
# token. Confirming again subscribes the user.
self._mlist.subscription_policy = \
SubscriptionPolicy.confirm_then_moderate
self._anne.verified_on = now()
# Runs until subscription confirmation.
token, token_owner, rmember = self._registrar.register(self._anne)
self.assertIsNotNone(token)
self.assertEqual(token_owner, TokenOwner.subscriber)
self.assertIsNone(rmember)
member = self._mlist.regular_members.get_member('*****@*****.**')
self.assertIsNone(member)
# Now confirm the subscription, and wait for the moderator to approve
# the subscription. She is still not subscribed.
new_token, token_owner, rmember = self._registrar.confirm(token)
# The new token, used for the moderator to approve the message, is not
# the same as the old token.
self.assertNotEqual(new_token, token)
self.assertIsNotNone(new_token)
self.assertEqual(token_owner, TokenOwner.moderator)
self.assertIsNone(rmember)
member = self._mlist.regular_members.get_member('*****@*****.**')
self.assertIsNone(member)
# Confirm once more, this time as the moderator approving the
# subscription. Now she's a member.
token, token_owner, rmember = self._registrar.confirm(new_token)
self.assertIsNone(token)
self.assertEqual(token_owner, TokenOwner.no_one)
member = self._mlist.regular_members.get_member('*****@*****.**')
self.assertEqual(rmember, member)
self.assertEqual(member.address, self._anne)
def test_user_is_subscribed_to_unsubscribe(self):
# A user must be subscribed to a list when trying to unsubscribe.
addr = self._user_manager.create_address('*****@*****.**')
addr.verfied_on = now()
workflow = UnSubscriptionWorkflow(self._mlist, addr)
self.assertRaises(AssertionError, workflow.run_thru,
'subscription_checks')
def test_member_subscribed_address_has_no_display_name(self):
address = getUtility(IUserManager).create_address('*****@*****.**')
address.verified_on = now()
self._mlist.subscribe(address)
items = get_queue_messages('virgin', expected_count=1)
message = items[0].msg
self.assertEqual(message['to'], '*****@*****.**')
def setUp(self):
self._mlist = create_list('*****@*****.**')
self._now = now()
# Enable just the dummy archiver.
config.push(
'dummy', """
[archiver.dummy]
class: mailman.runners.tests.test_archiver.DummyArchiver
enable: no
[archiver.prototype]
enable: no
[archiver.mhonarc]
enable: no
[archiver.mail_archive]
enable: no
""")
self._archiveq = config.switchboards['archive']
self._msg = mfs("""\
From: [email protected]
To: [email protected]
Subject: My first post
Message-ID: <first>
Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB
First post!
""")
self._runner = make_testable_runner(ArchiveRunner)
IListArchiverSet(self._mlist).get('dummy').is_enabled = True
def add(self, store, pendable, lifetime=None):
verifyObject(IPendable, pendable)
# Calculate the token and the lifetime.
if lifetime is None:
lifetime = as_timedelta(config.mailman.pending_request_life)
# Calculate a unique token. Algorithm vetted by the Timbot. time()
# has high resolution on Linux, clock() on Windows. random gives us
# about 45 bits in Python 2.2, 53 bits on Python 2.3. The time and
# clock values basically help obscure the random number generator, as
# does the hash calculation. The integral parts of the time values
# are discarded because they're the most predictable bits.
for attempts in range(3):
right_now = time.time()
x = random.random() + right_now % 1.0 + time.clock() % 1.0
# Use sha1 because it produces shorter strings.
token = hashlib.sha1(repr(x).encode('utf-8')).hexdigest()
# In practice, we'll never get a duplicate, but we'll be anal
# about checking anyway.
if store.query(Pended).filter_by(token=token).count() == 0:
break
else:
raise RuntimeError('Could not find a valid pendings token')
# Create the record, and then the individual key/value pairs.
pending = Pended(token=token, expiration_date=now() + lifetime)
for key, value in pendable.items():
# Both keys and values must be strings.
if isinstance(key, bytes):
key = key.decode('utf-8')
if isinstance(value, bytes):
# Make sure we can turn this back into a bytes.
value = dict(__encoding__='utf-8', value=value.decode('utf-8'))
keyval = PendedKeyValue(key=key, value=json.dumps(value))
pending.key_values.append(keyval)
store.add(pending)
return token
def test_delete_nonmembers_on_adding_member(self):
# GL: #237 - When a new address is subscribed, any existing nonmember
# subscriptions for this address; or any addresses also controlled by
# this user, are deleted.
anne_nonmember = add_member(
self._mlist,
RequestRecord('*****@*****.**', 'Anne Person',
DeliveryMode.regular,
system_preferences.preferred_language),
MemberRole.nonmember)
# Add a few other validated addresses to this user, and subscribe them
# as nonmembers.
for email in ('*****@*****.**', '*****@*****.**'):
address = anne_nonmember.user.register(email)
address.verified_on = now()
self._mlist.subscribe(address, MemberRole.nonmember)
# There are now three nonmembers.
self.assertEqual(
{address.email for address in self._mlist.nonmembers.addresses},
{'*****@*****.**',
'*****@*****.**',
'*****@*****.**',
})
# Let's now add one of Anne's addresses as a member. This deletes all
# of Anne's nonmember memberships.
anne_member = add_member(
self._mlist,
RequestRecord('*****@*****.**', 'Anne Person',
DeliveryMode.regular,
system_preferences.preferred_language),
MemberRole.member)
self.assertEqual(self._mlist.nonmembers.member_count, 0)
members = list(self._mlist.members.members)
self.assertEqual(len(members), 1)
self.assertEqual(members[0], anne_member)
def on_post(self, request, response):
# We don't care about the POST data, just do the action.
if self._action == 'verify' and self._address.verified_on is None:
self._address.verified_on = now()
elif self._action == 'unverify':
self._address.verified_on = None
no_content(response)
def test_cached_copy_is_expired(self):
cache_path = os.path.join(config.VAR_DIR, dmarc.LOCAL_FILE_NAME)
with open(cache_path, 'w', encoding='utf-8') as fp:
print('xyz', end='', file=fp)
# Expire the cache file. That way the current cached file will be
# invalid and a new one will be downloaded.
expires = (now() - timedelta(days=1)).timestamp()
os.utime(cache_path, (expires, expires))
new_path = dmarc.ensure_current_suffix_list()
self.assertEqual(cache_path, new_path)
with open(cache_path, 'r', encoding='utf-8') as fp:
contents = fp.read()
self.assertEqual(contents, 'abc')
self.assertEqual(
os.stat(new_path).st_mtime,
(now() + as_timedelta(config.dmarc.cache_lifetime)).timestamp())
def _step_do_confirm_verify(self):
# Restore a little extra state that can't be stored in the database
# (because the order of setattr() on restore is indeterminate), then
# continue with the confirmation/verification step.
if self.which is WhichSubscriber.address:
self.subscriber = self.address
else:
assert self.which is WhichSubscriber.user
self.subscriber = self.user
# Reset the token so it can't be used in a replay attack.
self._set_token(TokenOwner.no_one)
# The user has confirmed their subscription request, and also verified
# their email address if necessary. This latter needs to be set on the
# IAddress, but there's nothing more to do about the confirmation step.
# We just continue along with the workflow.
if self.address.verified_on is None:
self.address.verified_on = now()
# The next step depends on the mailing list's subscription policy.
next_step = ('moderation_checks'
if self.mlist.subscription_policy in (
SubscriptionPolicy.moderate,
SubscriptionPolicy.confirm_then_moderate,
)
else 'do_subscription')
self.push(next_step)
def setUp(self):
self._ant = create_list('*****@*****.**')
self._bee = create_list('*****@*****.**')
user_manager = getUtility(IUserManager)
self._anne = user_manager.create_user('*****@*****.**')
preferred = list(self._anne.addresses)[0]
preferred.verified_on = now()
self._anne.preferred_address = preferred
def __init__(self, email, display_name):
super(Address, self).__init__()
getUtility(IEmailValidator).validate(email)
lower_case = email.lower()
self.email = lower_case
self.display_name = display_name
self._original = (None if lower_case == email else email)
self.registered_on = now()
def __init__(self, list_id, email, msg, context=None):
self.list_id = list_id
self.email = email
self.timestamp = now()
msgid = msg['message-id']
self.message_id = msgid
self.context = (BounceContext.normal if context is None else context)
self.processed = False
def test_unsubscribe(self):
address = self._usermanager.create_address('*****@*****.**')
address.verified_on = now()
self._mlist.subscribe(address)
self.assertEqual(len(list(self._mlist.members.members)), 1)
member = self._mlist.members.get_member('*****@*****.**')
member.unsubscribe()
self.assertEqual(len(list(self._mlist.members.members)), 0)
def setUp(self):
self._ant = create_list('*****@*****.**')
self._bee = create_list('*****@*****.**')
user_manager = getUtility(IUserManager)
self._anne = user_manager.make_user('*****@*****.**', 'Anne Person')
preferred = list(self._anne.addresses)[0]
preferred.verified_on = now()
self._anne.preferred_address = preferred
def __init__(self, email, display_name):
super().__init__()
getUtility(IEmailValidator).validate(email)
lower_case = email.lower()
self.email = lower_case
self.display_name = display_name
self._original = (None if lower_case == email else email)
self.registered_on = now()
def test_unsubscribe(self):
address = self._usermanager.create_address('*****@*****.**')
address.verified_on = now()
self._mlist.subscribe(address)
self.assertEqual(len(list(self._mlist.members.members)), 1)
member = self._mlist.members.get_member('*****@*****.**')
member.unsubscribe()
self.assertEqual(len(list(self._mlist.members.members)), 0)
def on_post(self, request, response):
"""POST to /addresses
Add a new address to the user record.
"""
assert self._user is not None
preferred = None
user_manager = getUtility(IUserManager)
validator = Validator(email=str,
display_name=str,
preferred=as_boolean,
absorb_existing=bool,
_optional=('display_name', 'absorb_existing',
'preferred'))
try:
data = validator(request)
# We cannot set the address to be preferred when it is
# created so remove it from the arguments here and
# set it below.
preferred = data.pop('preferred', False)
except ValueError as error:
bad_request(response, str(error))
return
absorb_existing = data.pop('absorb_existing', False)
try:
address = user_manager.create_address(**data)
except InvalidEmailAddressError:
bad_request(response, b'Invalid email address')
return
except ExistingAddressError:
# If the address is not linked to any user, link it to the current
# user and return it. Since we're linking to an existing address,
# ignore any given display_name attribute.
address = user_manager.get_address(data['email'])
if address.user is None:
address.user = self._user
location = self.api.path_to('addresses/{}'.format(
address.email))
created(response, location)
return
elif not absorb_existing:
bad_request(response, 'Address belongs to other user')
return
else:
# The address exists and is linked but we can merge the users.
address = user_manager.get_address(data['email'])
self._user.absorb(address.user)
else:
# Link the address to the current user and return it.
address.user = self._user
# Set the preferred address here if we were signalled to do so.
if preferred:
address.verified_on = now()
self._user.preferred_address = address
location = self.api.path_to('addresses/{}'.format(address.email))
created(response, location)
def test_lp_1031391(self):
# LP: #1031391 msgdata['received_time'] gets added by the LMTP server.
# The value is a datetime. If this message gets held, it will break
# pending requests since they require string keys and values.
received_time = now()
msgdata = dict(received_time=received_time)
request_id = hold_message(self._mlist, self._msg, msgdata)
key, data = self._request_db.get_request(request_id)
self.assertEqual(data['received_time'], received_time)
def setUp(self):
with transaction():
self._mlist = create_list("*****@*****.**")
self._registrar = IRegistrar(self._mlist)
manager = getUtility(IUserManager)
self._anne = manager.create_address("*****@*****.**", "Anne Person")
self._bart = manager.make_user("*****@*****.**", "Bart Person")
preferred = list(self._bart.addresses)[0]
preferred.verified_on = now()
self._bart.preferred_address = preferred
def test_member_has_address_and_user_display_name(self):
user = getUtility(IUserManager).create_user("*****@*****.**", "Anne Person")
set_preferred(user)
address = getUtility(IUserManager).create_address("*****@*****.**", "Anne X Person")
address.verified_on = now()
user.link(address)
self._mlist.subscribe(address)
items = get_queue_messages("virgin", expected_count=1)
message = items[0].msg
self.assertEqual(message["to"], "Anne X Person <*****@*****.**>")
def __init__(self, list_id, email, msg, context=None):
self.list_id = list_id
self.email = email
self.timestamp = now()
msgid = msg['message-id']
if isinstance(msgid, bytes):
msgid = msgid.decode('ascii')
self.message_id = msgid
self.context = (BounceContext.normal if context is None else context)
self.processed = False
def test_verification_checks_with_verified_address(self):
# When the address is already verified, we skip straight to the
# confirmation checks.
anne = self._user_manager.create_address(self._anne)
anne.verified_on = now()
workflow = SubscriptionWorkflow(self._mlist, anne)
workflow.run_thru('verification_checks')
with patch.object(workflow, '_step_confirmation_checks') as step:
next(workflow)
step.assert_called_once_with()
