def add(self, store, pendable, lifetime=None):
verifyObject(IPendable, pendable)
# Calculate the token and the lifetime.
if lifetime is None:
lifetime = as_timedelta(config.mailman.pending_request_life)
# Calculate a unique token. Algorithm vetted by the Timbot. time()
# has high resolution on Linux, clock() on Windows. random gives us
# about 45 bits in Python 2.2, 53 bits on Python 2.3. The time and
# clock values basically help obscure the random number generator, as
# does the hash calculation. The integral parts of the time values
# are discarded because they're the most predictable bits.
for attempts in range(3):
right_now = time.time()
x = random.random() + right_now % 1.0 + time.clock() % 1.0
# Use sha1 because it produces shorter strings.
token = hashlib.sha1(repr(x).encode('utf-8')).hexdigest()
# In practice, we'll never get a duplicate, but we'll be anal
# about checking anyway.
if store.query(Pended).filter_by(token=token).count() == 0:
break
else:
raise RuntimeError('Could not find a valid pendings token')
# Create the record, and then the individual key/value pairs.
pending = Pended(token=token, expiration_date=now() + lifetime)
for key, value in pendable.items():
# Both keys and values must be strings.
if isinstance(key, bytes):
key = key.decode('utf-8')
if isinstance(value, bytes):
# Make sure we can turn this back into a bytes.
value = dict(__encoding__='utf-8', value=value.decode('utf-8'))
keyval = PendedKeyValue(key=key, value=json.dumps(value))
pending.key_values.append(keyval)
store.add(pending)
return token
def add(self, store, pendable, lifetime=None):
verifyObject(IPendable, pendable)
# Calculate the token and the lifetime.
if lifetime is None:
lifetime = as_timedelta(config.mailman.pending_request_life)
# Calculate a unique token. Algorithm vetted by the Timbot. time()
# has high resolution on Linux, clock() on Windows. random gives us
# about 45 bits in Python 2.2, 53 bits on Python 2.3. The time and
# clock values basically help obscure the random number generator, as
# does the hash calculation. The integral parts of the time values
# are discarded because they're the most predictable bits.
for attempts in range(3):
right_now = time.time()
x = random.random() + right_now % 1.0 + time.clock() % 1.0
# Use sha1 because it produces shorter strings.
token = hashlib.sha1(repr(x).encode("utf-8")).hexdigest()
# In practice, we'll never get a duplicate, but we'll be anal
# about checking anyway.
if store.query(Pended).filter_by(token=token).count() == 0:
break
else:
raise RuntimeError("Could not find a valid pendings token")
# Create the record, and then the individual key/value pairs.
pending = Pended(token=token, expiration_date=now() + lifetime)
for key, value in pendable.items():
# Both keys and values must be strings.
if isinstance(key, bytes):
key = key.decode("utf-8")
if isinstance(value, bytes):
# Make sure we can turn this back into a bytes.
value = dict(__encoding__="utf-8", value=value.decode("utf-8"))
keyval = PendedKeyValue(key=key, value=json.dumps(value))
pending.key_values.append(keyval)
store.add(pending)
return token
def throttle_date(self):
"""See `IDirectEmailAuthorization`."""
now = datetime.now(pytz.timezone('UTC'))
after = now - as_timedelta(
config.launchpad.user_to_user_throttle_interval)
throttlers = self._getThrottlers(after)
# We now have the set of emails that would throttle delivery. If the
# configuration variable has changed, this could produce more or less
# than the now-allowed number of throttlers. We should never get here
# if it's less because the contact would have been allowed.
#
# If it's more, then we really want to count back from the sorted end,
# because when /that/ contact record expires, they'll be able to
# resend. Here are two examples.
#
# affecters = A B C
# max allowed = 3
# index = len(affecters) - 3 == 0 == A
# when A's date < the interval, they can try again
#
# affecters = A B C D E F G
# max allowed (now) = 3
# index = len(affecters) - 3 = 4 == E (counting from zero)
# when E's date < than the interval, they can try again
affecters = sorted(throttlers, key=attrgetter('date_sent'))
max_throttlers = config.launchpad.user_to_user_max_messages
expiry = len(affecters) - max_throttlers
if expiry < 0:
# There were fewer affecters than are now allowed, so they can
# retry immediately. Remember that the caller adds the interval
# back, so this would give us 'now'.
return after
return affecters[expiry].date_sent
def add(self, store, pendable, lifetime=None):
verifyObject(IPendable, pendable)
# Calculate the token and the lifetime.
if lifetime is None:
lifetime = as_timedelta(config.mailman.pending_request_life)
for attempts in range(3):
token = token_factory.new()
# In practice, we'll never get a duplicate, but we'll be anal
# about checking anyway.
if store.query(Pended).filter_by(token=token).count() == 0:
break
else:
raise RuntimeError('Could not find a valid pendings token')
# Create the record, and then the individual key/value pairs.
pending = Pended(token=token, expiration_date=now() + lifetime)
pendable_type = pendable.get('type', pendable.PEND_TYPE)
pending.key_values.append(
PendedKeyValue(key='type', value=pendable_type))
for key, value in pendable.items():
# The type has been handled above.
if key == 'type':
continue
# Both keys and values must be strings.
if isinstance(key, bytes):
key = key.decode('utf-8')
if isinstance(value, bytes):
# Make sure we can turn this back into a bytes.
value = dict(__encoding__='utf-8', value=value.decode('utf-8'))
keyval = PendedKeyValue(key=key, value=json.dumps(value))
pending.key_values.append(keyval)
store.add(pending)
return token
def add(self, store, pendable, lifetime=None):
verifyObject(IPendable, pendable)
# Calculate the token and the lifetime.
if lifetime is None:
lifetime = as_timedelta(config.mailman.pending_request_life)
for attempts in range(3):
token = token_factory.new()
# In practice, we'll never get a duplicate, but we'll be anal
# about checking anyway.
if store.query(Pended).filter_by(token=token).count() == 0:
break
else:
raise RuntimeError('Could not find a valid pendings token')
# Create the record, and then the individual key/value pairs.
pending = Pended(
token=token,
expiration_date=now() + lifetime)
pendable_type = pendable.get('type', pendable.PEND_TYPE)
pending.key_values.append(
PendedKeyValue(key='type', value=pendable_type))
for key, value in pendable.items():
# The type has been handled above.
if key == 'type':
continue
# Both keys and values must be strings.
if isinstance(key, bytes):
key = key.decode('utf-8')
if isinstance(value, bytes):
# Make sure we can turn this back into a bytes.
value = dict(__encoding__='utf-8',
value=value.decode('utf-8'))
keyval = PendedKeyValue(key=key, value=json.dumps(value))
pending.key_values.append(keyval)
store.add(pending)
return token
def throttle_date(self):
"""See `IDirectEmailAuthorization`."""
now = datetime.now(pytz.timezone('UTC'))
after = now - as_timedelta(
config.launchpad.user_to_user_throttle_interval)
throttlers = self._getThrottlers(after)
# We now have the set of emails that would throttle delivery. If the
# configuration variable has changed, this could produce more or less
# than the now-allowed number of throttlers. We should never get here
# if it's less because the contact would have been allowed.
#
# If it's more, then we really want to count back from the sorted end,
# because when /that/ contact record expires, they'll be able to
# resend. Here are two examples.
#
# affecters = A B C
# max allowed = 3
# index = len(affecters) - 3 == 0 == A
# when A's date < the interval, they can try again
#
# affecters = A B C D E F G
# max allowed (now) = 3
# index = len(affecters) - 3 = 4 == E (counting from zero)
# when E's date < than the interval, they can try again
affecters = sorted(throttlers, key=attrgetter('date_sent'))
max_throttlers = config.launchpad.user_to_user_max_messages
expiry = len(affecters) - max_throttlers
if expiry < 0:
# There were fewer affecters than are now allowed, so they can
# retry immediately. Remember that the caller adds the interval
# back, so this would give us 'now'.
return after
return affecters[expiry].date_sent
def __init__(self, name, slice=None):
"""Create a runner.
:param slice: The slice number for this runner. This is passed
directly to the underlying `ISwitchboard` object. This is ignored
for runners that don't manage a queue.
:type slice: int or None
"""
# Grab the configuration section.
self.name = name
section = getattr(config, 'runner.' + name)
substitutions = config.paths
substitutions['name'] = name
numslices = int(section.instances)
# Check whether the runner is queue runner or not; non-queue runner
# should not have queue_directory or switchboard instance.
if self.is_queue_runner:
self.queue_directory = expand(section.path, substitutions)
self.switchboard = Switchboard(
name, self.queue_directory, slice, numslices, True)
else:
self.queue_directory = None
self.switchboard= None
self.sleep_time = as_timedelta(section.sleep_time)
# sleep_time is a timedelta; turn it into a float for time.sleep().
self.sleep_float = (86400 * self.sleep_time.days +
self.sleep_time.seconds +
self.sleep_time.microseconds / 1.0e6)
self.max_restarts = int(section.max_restarts)
self.start = as_boolean(section.start)
self._stop = False
self.status = 0
def __init__(self, name, slice=None):
"""Create a runner.
:param slice: The slice number for this runner. This is passed
directly to the underlying `ISwitchboard` object. This is ignored
for runners that don't manage a queue.
:type slice: int or None
"""
# Grab the configuration section.
self.name = name
section = getattr(config, 'runner.' + name)
substitutions = config.paths
substitutions['name'] = name
numslices = int(section.instances)
# Check whether the runner is queue runner or not; non-queue runner
# should not have queue_directory or switchboard instance.
if self.is_queue_runner:
self.queue_directory = expand(section.path, None, substitutions)
self.switchboard = Switchboard(name, self.queue_directory, slice,
numslices, True)
else:
self.queue_directory = None
self.switchboard = None
self.sleep_time = as_timedelta(section.sleep_time)
# sleep_time is a timedelta; turn it into a float for time.sleep().
self.sleep_float = (86400 * self.sleep_time.days +
self.sleep_time.seconds +
self.sleep_time.microseconds / 1.0e6)
self.max_restarts = int(section.max_restarts)
self.start = as_boolean(section.start)
self._stop = False
self.status = 0
def is_allowed(self):
"""See `IDirectEmailAuthorization`."""
# Users are only allowed to send X number of messages in a certain
# period of time. Both the number of messages and the time period
# are configurable.
now = datetime.now(pytz.timezone('UTC'))
after = now - as_timedelta(
config.launchpad.user_to_user_throttle_interval)
return self._isAllowedAfter(after)
def is_allowed(self):
"""See `IDirectEmailAuthorization`."""
# Users are only allowed to send X number of messages in a certain
# period of time. Both the number of messages and the time period
# are configurable.
now = datetime.now(pytz.timezone('UTC'))
after = now - as_timedelta(
config.launchpad.user_to_user_throttle_interval)
return self._isAllowedAfter(after)
def test_cached_copy_is_missing(self):
cache_path = os.path.join(config.VAR_DIR, dmarc.LOCAL_FILE_NAME)
self.assertFalse(os.path.exists(cache_path))
new_path = dmarc.ensure_current_suffix_list()
self.assertEqual(cache_path, new_path)
with open(cache_path, 'r', encoding='utf-8') as fp:
contents = fp.read()
self.assertEqual(contents, 'abc')
self.assertEqual(
os.stat(new_path).st_mtime,
(now() + as_timedelta(config.dmarc.cache_lifetime)).timestamp())
def wait_for_webservice(hostname=None, port=None):
"""Wait for the REST server to start serving requests."""
hostname = config.webservice.hostname if hostname is None else hostname
port = int(config.webservice.port) if port is None else port
until = datetime.datetime.now() + as_timedelta(config.devmode.wait)
while datetime.datetime.now() < until:
try:
socket.socket().connect((hostname, port))
except ConnectionRefusedError:
time.sleep(0.1)
else:
break
else:
raise RuntimeError('Connection refused')
def ensure_current_suffix_list():
# Read and parse the organizational domain suffix list. First look in the
# cached directory to see if we already have a valid copy of it.
cached_copy_path = os.path.join(config.VAR_DIR, LOCAL_FILE_NAME)
lifetime = as_timedelta(config.dmarc.cache_lifetime)
download = False
try:
mtime = os.stat(cached_copy_path).st_mtime
except FileNotFoundError:
vlog.info('No cached copy of the public suffix list found')
download = True
cache_found = False
else:
cache_found = True
# Is the cached copy out-of-date? Note that when we write a new cache
# version we explicitly set its mtime to the time in the future when
# the cache will expire.
if mtime < now().timestamp():
download = True
vlog.info('Cached copy of public suffix list is out of date')
if download:
try:
content = get(config.dmarc.org_domain_data_url)
except (URLError, HTTPError) as error:
elog.error('Unable to retrieve public suffix list from %s: %s',
config.dmarc.org_domain_data_url,
getattr(error, 'reason', str(error)))
if cache_found:
vlog.info('Using out of date public suffix list')
content = None
else:
# We couldn't access the URL and didn't even have an out of
# date suffix list cached. Use the shipped version.
content = resource_bytes('mailman.rules.data', LOCAL_FILE_NAME)
if content is not None:
# Content is either a string or UTF-8 encoded bytes.
if isinstance(content, bytes):
content = content.decode('utf-8')
# Write the cache atomically.
new_path = cached_copy_path + '.new'
with open(new_path, 'w', encoding='utf-8') as fp:
fp.write(content)
# Set the expiry time to the future.
mtime = (now() + lifetime).timestamp()
os.utime(new_path, (mtime, mtime))
# Flip the new file into the cached location. This does not
# modify the mtime.
os.rename(new_path, cached_copy_path)
return cached_copy_path
def wait_for_webservice():
"""Wait for the REST server to start serving requests."""
until = datetime.datetime.now() + as_timedelta(config.devmode.wait)
while datetime.datetime.now() < until:
try:
socket.socket().connect((config.webservice.hostname, int(config.webservice.port)))
except socket.error as error:
if error[0] == errno.ECONNREFUSED:
time.sleep(0.1)
else:
raise
else:
break
else:
raise RuntimeError("Connection refused")
def _wait_for_rest_server():
until = datetime.datetime.now() + as_timedelta(config.devmode.wait)
while datetime.datetime.now() < until:
try:
request = Request("http://localhost:9001/3.0/system")
basic_auth = "{0}:{1}".format(config.webservice.admin_user, config.webservice.admin_pass)
request.add_header("Authorization", "Basic " + b64encode(basic_auth))
fp = urlopen(request)
except URLError:
pass
else:
fp.close()
break
else:
raise RuntimeError("REST server did not start up")
def test_cached_copy_is_missing_download_404s(self):
# There's no cached file and we'll get a 404 with the .err file so
# we'll have to fall back to our internal copy.
cache_path = os.path.join(config.VAR_DIR, dmarc.LOCAL_FILE_NAME)
self.assertFalse(os.path.exists(cache_path))
new_path = dmarc.ensure_current_suffix_list()
self.assertEqual(cache_path, new_path)
with open(cache_path, 'r', encoding='utf-8') as fp:
contents = fp.read()
# The contents is *not* equal to our dummy test data, but don't tie it
# too closely to the in-tree file contents since that might change
# when and if we update that.
self.assertNotEqual(contents, 'abc')
self.assertEqual(
os.stat(new_path).st_mtime,
(now() + as_timedelta(config.dmarc.cache_lifetime)).timestamp())
def test_cached_copy_is_expired(self):
cache_path = os.path.join(config.VAR_DIR, dmarc.LOCAL_FILE_NAME)
with open(cache_path, 'w', encoding='utf-8') as fp:
print('xyz', end='', file=fp)
# Expire the cache file. That way the current cached file will be
# invalid and a new one will be downloaded.
expires = (now() - timedelta(days=1)).timestamp()
os.utime(cache_path, (expires, expires))
new_path = dmarc.ensure_current_suffix_list()
self.assertEqual(cache_path, new_path)
with open(cache_path, 'r', encoding='utf-8') as fp:
contents = fp.read()
self.assertEqual(contents, 'abc')
self.assertEqual(
os.stat(new_path).st_mtime,
(now() + as_timedelta(config.dmarc.cache_lifetime)).timestamp())
def wait_for_webservice():
"""Wait for the REST server to start serving requests."""
until = datetime.datetime.now() + as_timedelta(config.devmode.wait)
while datetime.datetime.now() < until:
try:
socket.socket().connect((config.webservice.hostname,
int(config.webservice.port)))
except IOError as error:
if error.errno == errno.ECONNREFUSED:
time.sleep(0.1)
else:
raise
else:
break
else:
raise RuntimeError('Connection refused')
def add(self, store, pendable, lifetime=None):
verifyObject(IPendable, pendable)
# Calculate the token and the lifetime.
if lifetime is None:
lifetime = as_timedelta(config.mailman.pending_request_life)
# Calculate a unique token. Algorithm vetted by the Timbot. time()
# has high resolution on Linux, clock() on Windows. random gives us
# about 45 bits in Python 2.2, 53 bits on Python 2.3. The time and
# clock values basically help obscure the random number generator, as
# does the hash calculation. The integral parts of the time values
# are discarded because they're the most predictable bits.
for attempts in range(3):
right_now = time.time()
x = random.random() + right_now % 1.0 + time.clock() % 1.0
# Use sha1 because it produces shorter strings.
token = hashlib.sha1(repr(x)).hexdigest()
# In practice, we'll never get a duplicate, but we'll be anal
# about checking anyway.
if store.query(Pended).filter_by(token=token).count() == 0:
break
else:
raise AssertionError('Could not find a valid pendings token')
# Create the record, and then the individual key/value pairs.
pending = Pended(
token=token,
expiration_date=now() + lifetime)
for key, value in pendable.items():
if isinstance(key, bytes):
key = key.decode('utf-8')
if isinstance(value, bytes):
value = value.decode('utf-8')
elif type(value) is int:
value = '__builtin__.int\1%s' % value
elif type(value) is float:
value = '__builtin__.float\1%s' % value
elif type(value) is bool:
value = '__builtin__.bool\1%s' % value
elif type(value) is list:
# We expect this to be a list of strings.
value = ('mailman.model.pending.unpack_list\1' +
'\2'.join(value))
keyval = PendedKeyValue(key=key, value=value)
pending.key_values.append(keyval)
store.add(pending)
return token
def add(self, store, key, contents, lifetime=None):
"""See `ICacheManager`."""
if lifetime is None:
lifetime = as_timedelta(config.mailman.cache_life)
is_bytes = isinstance(contents, bytes)
file_id = self._key_to_file_id(key)
# Is there already an unexpired entry under this id in the database?
# If the entry doesn't exist, create it. If it overwrite both the
# contents and lifetime.
entry = store.query(CacheEntry).filter(
CacheEntry.key == key).one_or_none()
if entry is None:
entry = CacheEntry(key, file_id, is_bytes, lifetime)
store.add(entry)
else:
entry.update(is_bytes, lifetime)
self._write_contents(file_id, contents, is_bytes)
return file_id
def test_one_temporary_failure(self):
# The first time there are temporary failures, the message just gets
# put in the retry queue, but with some metadata to prevent infinite
# retries.
temporary_failures.append('*****@*****.**')
self._outq.enqueue(self._msg, {}, listid='test.example.com')
self._runner.run()
events = list(self._processor.unprocessed)
self.assertEqual(len(events), 0)
items = get_queue_messages('retry', expected_count=1)
self.assertEqual(self._msg.as_string(), items[0].msg.as_string())
# The metadata has three keys which are used two decide whether the
# next temporary failure should be retried.
self.assertEqual(items[0].msgdata['last_recip_count'], 1)
deliver_until = (datetime(2005, 8, 1, 7, 49, 23) +
as_timedelta(config.mta.delivery_retry_period))
self.assertEqual(items[0].msgdata['deliver_until'], deliver_until)
self.assertEqual(items[0].msgdata['recipients'], ['*****@*****.**'])
def test_no_progress_on_retries_within_retry_period(self):
# Temporary failures cause queuing for a retry later on, unless no
# progress is being made on the retries and we've tried for the
# specified delivery retry period. This test ensures that even if no
# progress is made, if the retry period hasn't expired, the message
# will be requeued.
temporary_failures.append('*****@*****.**')
temporary_failures.append('*****@*****.**')
deliver_until = (datetime(2005, 8, 1, 7, 49, 23) +
as_timedelta(config.mta.delivery_retry_period))
msgdata = dict(last_recip_count=2, deliver_until=deliver_until)
self._outq.enqueue(self._msg, msgdata, listid='test.example.com')
self._runner.run()
# The retry queue should have our message waiting to be retried.
items = get_queue_messages('retry', expected_count=1)
self.assertEqual(items[0].msgdata['deliver_until'], deliver_until)
self.assertEqual(items[0].msgdata['recipients'],
['*****@*****.**', '*****@*****.**'])
def get_lmtp_client(quiet=False):
"""Return a connected LMTP client."""
# It's possible the process has started but is not yet accepting
# connections. Wait a little while.
lmtp = LMTP()
# lmtp.debuglevel = 1
until = datetime.datetime.now() + as_timedelta(config.devmode.wait)
while datetime.datetime.now() < until:
try:
response = lmtp.connect(config.mta.lmtp_host,
int(config.mta.lmtp_port))
if not quiet:
print(response)
return lmtp
except ConnectionRefusedError:
time.sleep(0.1)
else:
raise RuntimeError('Connection refused')
def test_one_temporary_failure(self):
# The first time there are temporary failures, the message just gets
# put in the retry queue, but with some metadata to prevent infinite
# retries.
temporary_failures.append('*****@*****.**')
self._outq.enqueue(self._msg, {}, listid='test.example.com')
self._runner.run()
events = list(self._processor.unprocessed)
self.assertEqual(len(events), 0)
items = get_queue_messages('retry', expected_count=1)
self.assertEqual(self._msg.as_string(), items[0].msg.as_string())
# The metadata has three keys which are used two decide whether the
# next temporary failure should be retried.
self.assertEqual(items[0].msgdata['last_recip_count'], 1)
deliver_until = (datetime(2005, 8, 1, 7, 49, 23) +
as_timedelta(config.mta.delivery_retry_period))
self.assertEqual(items[0].msgdata['deliver_until'], deliver_until)
self.assertEqual(items[0].msgdata['recipients'], ['*****@*****.**'])
def test_no_progress_on_retries_within_retry_period(self):
# Temporary failures cause queuing for a retry later on, unless no
# progress is being made on the retries and we've tried for the
# specified delivery retry period. This test ensures that even if no
# progress is made, if the retry period hasn't expired, the message
# will be requeued.
temporary_failures.append('*****@*****.**')
temporary_failures.append('*****@*****.**')
deliver_until = (datetime(2005, 8, 1, 7, 49, 23) +
as_timedelta(config.mta.delivery_retry_period))
msgdata = dict(last_recip_count=2,
deliver_until=deliver_until)
self._outq.enqueue(self._msg, msgdata, listid='test.example.com')
self._runner.run()
# The retry queue should have our message waiting to be retried.
items = get_queue_messages('retry', expected_count=1)
self.assertEqual(items[0].msgdata['deliver_until'], deliver_until)
self.assertEqual(items[0].msgdata['recipients'],
['*****@*****.**', '*****@*****.**'])
def get_lmtp_client(quiet=False):
"""Return a connected LMTP client."""
# It's possible the process has started but is not yet accepting
# connections. Wait a little while.
lmtp = LMTP()
# lmtp.debuglevel = 1
until = datetime.datetime.now() + as_timedelta(config.devmode.wait)
while datetime.datetime.now() < until:
try:
response = lmtp.connect(config.mta.lmtp_host, int(config.mta.lmtp_port))
if not quiet:
print(response)
return lmtp
except socket.error as error:
if error[0] == errno.ECONNREFUSED:
time.sleep(0.1)
else:
raise
else:
raise RuntimeError("Connection refused")
def _should_clobber(msg, msgdata, archiver):
"""Should the Date header in the original message get clobbered?"""
# Calculate the Date header of the message as a datetime. What if there
# are multiple Date headers, even in violation of the RFC? For now, take
# the first one. If there are no Date headers, then definitely clobber.
original_date = msg.get('date')
if original_date is None:
return True
section = getattr(config.archiver, archiver, None)
if section is None:
log.error('No archiver config section found: {0}'.format(archiver))
return False
try:
clobber = ClobberDate[section.clobber_date]
except ValueError:
log.error('Invalid clobber_date for "{0}": {1}'.format(
archiver, section.clobber_date))
return False
if clobber is ClobberDate.always:
return True
elif clobber is ClobberDate.never:
return False
# Maybe we'll clobber the date. Let's see if it's farther off from now
# than the skew period.
skew = as_timedelta(section.clobber_skew)
try:
time_tuple = parsedate_tz(original_date)
except (ValueError, OverflowError):
# The likely cause of this is that the year in the Date: field is
# horribly incorrect, e.g. (from SF bug # 571634):
#
# Date: Tue, 18 Jun 0102 05:12:09 +0500
#
# Obviously clobber such dates.
return True
if time_tuple is None:
# There was some other bogosity in the Date header.
return True
claimed_date = datetime.fromtimestamp(mktime_tz(time_tuple))
return (abs(now() - claimed_date) > skew)
def _should_clobber(msg, msgdata, archiver):
"""Should the Date header in the original message get clobbered?"""
# Calculate the Date header of the message as a datetime. What if there
# are multiple Date headers, even in violation of the RFC? For now, take
# the first one. If there are no Date headers, then definitely clobber.
original_date = msg.get('date')
if original_date is None:
return True
section = getattr(config.archiver, archiver, None)
if section is None:
log.error('No archiver config section found: {}'.format(archiver))
return False
try:
clobber = ClobberDate[section.clobber_date]
except ValueError:
log.error('Invalid clobber_date for "{}": {}'.format(
archiver, section.clobber_date))
return False
if clobber is ClobberDate.always:
return True
elif clobber is ClobberDate.never:
return False
# Maybe we'll clobber the date. Let's see if it's farther off from now
# than the skew period.
skew = as_timedelta(section.clobber_skew)
try:
time_tuple = parsedate_tz(original_date)
except (ValueError, OverflowError):
# The likely cause of this is that the year in the Date: field is
# horribly incorrect, e.g. (from SF bug # 571634):
#
# Date: Tue, 18 Jun 0102 05:12:09 +0500
#
# Obviously clobber such dates.
return True
if time_tuple is None:
# There was some other bogosity in the Date header.
return True
claimed_date = datetime.fromtimestamp(mktime_tz(time_tuple))
return (abs(now() - claimed_date) > skew)
def __init__(self, name, slice=None):
"""Create a runner.
:param slice: The slice number for this runner. This is passed
directly to the underlying `ISwitchboard` object. This is ignored
for runners that don't manage a queue.
:type slice: int or None
"""
# Grab the configuration section.
self.name = name
section = getattr(config, "runner." + name)
substitutions = config.paths
substitutions["name"] = name
self.queue_directory = expand(section.path, substitutions)
numslices = int(section.instances)
self.switchboard = Switchboard(name, self.queue_directory, slice, numslices, True)
self.sleep_time = as_timedelta(section.sleep_time)
# sleep_time is a timedelta; turn it into a float for time.sleep().
self.sleep_float = 86400 * self.sleep_time.days + self.sleep_time.seconds + self.sleep_time.microseconds / 1.0e6
self.max_restarts = int(section.max_restarts)
self.start = as_boolean(section.start)
self._stop = False
def test_no_progress_on_retries_with_expired_retry_period(self):
# We've had temporary failures with no progress, and the retry period
# has expired. In that case, a log entry is written and message is
# discarded. There's nothing more that can be done.
temporary_failures.append('*****@*****.**')
temporary_failures.append('*****@*****.**')
retry_period = as_timedelta(config.mta.delivery_retry_period)
deliver_until = datetime(2005, 8, 1, 7, 49, 23) + retry_period
msgdata = dict(last_recip_count=2, deliver_until=deliver_until)
self._outq.enqueue(self._msg, msgdata, listid='test.example.com')
# Before the runner runs, several days pass.
factory.fast_forward(retry_period.days + 1)
mark = LogFileMark('mailman.smtp')
self._runner.run()
# There should be no message in the retry or outgoing queues.
get_queue_messages('retry', expected_count=0)
get_queue_messages('out', expected_count=0)
# There should be a log message in the smtp log indicating that the
# message has been discarded.
line = mark.readline()
self.assertEqual(
line[-63:-1],
'Discarding message with persistent temporary failures: <first>')
def test_no_progress_on_retries_with_expired_retry_period(self):
# We've had temporary failures with no progress, and the retry period
# has expired. In that case, a log entry is written and message is
# discarded. There's nothing more that can be done.
temporary_failures.append('*****@*****.**')
temporary_failures.append('*****@*****.**')
retry_period = as_timedelta(config.mta.delivery_retry_period)
deliver_until = datetime(2005, 8, 1, 7, 49, 23) + retry_period
msgdata = dict(last_recip_count=2,
deliver_until=deliver_until)
self._outq.enqueue(self._msg, msgdata, listid='test.example.com')
# Before the runner runs, several days pass.
factory.fast_forward(retry_period.days + 1)
mark = LogFileMark('mailman.smtp')
self._runner.run()
# There should be no message in the retry or outgoing queues.
get_queue_messages('retry', expected_count=0)
get_queue_messages('out', expected_count=0)
# There should be a log message in the smtp log indicating that the
# message has been discarded.
line = mark.readline()
self.assertEqual(
line[-63:-1],
'Discarding message with persistent temporary failures: <first>')
def _dispose(self, mlist, msg, msgdata):
# See if we should retry delivery of this message again.
deliver_after = msgdata.get('deliver_after', datetime.fromtimestamp(0))
if now() < deliver_after:
return True
# Calculate whether we should VERP this message or not. The results of
# this set the 'verp' key in the message metadata.
interval = int(config.mta.verp_delivery_interval)
if 'verp' in msgdata:
# Honor existing settings.
pass
# If personalization is enabled for this list and we've configured
# Mailman to always VERP personalized deliveries, then yes we VERP it.
# Also, if personalization is /not/ enabled, but
# verp_delivery_interval is set (and we've hit this interval), then
# again, this message should be VERP'd. Otherwise, no.
elif mlist.personalize != Personalization.none:
if as_boolean(config.mta.verp_personalized_deliveries):
msgdata['verp'] = True
elif interval == 0:
# Never VERP.
msgdata['verp'] = False
elif interval == 1:
# VERP every time.
msgdata['verp'] = True
else:
# VERP every 'interval' number of times.
msgdata['verp'] = (mlist.post_id % interval == 0)
try:
debug_log.debug('[outgoing] {}: {}'.format(
self._func, msg.get('message-id', 'n/a')))
self._func(mlist, msg, msgdata)
self._logged = False
except socket.error:
# There was a problem connecting to the SMTP server. Log this
# once, but crank up our sleep time so we don't fill the error
# log.
port = int(config.mta.smtp_port)
if port == 0:
port = 'smtp' # Log this just once.
if not self._logged:
log.error('Cannot connect to SMTP server %s on port %s',
config.mta.smtp_host, port)
self._logged = True
return True
except SomeRecipientsFailed as error:
processor = getUtility(IBounceProcessor)
# BAW: msg is the original message that failed delivery, not a
# bounce message. This may be confusing if this is what's sent to
# the user in the probe message. Maybe we should craft a
# bounce-like message containing information about the permanent
# SMTP failure?
if 'probe_token' in msgdata:
# This is a failure of our local MTA to deliver to a probe
# message recipient. Register the bounce event for permanent
# failures. Start by grabbing and confirming (i.e. removing)
# the pendable record associated with this bounce token,
# regardless of what address was actually failing.
if len(error.permanent_failures) > 0:
pended = getUtility(IPendings).confirm(
msgdata['probe_token'])
# It's possible the token has been confirmed out of the
# database. Just ignore that.
if pended is not None:
# The UUID had to be pended as a unicode.
member = getUtility(ISubscriptionService).get_member(
UUID(hex=pended['member_id']))
processor.register(mlist, member.address.email, msg,
BounceContext.probe)
else:
# Delivery failed at SMTP time for some or all of the
# recipients. Permanent failures are registered as bounces,
# but temporary failures are retried for later.
for email in error.permanent_failures:
processor.register(mlist, email, msg, BounceContext.normal)
# Move temporary failures to the qfiles/retry queue which will
# occasionally move them back here for another shot at
# delivery.
if error.temporary_failures:
current_time = now()
recipients = error.temporary_failures
last_recip_count = msgdata.get('last_recip_count', 0)
deliver_until = msgdata.get('deliver_until', current_time)
if len(recipients) == last_recip_count:
# We didn't make any progress. If we've exceeded the
# configured retry period, log this failure and
# discard the message.
if current_time > deliver_until:
smtp_log.error('Discarding message with '
'persistent temporary failures: '
'{}'.format(msg['message-id']))
return False
else:
# We made some progress, so keep trying to delivery
# this message for a while longer.
deliver_until = current_time + as_timedelta(
config.mta.delivery_retry_period)
msgdata['last_recip_count'] = len(recipients)
msgdata['deliver_until'] = deliver_until
msgdata['recipients'] = recipients
self._retryq.enqueue(msg, msgdata)
# We've successfully completed handling of this message.
return False
def is_reject_or_quarantine(mlist, email, dmarc_domain, org=False):
# This takes a mailing list, an email address as in the From: header, the
# _dmarc host name for the domain in question, and a flag stating whether
# we should check the organizational domains. It returns one of three
# values:
# * True if the DMARC policy is reject or quarantine;
# * False if is not;
# * A special sentinel if we should continue looking
resolver = dns.resolver.Resolver()
resolver.timeout = as_timedelta(
config.dmarc.resolver_timeout).total_seconds()
resolver.lifetime = as_timedelta(
config.dmarc.resolver_lifetime).total_seconds()
try:
txt_recs = resolver.query(dmarc_domain, dns.rdatatype.TXT)
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
return KEEP_LOOKING
except (dns.resolver.NoNameservers):
elog.error('DNSException: No Nameservers available for %s (%s).',
email, dmarc_domain)
# Typically this means a dnssec validation error. Clients that don't
# perform validation *may* successfully see a _dmarc RR whereas a
# validating mailman server won't see the _dmarc RR. We should
# mitigate this email to be safe.
return True
except DNSException as error:
elog.error(
'DNSException: Unable to query DMARC policy for %s (%s). %s',
email, dmarc_domain, error.__doc__)
# While we can't be sure what caused the error, there is potentially
# a DMARC policy record that we missed and that a receiver of the mail
# might see. Thus, we should err on the side of caution and mitigate.
return True
# Be as robust as possible in parsing the result.
results_by_name = {}
cnames = {}
want_names = set([dmarc_domain + '.'])
# Check all the TXT records returned by DNS. Keep track of the CNAMEs for
# checking later on. Ignore any other non-TXT records.
for txt_rec in txt_recs.response.answer:
if txt_rec.rdtype == dns.rdatatype.CNAME:
cnames[txt_rec.name.to_text()] = (
txt_rec.items[0].target.to_text())
if txt_rec.rdtype != dns.rdatatype.TXT:
continue
result = EMPTYSTRING.join(
str(record, encoding='utf-8')
for record in txt_rec.items[0].strings)
name = txt_rec.name.to_text()
results_by_name.setdefault(name, []).append(result)
expands = list(want_names)
seen = set(expands)
while expands:
item = expands.pop(0)
if item in cnames:
if cnames[item] in seen:
# CNAME loop.
continue
expands.append(cnames[item])
seen.add(cnames[item])
want_names.add(cnames[item])
want_names.discard(item)
assert len(want_names) == 1, (
'Error in CNAME processing for {}; want_names != 1.'.format(
dmarc_domain))
for name in want_names:
if name not in results_by_name:
continue
dmarcs = [
record for record in results_by_name[name]
if record.startswith('v=DMARC1;')
]
if len(dmarcs) == 0:
return KEEP_LOOKING
if len(dmarcs) > 1:
elog.error(
'RRset of TXT records for %s has %d v=DMARC1 entries; '
'testing them all', dmarc_domain, len(dmarcs))
for entry in dmarcs:
mo = re.search(r'\bsp=(\w*)\b', entry, re.IGNORECASE)
if org and mo:
policy = mo.group(1).lower()
else:
mo = re.search(r'\bp=(\w*)\b', entry, re.IGNORECASE)
if mo:
policy = mo.group(1).lower()
else:
# This continue does actually get covered by
# TestDMARCRules.test_domain_with_subdomain_policy() and
# TestDMARCRules.test_no_policy() but because of
# Coverage BitBucket issue #198 and
# http://bugs.python.org/issue2506 coverage cannot report
# it as such, so just pragma it away.
continue # pragma: missed
if policy in ('reject', 'quarantine'):
vlog.info('%s: DMARC lookup for %s (%s) found p=%s in %s = %s',
mlist.list_name, email, dmarc_domain, policy, name,
entry)
return True
return False
def _dispose(self, mlist, msg, msgdata):
# See if we should retry delivery of this message again.
deliver_after = msgdata.get('deliver_after', datetime.fromtimestamp(0))
if now() < deliver_after:
return True
# Calculate whether we should VERP this message or not. The results of
# this set the 'verp' key in the message metadata.
interval = int(config.mta.verp_delivery_interval)
if 'verp' in msgdata:
# Honor existing settings.
pass
# If personalization is enabled for this list and we've configured
# Mailman to always VERP personalized deliveries, then yes we VERP it.
# Also, if personalization is /not/ enabled, but
# verp_delivery_interval is set (and we've hit this interval), then
# again, this message should be VERP'd. Otherwise, no.
elif mlist.personalize != Personalization.none:
if as_boolean(config.mta.verp_personalized_deliveries):
msgdata['verp'] = True
elif interval == 0:
# Never VERP.
msgdata['verp'] = False
elif interval == 1:
# VERP every time.
msgdata['verp'] = True
else:
# VERP every 'interval' number of times.
msgdata['verp'] = (mlist.post_id % interval == 0)
try:
debug_log.debug('[outgoing] {0}: {1}'.format(
self._func, msg.get('message-id', 'n/a')))
self._func(mlist, msg, msgdata)
self._logged = False
except socket.error:
# There was a problem connecting to the SMTP server. Log this
# once, but crank up our sleep time so we don't fill the error
# log.
port = int(config.mta.smtp_port)
if port == 0:
port = 'smtp' # Log this just once.
if not self._logged:
log.error('Cannot connect to SMTP server %s on port %s',
config.mta.smtp_host, port)
self._logged = True
return True
except SomeRecipientsFailed as error:
processor = getUtility(IBounceProcessor)
# BAW: msg is the original message that failed delivery, not a
# bounce message. This may be confusing if this is what's sent to
# the user in the probe message. Maybe we should craft a
# bounce-like message containing information about the permanent
# SMTP failure?
if 'probe_token' in msgdata:
# This is a failure of our local MTA to deliver to a probe
# message recipient. Register the bounce event for permanent
# failures. Start by grabbing and confirming (i.e. removing)
# the pendable record associated with this bounce token,
# regardless of what address was actually failing.
if len(error.permanent_failures) > 0:
pended = getUtility(IPendings).confirm(
msgdata['probe_token'])
# It's possible the token has been confirmed out of the
# database. Just ignore that.
if pended is not None:
# The UUID had to be pended as a unicode.
member = getUtility(ISubscriptionService).get_member(
UUID(hex=pended['member_id']))
processor.register(
mlist, member.address.email, msg,
BounceContext.probe)
else:
# Delivery failed at SMTP time for some or all of the
# recipients. Permanent failures are registered as bounces,
# but temporary failures are retried for later.
for email in error.permanent_failures:
processor.register(mlist, email, msg, BounceContext.normal)
# Move temporary failures to the qfiles/retry queue which will
# occasionally move them back here for another shot at
# delivery.
if error.temporary_failures:
current_time = now()
recipients = error.temporary_failures
last_recip_count = msgdata.get('last_recip_count', 0)
deliver_until = msgdata.get('deliver_until', current_time)
if len(recipients) == last_recip_count:
# We didn't make any progress. If we've exceeded the
# configured retry period, log this failure and
# discard the message.
if current_time > deliver_until:
smtp_log.error('Discarding message with '
'persistent temporary failures: '
'{0}'.format(msg['message-id']))
return False
else:
# We made some progress, so keep trying to delivery
# this message for a while longer.
deliver_until = current_time + as_timedelta(
config.mta.delivery_retry_period)
msgdata['last_recip_count'] = len(recipients)
msgdata['deliver_until'] = deliver_until
msgdata['recipients'] = recipients
self._retryq.enqueue(msg, msgdata)
# We've successfully completed handling of this message.
return False