def test_file_binary(self):
with TemporaryDirectory() as tempdir:
path = os.path.join(tempdir, 'my-file')
with open(path, 'wb') as fp:
fp.write(b'xxx')
contents = protocols.get('file:///{}'.format(path), mode='rb')
self.assertEqual(contents, b'xxx')
def get(self, store, name, context, **kws):
"""See `ITemplateManager`."""
template = store.query(Template).filter(
Template.name == name, Template.context == context).one_or_none()
if template is None:
return None
actual_uri = expand(template.uri, None, kws)
cache_mgr = getUtility(ICacheManager)
contents = cache_mgr.get(actual_uri)
if contents is None:
# It's likely that the cached contents have expired.
auth = {}
if template.username is not None:
auth['auth'] = (template.username, template.password)
try:
contents = protocols.get(actual_uri, **auth)
except HTTPError as error:
# 404/NotFound errors are interpreted as missing templates,
# for which we'll return the default (i.e. the empty string).
# All other exceptions get passed up the chain.
if error.response.status_code != 404:
raise
log.exception('Cannot retrieve template at {} ({})'.format(
actual_uri, auth.get('auth', '<no authorization>')))
return ''
# We don't need to cache mailman: contents since those are already
# on the file system.
if urlparse(actual_uri).scheme != 'mailman':
cache_mgr.add(actual_uri, contents)
return contents
def test_file(self):
with TemporaryDirectory() as tempdir:
path = os.path.join(tempdir, 'my-file')
with open(path, 'w', encoding='utf-8') as fp:
print('Some contents', end='', file=fp)
contents = protocols.get('file:///{}'.format(path))
self.assertEqual(contents, 'Some contents')
def test_non_ascii(self):
# mailman://demo.txt with non-ascii content.
test_text = b'\xe4\xb8\xad'
path = os.path.join(self.var_dir, 'templates', 'site', 'it')
os.makedirs(path)
with open(os.path.join(path, 'demo.txt'), 'wb') as fp:
fp.write(test_text)
content = protocols.get('mailman:///it/demo.txt')
self.assertIsInstance(content, str)
self.assertEqual(content, test_text.decode('utf-8'))
def ensure_current_suffix_list():
# Read and parse the organizational domain suffix list. First look in the
# cached directory to see if we already have a valid copy of it.
cached_copy_path = os.path.join(config.VAR_DIR, LOCAL_FILE_NAME)
lifetime = as_timedelta(config.dmarc.cache_lifetime)
download = False
try:
mtime = os.stat(cached_copy_path).st_mtime
except FileNotFoundError:
vlog.info('No cached copy of the public suffix list found')
download = True
cache_found = False
else:
cache_found = True
# Is the cached copy out-of-date? Note that when we write a new cache
# version we explicitly set its mtime to the time in the future when
# the cache will expire.
if mtime < now().timestamp():
download = True
vlog.info('Cached copy of public suffix list is out of date')
if download:
try:
content = get(config.dmarc.org_domain_data_url)
except (URLError, HTTPError) as error:
elog.error('Unable to retrieve public suffix list from %s: %s',
config.dmarc.org_domain_data_url,
getattr(error, 'reason', str(error)))
if cache_found:
vlog.info('Using out of date public suffix list')
content = None
else:
# We couldn't access the URL and didn't even have an out of
# date suffix list cached. Use the shipped version.
content = resource_bytes('mailman.rules.data', LOCAL_FILE_NAME)
if content is not None:
# Content is either a string or UTF-8 encoded bytes.
if isinstance(content, bytes):
content = content.decode('utf-8')
# Write the cache atomically.
new_path = cached_copy_path + '.new'
with open(new_path, 'w', encoding='utf-8') as fp:
fp.write(content)
# Set the expiry time to the future.
mtime = (now() + lifetime).timestamp()
os.utime(new_path, (mtime, mtime))
# Flip the new file into the cached location. This does not
# modify the mtime.
os.rename(new_path, cached_copy_path)
return cached_copy_path
def test_mailman_uri_with_english_fallback(self):
content = protocols.get('mailman:///it/demo.txt')
self.assertEqual(content, 'Test content')
def test_mailman_uri_with_language(self):
content = protocols.get('mailman:///en/demo.txt')
self.assertEqual(content, 'Test content')
def test_mailman_internal_uris_twice(self):
# mailman:///demo.txt
content = protocols.get('mailman:///demo.txt')
self.assertEqual(content, 'Test content')
content = protocols.get('mailman:///demo.txt')
self.assertEqual(content, 'Test content')
def test_mailman_full_uri(self):
content = protocols.get('mailman:///[email protected]/en/demo.txt')
self.assertEqual(content, 'Test content')
def test_no_such_language(self):
with self.assertRaises(URLError) as cm:
protocols.get('mailman:///[email protected]/xx/demo.txt')
self.assertEqual(cm.exception.reason, 'No such language')
def test_missing_mailing_list(self):
with self.assertRaises(URLError) as cm:
protocols.get('mailman:///[email protected]/it/demo.txt')
self.assertEqual(cm.exception.reason, 'Missing list')
def test_bad_mailing_list(self):
with self.assertRaises(URLError) as cm:
protocols.get('mailman:///[email protected]/demo.txt')
self.assertEqual(cm.exception.reason, 'Bad language or list name')
def test_bad_language(self):
with self.assertRaises(URLError) as cm:
protocols.get('mailman:///xx/demo.txt')
self.assertEqual(cm.exception.reason, 'Bad language or list name')
def test_short_url_error(self):
with self.assertRaises(URLError) as cm:
protocols.get('mailman://')
self.assertEqual(cm.exception.reason, 'No template specified')
def test_uri_not_found(self):
with self.assertRaises(URLError) as cm:
protocols.get('mailman:///missing.txt')
self.assertEqual(cm.exception.reason, 'No such file')
def test_too_many_path_components(self):
with self.assertRaises(URLError) as cm:
protocols.get('mailman:///[email protected]/en/foo/demo.txt')
self.assertEqual(cm.exception.reason, 'No such file')
def test_mailman_uri_with_list_name(self):
content = protocols.get('mailman:///[email protected]/demo.txt')
self.assertEqual(content, 'Test content')