def send_email(subject, body=None, html=None, recipients=None, throttle=None):
"""Send an email. Optionally throttle the amount an identical email goes out.
If the throttle argument is set, an md5 checksum derived from the subject, body, html, and recipients is stored in
Redis with a lock timeout. On the first email sent, the email goes out like normal. But when other emails with the
same subject, body, html, and recipients is supposed to go out, and the lock hasn't expired yet, the email will be
dropped and never sent.
Positional arguments:
subject -- the subject line of the email.
Keyword arguments.
body -- the body of the email (no HTML).
html -- the body of the email, can be HTML (overrides body).
recipients -- list or set (not string) of email addresses to send the email to. Defaults to the ADMINS list in the
Flask config.
throttle -- time in seconds or datetime.timedelta object between sending identical emails.
"""
recipients = recipients or current_app.config['ADMINS']
if throttle is not None:
md5 = hashlib.md5('{}{}{}{}'.format(subject, body, html,
recipients)).hexdigest()
seconds = throttle.total_seconds() if hasattr(
throttle, 'total_seconds') else throttle
lock = redis.lock(EMAIL_THROTTLE.format(md5=md5), timeout=int(seconds))
have_lock = lock.acquire(blocking=False)
if not have_lock:
LOG.debug('Suppressing email: {}'.format(subject))
return
msg = Message(subject=subject, recipients=recipients, body=body, html=html)
mail.send(msg)
def send_email(subject, body=None, html=None, recipients=None, throttle=None):
"""Send an email. Optionally throttle the amount an identical email goes out.
If the throttle argument is set, an md5 checksum derived from the subject, body, html, and recipients is stored in
Redis with a lock timeout. On the first email sent, the email goes out like normal. But when other emails with the
same subject, body, html, and recipients is supposed to go out, and the lock hasn't expired yet, the email will be
dropped and never sent.
Positional arguments:
subject -- the subject line of the email.
Keyword arguments.
body -- the body of the email (no HTML).
html -- the body of the email, can be HTML (overrides body).
recipients -- list or set (not string) of email addresses to send the email to. Defaults to the ADMINS list in the
Flask config.
throttle -- time in seconds or datetime.timedelta object between sending identical emails.
"""
recipients = recipients or current_app.config['ADMINS']
if throttle is not None:
md5 = hashlib.md5('{}{}{}{}'.format(subject, body, html, recipients)).hexdigest()
seconds = throttle.total_seconds() if hasattr(throttle, 'total_seconds') else throttle
lock = redis.lock(EMAIL_THROTTLE.format(md5=md5), timeout=int(seconds))
have_lock = lock.acquire(blocking=False)
if not have_lock:
LOG.debug('Suppressing email: {}'.format(subject))
return
msg = Message(subject=subject, recipients=recipients, body=body, html=html)
mail.send(msg)
def send_exception(subject):
"""Send Python exception tracebacks via email to the ADMINS list.
Use the same HTML styling as Flask tracebacks in debug web servers.
This function must be called while the exception is happening. It picks up the raised exception with sys.exc_info().
Positional arguments:
subject -- subject line of the email (to be prepended by 'Application Error: ').
"""
# Generate and modify html.
tb = tbtools.get_current_traceback() # Get exception information.
with _override_html():
html = tb.render_full().encode('utf-8', 'replace')
html = html.replace('<blockquote>',
'<blockquote style="margin: 1em 0 0; padding: 0;">')
subject = 'Application Error: {}'.format(subject)
# Apply throttle.
md5 = hashlib.md5('{}{}'.format(subject, html)).hexdigest()
seconds = int(current_app.config['MAIL_EXCEPTION_THROTTLE'])
lock = redis.lock(EMAIL_THROTTLE.format(md5=md5), timeout=seconds)
have_lock = lock.acquire(blocking=False)
if not have_lock:
LOG.debug('Suppressing email: {}'.format(subject))
return
# Send email.
msg = Message(subject=subject,
recipients=current_app.config['ADMINS'],
html=html)
mail.send(msg)
def send_exception(subject):
"""Send Python exception tracebacks via email to the ADMINS list.
Use the same HTML styling as Flask tracebacks in debug web servers.
This function must be called while the exception is happening. It picks up the raised exception with sys.exc_info().
Positional arguments:
subject -- subject line of the email (to be prepended by 'Application Error: ').
"""
# Generate and modify html.
tb = tbtools.get_current_traceback() # Get exception information.
with _override_html():
html = tb.render_full().encode('utf-8', 'replace')
html = html.replace('<blockquote>', '<blockquote style="margin: 1em 0 0; padding: 0;">')
subject = 'Application Error: {}'.format(subject)
# Apply throttle.
md5 = hashlib.md5('{}{}'.format(subject, html)).hexdigest()
seconds = int(current_app.config['MAIL_EXCEPTION_THROTTLE'])
lock = redis.lock(EMAIL_THROTTLE.format(md5=md5), timeout=seconds)
have_lock = lock.acquire(blocking=False)
if not have_lock:
LOG.debug('Suppressing email: {}'.format(subject))
return
# Send email.
msg = Message(subject=subject, recipients=current_app.config['ADMINS'], html=html)
mail.send(msg)
def update_package_list():
"""Get a list of all packages from PyPI through their XMLRPC API.
This task returns something in case the user schedules it from a view. The view can wait up to a certain amount of
time for this task to finish, and if nothing times out, it can tell the user if it found any new packages.
Since views can schedule this task, we don't want some rude person hammering PyPI or our application with repeated
requests. This task is limited to one run per 1 hour at most.
Returns:
List of new packages found. Returns None if task is rate-limited.
"""
# Rate limit.
lock = redis.lock(POLL_SIMPLE_THROTTLE, timeout=int(THROTTLE))
have_lock = lock.acquire(blocking=False)
if not have_lock:
LOG.warning(
'poll_simple() task has already executed in the past 4 hours. Rate limiting.'
)
return None
# Query API.
client = xmlrpclib.ServerProxy('https://pypi.python.org/pypi')
results = client.search(dict(summary=''))
if not results:
LOG.error('Reply from API had no results.')
return list()
LOG.debug('Sorting results.')
results.sort(key=lambda x: (x['name'], LooseVersion(x['version'])))
filtered = (r for r in results if r['version'][0].isdigit())
packages = {
r['name']: dict(summary=r['summary'], version=r['version'], id=0)
for r in filtered
}
LOG.debug('Pruning unchanged packages.')
for row in db.session.query(Package.id, Package.name, Package.summary,
Package.latest_version):
if packages.get(row[1]) == dict(summary=row[2], version=row[3], id=0):
packages.pop(row[1])
elif row[1] in packages:
packages[row[1]]['id'] = row[0]
new_package_names = {n for n, d in packages.items() if not d['id']}
# Merge into database.
LOG.debug('Found {} new packages in PyPI, updating {} total.'.format(
len(new_package_names), len(packages)))
with db.session.begin_nested():
for name, data in packages.items():
db.session.merge(
Package(id=data['id'],
name=name,
summary=data['summary'],
latest_version=data['version']))
db.session.commit()
return list(new_package_names)
def update_package_list():
"""Get a list of all packages from PyPI through their XMLRPC API.
This task returns something in case the user schedules it from a view. The view can wait up to a certain amount of
time for this task to finish, and if nothing times out, it can tell the user if it found any new packages.
Since views can schedule this task, we don't want some rude person hammering PyPI or our application with repeated
requests. This task is limited to one run per 1 hour at most.
Returns:
List of new packages found. Returns None if task is rate-limited.
"""
# Rate limit.
lock = redis.lock(POLL_SIMPLE_THROTTLE, timeout=int(THROTTLE))
have_lock = lock.acquire(blocking=False)
if not have_lock:
LOG.warning('poll_simple() task has already executed in the past 4 hours. Rate limiting.')
return None
# Query API.
client = xmlrpclib.ServerProxy('https://pypi.python.org/pypi')
results = client.search(dict(summary=''))
if not results:
LOG.error('Reply from API had no results.')
return list()
LOG.debug('Sorting results.')
results.sort(key=lambda x: (x['name'], LooseVersion(x['version'])))
filtered = (r for r in results if r['version'][0].isdigit())
packages = {r['name']: dict(summary=r['summary'], version=r['version'], id=0) for r in filtered}
LOG.debug('Pruning unchanged packages.')
for row in db.session.query(Package.id, Package.name, Package.summary, Package.latest_version):
if packages.get(row[1]) == dict(summary=row[2], version=row[3], id=0):
packages.pop(row[1])
elif row[1] in packages:
packages[row[1]]['id'] = row[0]
new_package_names = {n for n, d in packages.items() if not d['id']}
# Merge into database.
LOG.debug('Found {} new packages in PyPI, updating {} total.'.format(len(new_package_names), len(packages)))
with db.session.begin_nested():
for name, data in packages.items():
db.session.merge(Package(id=data['id'], name=name, summary=data['summary'], latest_version=data['version']))
db.session.commit()
return list(new_package_names)
def test_sync_parallel(alter_xmlrpc):
alter_xmlrpc([dict(name='packageD', summary='Test package.', version='3.0.0'), ])
redis.delete(POLL_SIMPLE_THROTTLE)
redis_key = CELERY_LOCK.format(task_name='pypi_portal.tasks.pypi.update_package_list')
lock = redis.lock(redis_key, timeout=1)
assert lock.acquire(blocking=False)
assert '302 FOUND' == current_app.test_client().get('/pypi/sync').status
expected = [('packageB', 'Test package.', '3.0.0'), ]
actual = db.session.query(Package.name, Package.summary, Package.latest_version).all()
assert expected == actual
try:
lock.release()
except LockError:
pass
def test_sync_parallel(alter_xmlrpc):
alter_xmlrpc([
dict(name='packageD', summary='Test package.', version='3.0.0'),
])
redis.delete(POLL_SIMPLE_THROTTLE)
redis_key = CELERY_LOCK.format(
task_name='pypi_portal.tasks.pypi.update_package_list')
lock = redis.lock(redis_key, timeout=1)
assert lock.acquire(blocking=False)
assert '302 FOUND' == current_app.test_client().get('/pypi/sync').status
expected = [
('packageB', 'Test package.', '3.0.0'),
]
actual = db.session.query(Package.name, Package.summary,
Package.latest_version).all()
assert expected == actual
try:
lock.release()
except LockError:
pass
