def send(func):
# Authenticate to smtp (if any auth needed) and send all emails
smtp = smtp_connect()
subscriber_sent, user_sent, subscriber_refused, user_refused = 0, 0, 0, 0
site_url = '%s://%s' % (settings.URL_SCHEME, settings.SITE_DOMAIN)
# iterate and send
while True:
try:
s, is_subscriber = func()
hashed_id = hashids.encode(int(s.id))
headers['List-Unsubscribe'] = '%s/usuarios/nlunsubscribe/c/%s/%s/' % (
site_url, category.slug, hashed_id)
msg = Message(html=render_to_string(
'category/newsletter/%s.html' % category.slug, {
'cover_article': cover_article, 'category': category, 'opinion_article': opinion_article,
'datos_article': datos_article, 'site_url': site_url, 'articles': top_articles,
'hashed_id': hashed_id, 'ga_property_id': getattr(settings, 'GA_PROPERTY_ID', None),
'subscriber_id': s.id, 'is_subscriber': is_subscriber,
'cover_article_section': cover_article_section,
}),
mail_to=(s.name, s.user.email), subject=remove_markup(cover_article.headline),
mail_from=(u'la diaria ' + category.name, EMAIL_FROM_ADDR), headers=headers)
# attach ads if any
for f_ad in f_ads:
f_ad_basename = os.path.basename(f_ad)
msg.attach(filename=f_ad_basename, data=open(f_ad, "rb"))
# send using smtp to receive bounces in another mailbox
try:
if not no_deliver:
smtp.sendmail(settings.NOTIFICATIONS_FROM_MX, [s.user.email], msg.as_string())
log.info("%s Email %s to %ssubscriber %s\t%s" % (
today, 'simulated' if no_deliver else 'sent', '' if is_subscriber else 'non-',
s.id, s.user.email))
if is_subscriber:
subscriber_sent += 1
else:
user_sent += 1
except smtplib.SMTPRecipientsRefused:
log.warning("%s Email refused for %ssubscriber %s\t%s" % (
today, '' if is_subscriber else 'non-', s.id, s.user.email))
if is_subscriber:
subscriber_refused += 1
else:
user_refused += 1
except smtplib.SMTPServerDisconnected:
log.error("MTA down, not sending - %s" % s.user.email)
log.info("Trying to reconnect for the next delivery...")
smtp = smtp_connect()
except StopIteration:
# append data processed to global results and quit.
# this can be done because list.append is threadsafe.
counters.append((subscriber_sent, user_sent, subscriber_refused, user_refused))
break
try:
smtp.quit()
except smtplib.SMTPServerDisconnected:
pass
def send_notification(user, email_template, email_subject, extra_context={}):
extra_context.update({
'SITE_URL': settings.SITE_URL,
'URL_SCHEME': settings.URL_SCHEME,
'site': Site.objects.get_current(),
'logo_url': settings.HOMEV3_SECONDARY_LOGO
})
msg = Message(html=render_to_string(email_template, extra_context),
mail_to=(user.get_full_name(), user.email),
subject=email_subject,
mail_from=(settings.NOTIFICATIONS_FROM_NAME,
getattr(
settings, 'NOTIFICATIONS_FROM_ADDR2'
if extra_context.get('seller_email') else
'NOTIFICATIONS_FROM_ADDR1')))
# Authenticate to SMTP
s = smtplib.SMTP(settings.EMAIL_HOST, settings.EMAIL_PORT)
try:
s.login(settings.EMAIL_HOST_USER, settings.EMAIL_HOST_PASSWORD)
except smtplib.SMTPException:
pass
# send using smtp to receive bounces in another mailbox
s.sendmail(settings.NOTIFICATIONS_FROM_MX, [user.email], msg.as_string())
s.quit()
def send_confirmation_link(*args, **kwargs):
request = args[0]
subject = kwargs['subject']
message_template = kwargs['message_template']
user = kwargs['user']
if 'edition' in kwargs:
generator_params = {'user': kwargs['user'], 'edition': kwargs['edition']}
else:
generator_params = {'user': kwargs['user']}
url_generator = kwargs['url_generator']
extra_context = kwargs['extra_context']
from_mail = getattr(settings, 'DEFAULT_FROM_EMAIL')
subject = '%s %s' % (getattr(settings, 'EMAIL_SUBJECT_PREFIX', ''), subject)
context = {'validation_url': url_generator(**generator_params)}
if extra_context:
context.update(extra_context)
message = Message(
html=loader.render_to_string(message_template, context, request),
mail_to=(user.first_name, user.email),
mail_from=from_mail,
subject=subject,
)
smtp = smtp_connect()
try:
smtp.sendmail(settings.NOTIFICATIONS_FROM_MX, [user.email], message.as_string())
if settings.DEBUG:
print('DEBUG: an email was sent from send_confirmation_link function')
smtp.quit()
except Exception:
# fail silently
pass
def test_django_message_send(django_email_backend):
message_params = {'html': '<p>Test from python-emails',
'mail_from': '*****@*****.**',
'subject': 'Test from python-emails'}
msg = DjangoMessage(**message_params)
assert not msg.recipients()
TO = '*****@*****.**'
msg.send(mail_to=TO, set_mail_to=False)
assert msg.recipients() == [TO, ]
assert not msg.mail_to
TO = 'x'+TO
msg.send(mail_to=TO)
assert msg.recipients() == [TO, ]
assert msg.mail_to[0][1] == TO
msg.send(context={'a': 1})
def send_notification(user, email_template, email_subject, extra_context={}):
extra_context.update(
{
'SITE_URL': settings.SITE_URL,
'URL_SCHEME': settings.URL_SCHEME,
'site': Site.objects.get_current(),
'logo_url': settings.HOMEV3_SECONDARY_LOGO,
}
)
msg = Message(
html=render_to_string(email_template, extra_context),
mail_to=(user.get_full_name(), user.email),
subject=email_subject,
mail_from=(settings.NOTIFICATIONS_FROM_NAME, settings.NOTIFICATIONS_FROM_ADDR1),
)
# send using smtp to receive bounces in another mailbox
s = smtp_connect()
s.sendmail(settings.NOTIFICATIONS_FROM_MX, [user.email], msg.as_string())
s.quit()
def send_confirmation_link(*args, **kwargs):
request = args[0]
subject = kwargs['subject']
message_template = kwargs['message_template']
user = kwargs['user']
if 'edition' in kwargs:
generator_params = {'user': kwargs['user'], 'edition': kwargs['edition']}
else:
generator_params = {'user': kwargs['user']}
url_generator = kwargs['url_generator']
extra_context = kwargs['extra_context']
from_mail = getattr(settings, 'DEFAULT_FROM_EMAIL')
subject = '%s %s' % (getattr(settings, 'EMAIL_SUBJECT_PREFIX', ''), subject)
context = {'validation_url': url_generator(**generator_params)}
if extra_context:
context.update(extra_context)
message = Message(
html=loader.render_to_string(
message_template, context, context_instance=RequestContext(request) if request else {}),
mail_to=(user.first_name, user.email), mail_from=from_mail, subject=subject)
message.send()
def test_django_message_commons():
mp = {
'html': '<p>Test from python-emails',
'mail_from': '*****@*****.**',
'mail_to': '*****@*****.**',
'charset': 'XXX-Y'
}
msg = DjangoMessage(**mp)
assert msg.encoding == mp['charset']
# --- check recipients()
assert msg.recipients() == [
mp['mail_to'],
]
msg._set_emails(mail_to='A', set_mail_to=False)
assert msg.recipients() == [
'A',
]
assert msg.mail_to[0][1] == mp['mail_to']
msg._set_emails(mail_to='*****@*****.**', set_mail_to=True)
assert msg.recipients() == [
'*****@*****.**',
]
assert msg.mail_to[0][1] == '*****@*****.**'
# --- check from_email
assert msg.from_email == mp['mail_from']
msg._set_emails(mail_from='*****@*****.**', set_mail_from=False)
assert msg.from_email == '*****@*****.**'
assert msg.mail_from[1] == mp['mail_from']
msg._set_emails(mail_from='*****@*****.**', set_mail_from=True)
assert msg.from_email == '*****@*****.**'
assert msg.mail_from[1] == '*****@*****.**'
def test_django_message_send(django_email_backend):
message_params = {
'html': '<p>Test from python-emails',
'mail_from': '*****@*****.**',
'subject': 'Test from python-emails'
}
msg = DjangoMessage(**message_params)
assert not msg.recipients()
TO = '*****@*****.**'
msg.send(mail_to=TO, set_mail_to=False)
assert msg.recipients() == [
TO,
]
assert not msg.mail_to
TO = 'x' + TO
msg.send(mail_to=TO)
assert msg.recipients() == [
TO,
]
assert msg.mail_to[0][1] == TO
msg.send(context={'a': 1})
def test_django_message_commons():
mp = {'html': '<p>Test from python-emails',
'mail_from': '*****@*****.**',
'mail_to': '*****@*****.**',
'charset': 'XXX-Y'}
msg = DjangoMessage(**mp)
assert msg.encoding == mp['charset']
# --- check recipients()
assert msg.recipients() == [mp['mail_to'], ]
msg._set_emails(mail_to='A', set_mail_to=False)
assert msg.recipients() == ['A', ]
assert msg.mail_to[0][1] == mp['mail_to']
msg._set_emails(mail_to='*****@*****.**', set_mail_to=True)
assert msg.recipients() == ['*****@*****.**', ]
assert msg.mail_to[0][1] == '*****@*****.**'
# --- check from_email
assert msg.from_email == mp['mail_from']
msg._set_emails(mail_from='*****@*****.**', set_mail_from=False)
assert msg.from_email == '*****@*****.**'
assert msg.mail_from[1] == mp['mail_from']
msg._set_emails(mail_from='*****@*****.**', set_mail_from=True)
assert msg.from_email == '*****@*****.**'
assert msg.mail_from[1] == '*****@*****.**'
def build_and_send(
category,
no_deliver,
offline,
export_subscribers,
export_context,
site_id,
starting_from_s,
starting_from_ns,
partitions,
mod,
subscriber_ids,
):
site = Site.objects.get(
id=site_id) if site_id else Site.objects.get_current()
category_slug, export_only = category if offline else category.slug, export_subscribers or export_context
context = {'category': category, 'newsletter_campaign': category_slug}
export_ctx = {'newsletter_campaign': category_slug}
try:
offline_ctx_file = join(settings.SENDNEWSLETTER_EXPORT_DIR,
'%s_ctx.json' % category_slug)
offline_csv_file = join(settings.SENDNEWSLETTER_EXPORT_DIR,
'%s_subscribers.csv' % category_slug)
if offline:
if starting_from_s or starting_from_ns:
log.error(
'--starting-from* options for offline usage not yet implemented'
)
return
context = json.loads(open(offline_ctx_file).read())
# de-serialize dates
dp_cover = datetime.strptime(
context['cover_article']['date_published'], '%Y-%m-%d').date()
context['cover_article']['date_published'] = dp_cover
featured_article = context['featured_article']
if featured_article:
dp_featured = datetime.strptime(
featured_article['date_published'], '%Y-%m-%d').date()
context['featured_article']['date_published'] = dp_featured
dp_articles = []
for a, a_section in context['articles']:
dp_article = datetime.strptime(a['date_published'],
'%Y-%m-%d').date()
a['date_published'] = dp_article
dp_articles.append((a, a_section))
context['articles'] = dp_articles
elif not export_subscribers or export_context:
category_nl = CategoryNewsletter.objects.get(
category=category, valid_until__gt=datetime.now())
cover_article, featured_article = category_nl.cover(
), category_nl.featured_article()
featured_article_section = featured_article.publication_section(
) if featured_article else None
if export_context:
export_ctx.update({
'articles': [(a.nl_serialize(position == 0), {
'name': section.name,
'slug': section.slug
}) for position, (a, section) in enumerate(
[(a, a.publication_section())
for a in category_nl.non_cover_articles()])],
'featured_article_section':
featured_article_section.name
if featured_article_section else None,
# TODO: check if cover_article_section and featured_articles entries are needed
})
else:
context.update({
'cover_article_section':
cover_article.publication_section().name
if cover_article else None,
'articles': [(a, a.publication_section())
for a in category_nl.non_cover_articles()],
'featured_article_section':
featured_article_section,
'featured_articles':
[(a, a.publication_section())
for a in category_nl.non_cover_featured_articles()],
})
except CategoryNewsletter.DoesNotExist:
if not (offline or export_subscribers) or export_context:
cover_article = category.home.cover()
cover_article_section = cover_article.publication_section(
) if cover_article else None
top_articles = [(a, a.publication_section())
for a in category.home.non_cover_articles()]
listonly_section = getattr(
settings, 'CORE_CATEGORY_NEWSLETTER_LISTONLY_SECTIONS',
{}).get(category_slug)
if listonly_section:
top_articles = [
t for t in top_articles if t[1].slug == listonly_section
]
if cover_article_section.slug != listonly_section:
cover_article = top_articles.pop(
0)[0] if top_articles else None
cover_article_section = cover_article.publication_section(
) if cover_article else None
featured_article_id = getattr(settings,
'NEWSLETTER_FEATURED_ARTICLE', False)
nl_featured = Article.objects.filter(
id=featured_article_id
) if featured_article_id else get_latest_edition(
).newsletter_featured_articles()
opinion_article = nl_featured[0] if nl_featured else None
# featured_article (a featured section in the category)
try:
featured_section, days_ago = settings.CORE_CATEGORY_NEWSLETTER_FEATURED_SECTIONS[
category_slug]
featured_article = category.section_set.get(
slug=featured_section).latest_article()[0]
assert (featured_article.date_published >=
datetime.now() - timedelta(days_ago))
except (KeyError, Section.DoesNotExist,
Section.MultipleObjectsReturned, IndexError,
AssertionError):
featured_article = None
if export_context:
export_ctx['articles'] = [(t[0].nl_serialize(position == 0), {
'name': t[1].name,
'slug': t[1].slug
}) for position, t in enumerate(top_articles)]
else:
context.update({
'opinion_article': opinion_article,
'cover_article_section': cover_article_section,
'articles': top_articles,
})
# any custom attached files
# TODO: make this a feature in the admin using adzone also make it path-setting instead of absolute
# f_ads = ['/srv/ldsocial/portal/media/document.pdf']
f_ads = []
if not offline:
receivers = Subscriber.objects.filter(user__is_active=True).exclude(
user__email='')
if subscriber_ids:
receivers = receivers.filter(id__in=subscriber_ids)
else:
receivers = receivers.filter(
category_newsletters__slug=category_slug).exclude(
user__email__in=blacklisted)
# if both "starting_from" we can filter now with the minimum
if starting_from_s and starting_from_ns:
receivers = receivers.filter(
user__email__gt=min(starting_from_s, starting_from_ns))
if partitions is not None and mod is not None:
receivers = receivers.extra(where=[
'MOD(%s.id,%d)=%d' %
(Subscriber._meta.db_table, partitions, mod)
])
if offline:
custom_subject = context['custom_subject']
email_subject = context['email_subject']
email_from = context['email_from']
site_url = context['site_url']
list_id = context['list_id']
ga_property_id = context['ga_property_id']
r = reader(open(offline_csv_file))
if subscriber_ids:
subscribers_iter = subscribers_nl_iter_filter(
r, lambda row: int(row[0]) in subscriber_ids)
elif partitions is not None and mod is not None:
subscribers_iter = subscribers_nl_iter_filter(
r, lambda row: int(row[0]) % partitions == mod)
else:
subscribers_iter = r
elif not export_subscribers or export_context:
site_url = '%s://%s' % (settings.URL_SCHEME, settings.SITE_DOMAIN)
list_id = '%s <%s.%s>' % (category_slug, __name__,
settings.SITE_DOMAIN)
ga_property_id = getattr(settings, 'GA_PROPERTY_ID', None)
custom_subject = category.newsletter_automatic_subject is False and category.newsletter_subject
email_subject = custom_subject or (
getattr(settings, 'CORE_CATEGORY_NL_SUBJECT_PREFIX', {}).get(
category_slug, u'') + remove_markup(cover_article.headline))
email_from = (
site.name if category_slug in getattr(
settings, 'CORE_CATEGORY_NL_FROM_NAME_SITEONLY', ()) else
(u'%s %s' % (site.name, category.name)),
settings.NOTIFICATIONS_FROM_ADDR1,
)
translation.activate(settings.LANGUAGE_CODE)
if not export_subscribers or export_context:
common_ctx = {
'site_url': site_url,
'ga_property_id': ga_property_id,
'custom_subject': custom_subject
}
if export_only:
if export_context:
export_ctx.update(common_ctx)
export_ctx.update({
'email_subject':
email_subject,
'email_from':
email_from,
'list_id':
list_id,
'cover_article':
cover_article.nl_serialize(True),
'featured_article':
featured_article.nl_serialize(True)
if featured_article else None,
})
open(offline_ctx_file, 'w').write(json.dumps(export_ctx))
if export_subscribers:
export_subscribers_writer = writer(open(offline_csv_file, 'w'))
else:
return
elif not offline:
context.update(common_ctx)
context.update({
'cover_article': cover_article,
'featured_article': featured_article
})
if not offline:
subscribers_iter = subscribers_nl_iter(receivers, starting_from_s,
starting_from_ns)
# Connect to the SMTP server and send all emails
try:
smtp = None if no_deliver or export_only else smtp_connect()
except error:
log.error("MTA down, '%s %s' was used for partitions and mod" %
(partitions, mod))
return
subscriber_sent, user_sent, subscriber_refused, user_refused = 0, 0, 0, 0
retry_last_delivery, s_id, is_subscriber = False, None, None
email_template = Engine.get_default().get_template(
'%s/newsletter/%s.html' %
(settings.CORE_CATEGORIES_TEMPLATE_DIR, category_slug))
while True:
try:
if not retry_last_delivery:
if offline:
s_id, s_name, s_user_email, hashed_id, is_subscriber, is_subscriber_any, is_subscriber_default = (
subscribers_iter.next())
is_subscriber = eval(is_subscriber)
is_subscriber_any = eval(is_subscriber_any)
is_subscriber_default = eval(is_subscriber_default)
else:
s, is_subscriber = subscribers_iter.next()
s_id, s_name, s_user_email = s.id, s.name, s.user.email
hashed_id = hashids.encode(int(s_id))
is_subscriber_any = s.is_subscriber_any()
is_subscriber_default = s.is_subscriber(
settings.DEFAULT_PUB)
if export_subscribers:
export_subscribers_writer.writerow([
s_id, s_name, s_user_email, hashed_id, is_subscriber,
is_subscriber_any, is_subscriber_default
])
elif not export_context:
headers = {'List-ID': list_id}
unsubscribe_url = '%s/usuarios/nlunsubscribe/c/%s/%s/?utm_source=newsletter&utm_medium=email' \
'&utm_campaign=%s&utm_content=unsubscribe' % (site_url, category_slug, hashed_id, category_slug)
headers['List-Unsubscribe'] = headers[
'List-Unsubscribe-Post'] = '<%s>' % unsubscribe_url
context.update({
'hashed_id': hashed_id,
'unsubscribe_url': unsubscribe_url,
'subscriber_id': s_id,
'is_subscriber': is_subscriber,
'is_subscriber_any': is_subscriber_any,
'is_subscriber_default': is_subscriber_default,
})
msg = Message(
html=email_template.render(Context(context)),
mail_to=(s_name, s_user_email),
subject=email_subject,
mail_from=email_from,
headers=headers,
)
# attach ads if any
for f_ad in f_ads:
f_ad_basename = basename(f_ad)
msg.attach(filename=f_ad_basename, data=open(f_ad, "rb"))
# send using smtp to receive bounces in another mailbox
try:
if not no_deliver:
smtp.sendmail(settings.NOTIFICATIONS_FROM_MX,
[s_user_email], msg.as_string())
log.info(
"Email %s to %ssubscriber %s\t%s" %
('simulated' if no_deliver else 'sent',
'' if is_subscriber else 'non-', s_id, s_user_email))
if is_subscriber:
subscriber_sent += 1
else:
user_sent += 1
except smtplib.SMTPRecipientsRefused:
log.warning(
"Email refused for %ssubscriber %s\t%s" %
('' if is_subscriber else 'non-', s_id, s_user_email))
if is_subscriber:
subscriber_refused += 1
else:
user_refused += 1
except smtplib.SMTPServerDisconnected:
# Retries are made only once per iteration to avoid infinite loop if MTA got down at all
retry_last_delivery = not retry_last_delivery
log_message = "MTA down, email to %s not sent. Reconnecting " % s_user_email
if retry_last_delivery:
log.warning(log_message + "to retry ...")
else:
log.error(log_message + "for the next delivery ...")
try:
smtp = smtp_connect()
except error:
log.warning('MTA reconnect failed')
else:
retry_last_delivery = False
except (ProgrammingError, OperationalError, StopIteration) as exc:
# the connection to databse can be killed, if that is the case print useful log to continue
if isinstance(exc, (ProgrammingError, OperationalError)):
log.error(
'DB connection error, (%s, %s, %s, %s) was the last delivery attempt'
% (s_user_email if s_id else None, is_subscriber,
partitions, mod))
break
if not export_only:
if not no_deliver:
try:
smtp.quit()
except smtplib.SMTPServerDisconnected:
pass
# update log stats counters only if subscriber_ids not given
if not subscriber_ids:
try:
# close connections because reach this point can take several minutes
close_old_connections()
# A transaction is needed because autocommit in django is broken in concurrent management processes
cursor = connection.cursor()
cursor.execute('BEGIN')
cursor.execute("""
INSERT INTO dashboard_newsletterdelivery(
delivery_date,newsletter_name,user_sent,subscriber_sent,user_refused,subscriber_refused
)
VALUES('%s','%s',%d,%d,%d,%d)
""" % (today, category_slug, user_sent, subscriber_sent,
user_refused, subscriber_refused))
cursor.execute('COMMIT')
except IntegrityError:
nl_delivery = NewsletterDelivery.objects.get(
delivery_date=today, newsletter_name=category_slug)
nl_delivery.user_sent = (nl_delivery.user_sent
or 0) + user_sent
nl_delivery.subscriber_sent = (nl_delivery.subscriber_sent
or 0) + subscriber_sent
nl_delivery.user_refused = (nl_delivery.user_refused
or 0) + user_refused
nl_delivery.subscriber_refused = (
nl_delivery.subscriber_refused or 0) + subscriber_refused
nl_delivery.save()
except Exception as e:
log.error(u'Delivery stats not updated: %s' % e)
log.info(
u'%s stats: user_sent: %d, subscriber_sent: %s, user_refused: %d, subscriber_refused: %d'
% (
'Simulation' if no_deliver else 'Delivery',
user_sent,
subscriber_sent,
user_refused,
subscriber_refused,
))