def category_detail(request, slug):
category, inner_sections = get_object_or_404(Category, slug=slug), []
question_list, questions_topic = [], None
category_home = get_object_or_404(CategoryHome, category=category)
try:
featured_section1 = category.section_set.get(home_order=1)
except (Section.DoesNotExist, Section.MultipleObjectsReturned):
featured_section1 = None
try:
featured_section2 = category.section_set.get(home_order=2)
except (Section.DoesNotExist, Section.MultipleObjectsReturned):
featured_section2 = None
try:
featured_section3 = category.section_set.get(home_order=3)
except (Section.DoesNotExist, Section.MultipleObjectsReturned):
featured_section3 = None
for inner_section_slug in getattr(settings, 'CORE_CATEGORY_INNER_SECTIONS',
{}).get(slug, ()):
try:
inner_sections.append(
category.section_set.get(slug=inner_section_slug))
except Section.DoesNotExist:
pass
if slug in getattr(settings, 'CORE_CATEGORIES_ENABLE_QUESTIONS', ()):
question_list = Question.published.filter(topic__slug=slug)
try:
questions_topic = Topic.objects.get(slug=slug)
except Topic.DoesNotExist:
pass
return render(
request,
'%s/%s.html' %
(getattr(settings, 'CORE_CATEGORIES_TEMPLATE_DIR',
'core/templates/category'), category.slug if category.slug
in getattr(settings, 'CORE_CATEGORIES_CUSTOM_TEMPLATES',
()) else 'detail'),
{
'category': category,
'cover_article': category_home.cover(),
'destacados': category_home.non_cover_articles(),
'is_portada': True,
'featured_section1': featured_section1,
'featured_section2': featured_section2,
'featured_section3': featured_section3,
'inner_sections': inner_sections,
'edition': get_latest_edition(),
'question_list': question_list,
'questions_topic': questions_topic,
'big_photo': category.full_width_cover_image,
},
)
class HomeArticleViewSet(viewsets.ModelViewSet):
try:
edition = get_current_edition() or get_latest_edition()
except Exception:
edition = None
pk_list = [a.id for a in edition.top_articles] if edition else []
clauses = ' '.join(['WHEN id=%s THEN %s' % (pk, i) for i, pk in enumerate(pk_list)])
ordering = 'CASE %s END' % clauses
queryset = Article.objects.filter(id__in=pk_list).extra(select={'ordering': ordering}, order_by=('ordering', ))
serializer_class = ArticleSerializer
http_method_names = ['get', 'head']
filter_fields = ('headline', 'sections')
def newsletter_preview(request, slug):
# allow only staff members or requests from localhost
if not (request.user.is_staff or request.META.get('REMOTE_ADDR') in (
socket.gethostbyname('localhost'), socket.gethostbyname(socket.gethostname()))):
return HttpResponseForbidden()
# removed or changed categories redirects by settings
if slug in getattr(settings, 'CORE_CATEGORY_REDIRECT', {}):
return HttpResponsePermanentRedirect(
reverse('category-nl-preview', args=(settings.CORE_CATEGORY_REDIRECT[slug], )))
category = get_object_or_404(Category, slug=slug)
category_home = get_object_or_404(Home, category=category)
cat_modules = category_home.modules.all()
try:
cover_article = category_home.cover
cover_article_section = cover_article.publication_section()
top_articles = [
(a, a.publication_section(), 0) for a in (cat_modules[0].articles_as_list if cat_modules else [])]
opinion_article = None
nl_featured = Article.objects.filter(id=settings.NEWSLETTER_FEATURED_ARTICLE) if \
getattr(settings, 'NEWSLETTER_FEATURED_ARTICLE', False) else \
get_latest_edition().newsletter_featured_articles()
if nl_featured:
opinion_article = nl_featured[0]
# datos_article for elecciones
try:
datos_article = category.section_set.get(slug='datos-elecciones-2019').latest_article()[0]
except (Section.DoesNotExist, Section.MultipleObjectsReturned, IndexError):
datos_article = None
try:
hashed_id = hashids.encode(int(request.user.subscriber.id))
except AttributeError:
hashed_id = hashids.encode(0)
return 'newsletter/%s.html' % slug, {
'cover_article': cover_article, 'category': category, 'cover_article_section': cover_article_section,
'articles': top_articles, 'hashed_id': hashed_id, 'opinion_article': opinion_article,
'site_url': '%s://%s' % (settings.URL_SCHEME, settings.SITE_DOMAIN), 'datos_article':
datos_article if
datos_article and datos_article.date_published >= datetime.now() - timedelta(1) else None}
except Exception as e:
return HttpResponseServerError(u'ERROR: %s' % e)
def category_detail(request, slug):
# deporte custom redirect
if slug == u'deporte':
return redirect(Publication.objects.get(slug=u'garra').get_absolute_url(), permanent=True)
category, inner_sections = get_object_or_404(Category, slug=slug), []
question_list, questions_topic = [], None
category_home = get_object_or_404(Home, category=category)
cat_modules = category_home.modules.all()
top_articles = cat_modules[0].articles_as_list if cat_modules else []
try:
featured_section1 = category.section_set.get(home_order=1)
except (Section.DoesNotExist, Section.MultipleObjectsReturned):
featured_section1 = None
try:
featured_section2 = category.section_set.get(home_order=2)
except (Section.DoesNotExist, Section.MultipleObjectsReturned):
featured_section2 = None
try:
featured_section3 = category.section_set.get(home_order=3)
except (Section.DoesNotExist, Section.MultipleObjectsReturned):
featured_section3 = None
# custom inner sections
if slug == u'elecciones':
# example dropdown menu (elecciones ir disabled now by redirect)
inner_sections = category.section_set.filter(
Q(slug__startswith='partido-') | Q(slug__in=(
'frente-amplio', 'unidad-popular', 'cabildo-abierto', 'otros')))
if slug == u'coronavirus':
question_list = Question.published.filter(topic__slug='coronavirus')
try:
questions_topic = Topic.objects.get(slug="coronavirus")
except Topic.DoesNotExist:
pass
return render_to_response(
'%s/%s.html' % (
getattr(settings, 'CORE_CATEGORIES_TEMPLATE_DIR', 'core/templates/category'),
category.slug if category.slug in getattr(settings, 'CORE_CATEGORIES_CUSTOM_TEMPLATES', ()) else 'detail'
), {
'category': category, 'cover_article': category_home.cover, 'destacados': top_articles, 'is_portada': True,
'featured_section1': featured_section1, 'featured_section2': featured_section2,
'featured_section3': featured_section3, 'inner_sections': inner_sections, 'edition': get_latest_edition(),
'question_list': question_list, 'questions_topic': questions_topic,
'big_photo': category.full_width_cover_image}, context_instance=RequestContext(request))
def build_and_send(
category,
no_deliver,
offline,
export_subscribers,
export_context,
site_id,
starting_from_s,
starting_from_ns,
partitions,
mod,
subscriber_ids,
):
site = Site.objects.get(
id=site_id) if site_id else Site.objects.get_current()
category_slug, export_only = category if offline else category.slug, export_subscribers or export_context
context = {'category': category, 'newsletter_campaign': category_slug}
export_ctx = {'newsletter_campaign': category_slug}
try:
offline_ctx_file = join(settings.SENDNEWSLETTER_EXPORT_DIR,
'%s_ctx.json' % category_slug)
offline_csv_file = join(settings.SENDNEWSLETTER_EXPORT_DIR,
'%s_subscribers.csv' % category_slug)
if offline:
if starting_from_s or starting_from_ns:
log.error(
'--starting-from* options for offline usage not yet implemented'
)
return
context = json.loads(open(offline_ctx_file).read())
# de-serialize dates
dp_cover = datetime.strptime(
context['cover_article']['date_published'], '%Y-%m-%d').date()
context['cover_article']['date_published'] = dp_cover
featured_article = context['featured_article']
if featured_article:
dp_featured = datetime.strptime(
featured_article['date_published'], '%Y-%m-%d').date()
context['featured_article']['date_published'] = dp_featured
dp_articles = []
for a, a_section in context['articles']:
dp_article = datetime.strptime(a['date_published'],
'%Y-%m-%d').date()
a['date_published'] = dp_article
dp_articles.append((a, a_section))
context['articles'] = dp_articles
elif not export_subscribers or export_context:
category_nl = CategoryNewsletter.objects.get(
category=category, valid_until__gt=datetime.now())
cover_article, featured_article = category_nl.cover(
), category_nl.featured_article()
featured_article_section = featured_article.publication_section(
) if featured_article else None
if export_context:
export_ctx.update({
'articles': [(a.nl_serialize(position == 0), {
'name': section.name,
'slug': section.slug
}) for position, (a, section) in enumerate(
[(a, a.publication_section())
for a in category_nl.non_cover_articles()])],
'featured_article_section':
featured_article_section.name
if featured_article_section else None,
# TODO: check if cover_article_section and featured_articles entries are needed
})
else:
context.update({
'cover_article_section':
cover_article.publication_section().name
if cover_article else None,
'articles': [(a, a.publication_section())
for a in category_nl.non_cover_articles()],
'featured_article_section':
featured_article_section,
'featured_articles':
[(a, a.publication_section())
for a in category_nl.non_cover_featured_articles()],
})
except CategoryNewsletter.DoesNotExist:
if not (offline or export_subscribers) or export_context:
cover_article = category.home.cover()
cover_article_section = cover_article.publication_section(
) if cover_article else None
top_articles = [(a, a.publication_section())
for a in category.home.non_cover_articles()]
listonly_section = getattr(
settings, 'CORE_CATEGORY_NEWSLETTER_LISTONLY_SECTIONS',
{}).get(category_slug)
if listonly_section:
top_articles = [
t for t in top_articles if t[1].slug == listonly_section
]
if cover_article_section.slug != listonly_section:
cover_article = top_articles.pop(
0)[0] if top_articles else None
cover_article_section = cover_article.publication_section(
) if cover_article else None
featured_article_id = getattr(settings,
'NEWSLETTER_FEATURED_ARTICLE', False)
nl_featured = Article.objects.filter(
id=featured_article_id
) if featured_article_id else get_latest_edition(
).newsletter_featured_articles()
opinion_article = nl_featured[0] if nl_featured else None
# featured_article (a featured section in the category)
try:
featured_section, days_ago = settings.CORE_CATEGORY_NEWSLETTER_FEATURED_SECTIONS[
category_slug]
featured_article = category.section_set.get(
slug=featured_section).latest_article()[0]
assert (featured_article.date_published >=
datetime.now() - timedelta(days_ago))
except (KeyError, Section.DoesNotExist,
Section.MultipleObjectsReturned, IndexError,
AssertionError):
featured_article = None
if export_context:
export_ctx['articles'] = [(t[0].nl_serialize(position == 0), {
'name': t[1].name,
'slug': t[1].slug
}) for position, t in enumerate(top_articles)]
else:
context.update({
'opinion_article': opinion_article,
'cover_article_section': cover_article_section,
'articles': top_articles,
})
# any custom attached files
# TODO: make this a feature in the admin using adzone also make it path-setting instead of absolute
# f_ads = ['/srv/ldsocial/portal/media/document.pdf']
f_ads = []
if not offline:
receivers = Subscriber.objects.filter(user__is_active=True).exclude(
user__email='')
if subscriber_ids:
receivers = receivers.filter(id__in=subscriber_ids)
else:
receivers = receivers.filter(
category_newsletters__slug=category_slug).exclude(
user__email__in=blacklisted)
# if both "starting_from" we can filter now with the minimum
if starting_from_s and starting_from_ns:
receivers = receivers.filter(
user__email__gt=min(starting_from_s, starting_from_ns))
if partitions is not None and mod is not None:
receivers = receivers.extra(where=[
'MOD(%s.id,%d)=%d' %
(Subscriber._meta.db_table, partitions, mod)
])
if offline:
custom_subject = context['custom_subject']
email_subject = context['email_subject']
email_from = context['email_from']
site_url = context['site_url']
list_id = context['list_id']
ga_property_id = context['ga_property_id']
r = reader(open(offline_csv_file))
if subscriber_ids:
subscribers_iter = subscribers_nl_iter_filter(
r, lambda row: int(row[0]) in subscriber_ids)
elif partitions is not None and mod is not None:
subscribers_iter = subscribers_nl_iter_filter(
r, lambda row: int(row[0]) % partitions == mod)
else:
subscribers_iter = r
elif not export_subscribers or export_context:
site_url = '%s://%s' % (settings.URL_SCHEME, settings.SITE_DOMAIN)
list_id = '%s <%s.%s>' % (category_slug, __name__,
settings.SITE_DOMAIN)
ga_property_id = getattr(settings, 'GA_PROPERTY_ID', None)
custom_subject = category.newsletter_automatic_subject is False and category.newsletter_subject
email_subject = custom_subject or (
getattr(settings, 'CORE_CATEGORY_NL_SUBJECT_PREFIX', {}).get(
category_slug, u'') + remove_markup(cover_article.headline))
email_from = (
site.name if category_slug in getattr(
settings, 'CORE_CATEGORY_NL_FROM_NAME_SITEONLY', ()) else
(u'%s %s' % (site.name, category.name)),
settings.NOTIFICATIONS_FROM_ADDR1,
)
translation.activate(settings.LANGUAGE_CODE)
if not export_subscribers or export_context:
common_ctx = {
'site_url': site_url,
'ga_property_id': ga_property_id,
'custom_subject': custom_subject
}
if export_only:
if export_context:
export_ctx.update(common_ctx)
export_ctx.update({
'email_subject':
email_subject,
'email_from':
email_from,
'list_id':
list_id,
'cover_article':
cover_article.nl_serialize(True),
'featured_article':
featured_article.nl_serialize(True)
if featured_article else None,
})
open(offline_ctx_file, 'w').write(json.dumps(export_ctx))
if export_subscribers:
export_subscribers_writer = writer(open(offline_csv_file, 'w'))
else:
return
elif not offline:
context.update(common_ctx)
context.update({
'cover_article': cover_article,
'featured_article': featured_article
})
if not offline:
subscribers_iter = subscribers_nl_iter(receivers, starting_from_s,
starting_from_ns)
# Connect to the SMTP server and send all emails
try:
smtp = None if no_deliver or export_only else smtp_connect()
except error:
log.error("MTA down, '%s %s' was used for partitions and mod" %
(partitions, mod))
return
subscriber_sent, user_sent, subscriber_refused, user_refused = 0, 0, 0, 0
retry_last_delivery, s_id, is_subscriber = False, None, None
email_template = Engine.get_default().get_template(
'%s/newsletter/%s.html' %
(settings.CORE_CATEGORIES_TEMPLATE_DIR, category_slug))
while True:
try:
if not retry_last_delivery:
if offline:
s_id, s_name, s_user_email, hashed_id, is_subscriber, is_subscriber_any, is_subscriber_default = (
subscribers_iter.next())
is_subscriber = eval(is_subscriber)
is_subscriber_any = eval(is_subscriber_any)
is_subscriber_default = eval(is_subscriber_default)
else:
s, is_subscriber = subscribers_iter.next()
s_id, s_name, s_user_email = s.id, s.name, s.user.email
hashed_id = hashids.encode(int(s_id))
is_subscriber_any = s.is_subscriber_any()
is_subscriber_default = s.is_subscriber(
settings.DEFAULT_PUB)
if export_subscribers:
export_subscribers_writer.writerow([
s_id, s_name, s_user_email, hashed_id, is_subscriber,
is_subscriber_any, is_subscriber_default
])
elif not export_context:
headers = {'List-ID': list_id}
unsubscribe_url = '%s/usuarios/nlunsubscribe/c/%s/%s/?utm_source=newsletter&utm_medium=email' \
'&utm_campaign=%s&utm_content=unsubscribe' % (site_url, category_slug, hashed_id, category_slug)
headers['List-Unsubscribe'] = headers[
'List-Unsubscribe-Post'] = '<%s>' % unsubscribe_url
context.update({
'hashed_id': hashed_id,
'unsubscribe_url': unsubscribe_url,
'subscriber_id': s_id,
'is_subscriber': is_subscriber,
'is_subscriber_any': is_subscriber_any,
'is_subscriber_default': is_subscriber_default,
})
msg = Message(
html=email_template.render(Context(context)),
mail_to=(s_name, s_user_email),
subject=email_subject,
mail_from=email_from,
headers=headers,
)
# attach ads if any
for f_ad in f_ads:
f_ad_basename = basename(f_ad)
msg.attach(filename=f_ad_basename, data=open(f_ad, "rb"))
# send using smtp to receive bounces in another mailbox
try:
if not no_deliver:
smtp.sendmail(settings.NOTIFICATIONS_FROM_MX,
[s_user_email], msg.as_string())
log.info(
"Email %s to %ssubscriber %s\t%s" %
('simulated' if no_deliver else 'sent',
'' if is_subscriber else 'non-', s_id, s_user_email))
if is_subscriber:
subscriber_sent += 1
else:
user_sent += 1
except smtplib.SMTPRecipientsRefused:
log.warning(
"Email refused for %ssubscriber %s\t%s" %
('' if is_subscriber else 'non-', s_id, s_user_email))
if is_subscriber:
subscriber_refused += 1
else:
user_refused += 1
except smtplib.SMTPServerDisconnected:
# Retries are made only once per iteration to avoid infinite loop if MTA got down at all
retry_last_delivery = not retry_last_delivery
log_message = "MTA down, email to %s not sent. Reconnecting " % s_user_email
if retry_last_delivery:
log.warning(log_message + "to retry ...")
else:
log.error(log_message + "for the next delivery ...")
try:
smtp = smtp_connect()
except error:
log.warning('MTA reconnect failed')
else:
retry_last_delivery = False
except (ProgrammingError, OperationalError, StopIteration) as exc:
# the connection to databse can be killed, if that is the case print useful log to continue
if isinstance(exc, (ProgrammingError, OperationalError)):
log.error(
'DB connection error, (%s, %s, %s, %s) was the last delivery attempt'
% (s_user_email if s_id else None, is_subscriber,
partitions, mod))
break
if not export_only:
if not no_deliver:
try:
smtp.quit()
except smtplib.SMTPServerDisconnected:
pass
# update log stats counters only if subscriber_ids not given
if not subscriber_ids:
try:
# close connections because reach this point can take several minutes
close_old_connections()
# A transaction is needed because autocommit in django is broken in concurrent management processes
cursor = connection.cursor()
cursor.execute('BEGIN')
cursor.execute("""
INSERT INTO dashboard_newsletterdelivery(
delivery_date,newsletter_name,user_sent,subscriber_sent,user_refused,subscriber_refused
)
VALUES('%s','%s',%d,%d,%d,%d)
""" % (today, category_slug, user_sent, subscriber_sent,
user_refused, subscriber_refused))
cursor.execute('COMMIT')
except IntegrityError:
nl_delivery = NewsletterDelivery.objects.get(
delivery_date=today, newsletter_name=category_slug)
nl_delivery.user_sent = (nl_delivery.user_sent
or 0) + user_sent
nl_delivery.subscriber_sent = (nl_delivery.subscriber_sent
or 0) + subscriber_sent
nl_delivery.user_refused = (nl_delivery.user_refused
or 0) + user_refused
nl_delivery.subscriber_refused = (
nl_delivery.subscriber_refused or 0) + subscriber_refused
nl_delivery.save()
except Exception as e:
log.error(u'Delivery stats not updated: %s' % e)
log.info(
u'%s stats: user_sent: %d, subscriber_sent: %s, user_refused: %d, subscriber_refused: %d'
% (
'Simulation' if no_deliver else 'Delivery',
user_sent,
subscriber_sent,
user_refused,
subscriber_refused,
))
def newsletter_preview(request, slug):
# allow only staff members or requests from localhost
if not (request.user.is_staff or request.META.get('REMOTE_ADDR')
in (socket.gethostbyname('localhost'),
socket.gethostbyname(socket.gethostname()))):
return HttpResponseForbidden()
# removed or changed categories redirects by settings
category_redirections = getattr(settings, 'CORE_CATEGORY_REDIRECT', {})
if slug in category_redirections:
redirect_slug = category_redirections[slug]
if redirect_slug:
return HttpResponsePermanentRedirect(
reverse('category-nl-preview',
args=(settings.CORE_CATEGORY_REDIRECT[slug], )))
site = Site.objects.get_current()
category = get_object_or_404(Category, slug=slug)
try:
context = {'category': category, 'newsletter_campaign': category.slug}
try:
category_nl = CategoryNewsletter.objects.get(
category=category, valid_until__gt=datetime.now())
cover_article, featured_article = category_nl.cover(
), category_nl.featured_article()
context.update({
'cover_article_section':
cover_article.publication_section() if cover_article else None,
'articles': [(a, a.publication_section())
for a in category_nl.non_cover_articles()],
'featured_article_section':
featured_article.publication_section()
if featured_article else None,
'featured_articles':
[(a, a.publication_section())
for a in category_nl.non_cover_featured_articles()],
})
except CategoryNewsletter.DoesNotExist:
category_home = get_object_or_404(CategoryHome, category=category)
cover_article = category_home.cover()
cover_article_section = cover_article.publication_section(
) if cover_article else None
top_articles = [(a, a.publication_section())
for a in category_home.non_cover_articles()]
listonly_section = getattr(
settings, 'CORE_CATEGORY_NEWSLETTER_LISTONLY_SECTIONS',
{}).get(category.slug)
if listonly_section:
top_articles = [
t for t in top_articles if t[1].slug == listonly_section
]
if cover_article_section.slug != listonly_section:
cover_article = top_articles.pop(
0)[0] if top_articles else None
cover_article_section = cover_article.publication_section(
) if cover_article else None
featured_article_id = getattr(settings,
'NEWSLETTER_FEATURED_ARTICLE', False)
nl_featured = Article.objects.filter(
id=featured_article_id
) if featured_article_id else get_latest_edition(
).newsletter_featured_articles()
opinion_article = nl_featured[0] if nl_featured else None
# featured_article (a featured section in the category)
try:
featured_section, days_ago = settings.CORE_CATEGORY_NEWSLETTER_FEATURED_SECTIONS[
category.slug]
featured_article = category.section_set.get(
slug=featured_section).latest_article()[0]
assert (featured_article.date_published >=
datetime.now() - timedelta(days_ago))
except (KeyError, Section.DoesNotExist,
Section.MultipleObjectsReturned, IndexError,
AssertionError):
featured_article = None
context.update({
'opinion_article': opinion_article,
'cover_article_section': cover_article_section,
'articles': top_articles,
})
try:
hashed_id = hashids.encode(int(request.user.subscriber.id))
except AttributeError:
hashed_id = hashids.encode(0)
custom_subject = category.newsletter_automatic_subject is False and category.newsletter_subject
email_subject = custom_subject or (
getattr(settings, 'CORE_CATEGORY_NL_SUBJECT_PREFIX', {}).get(
category.slug, u'') + remove_markup(cover_article.headline))
email_from = u'%s <%s>' % (
site.name if category.slug in getattr(
settings, 'CORE_CATEGORY_NL_FROM_NAME_SITEONLY', ()) else
(u'%s %s' % (site.name, category.name)),
settings.NOTIFICATIONS_FROM_ADDR1,
)
headers = {'From': email_from, 'Subject': email_subject}
site_url = '%s://%s' % (settings.URL_SCHEME, settings.SITE_DOMAIN)
unsubscribe_url = '%s/usuarios/nlunsubscribe/c/%s/%s/?utm_source=newsletter&utm_medium=email' \
'&utm_campaign=%s&utm_content=unsubscribe' % (site_url, category.slug, hashed_id, category.slug)
headers['List-Unsubscribe'] = headers[
'List-Unsubscribe-Post'] = '<%s>' % unsubscribe_url
context.update({
'site_url': site_url,
'hashed_id': hashed_id,
'unsubscribe_url': unsubscribe_url,
'custom_subject': custom_subject,
'headers_preview': headers,
'cover_article': cover_article,
'featured_article': featured_article,
})
# override is_subscriber_any and _default only to "False"
for suffix in ('any', 'default'):
is_subscriber_var = 'is_subscriber_' + suffix
is_subscriber_val = request.GET.get(is_subscriber_var)
if is_subscriber_val and is_subscriber_val.lower() in (u'false',
u'0'):
context[is_subscriber_var] = False
return render(
request, '%s/newsletter/%s.html' %
(settings.CORE_CATEGORIES_TEMPLATE_DIR, slug), context)
except Exception as e:
if settings.DEBUG:
exc_type, exc_value, exc_traceback = sys.exc_info()
print(exc_type)
print(exc_value)
print(traceback.extract_tb(exc_traceback))
return HttpResponseServerError(u'ERROR: %s' % e)
def build_and_send(category, nthreads, no_deliver, starting_from_s, starting_from_ns, subscriber_ids, even, odd):
headers = {}
category_home = Home.objects.get(category=category)
cat_modules = category_home.modules.all()
cover_article = category_home.cover
cover_article_section = cover_article.publication_section()
top_articles = [(a, a.publication_section(), 0) for a in (cat_modules[0].articles_as_list if cat_modules else [])]
opinion_article = None
nl_featured = Article.objects.filter(id=settings.NEWSLETTER_FEATURED_ARTICLE) if \
getattr(settings, 'NEWSLETTER_FEATURED_ARTICLE', False) else \
get_latest_edition().newsletter_featured_articles()
if nl_featured:
opinion_article = nl_featured[0]
# datos_article (a featured section in the category)
try:
datos_article = category.section_set.get(slug='datos').latest_article()[0]
assert (datos_article.date_published >= datetime.now() - timedelta(1))
except (Section.DoesNotExist, Section.MultipleObjectsReturned, IndexError, AssertionError):
datos_article = None
# any custom attached files
# TODO: make this a feature in the admin using adzone also make it path-setting instead of absolute
# f_ads = ['/srv/ldsocial/portal/media/document.pdf']
f_ads = []
filter_args = {'user__is_active': True}
if subscriber_ids:
filter_args['id__in'] = subscriber_ids
else:
filter_args['category_newsletters__slug'] = category.slug
if even or odd:
filter_args['id__iregex'] = r'^\d*[%s]$' % ('02468' if even else '13579')
blacklisted = set([row[0] for row in csv.reader(open(settings.CORE_NEWSLETTER_BLACKLIST))])
# iterate over receivers and yield the subscribers first, saving the
# not subscribers ids in a temporal list an then yield them also
@threadsafe_generator
def subscribers():
receivers2 = []
for s in Subscriber.objects.filter(**filter_args).distinct().order_by('user__email').iterator():
if s.user and s.user.email and s.user.email not in blacklisted:
if s.is_subscriber():
if not starting_from_s or (starting_from_s and s.user.email > starting_from_s):
yield s, True
else:
if not starting_from_ns or (starting_from_ns and s.user.email > starting_from_ns):
receivers2.append(s.id)
for sus_id in receivers2:
yield Subscriber.objects.get(id=sus_id), False
# create global counters
counters = []
# define the function to be executed by each thread
def send(func):
# Authenticate to smtp (if any auth needed) and send all emails
smtp = smtp_connect()
subscriber_sent, user_sent, subscriber_refused, user_refused = 0, 0, 0, 0
site_url = '%s://%s' % (settings.URL_SCHEME, settings.SITE_DOMAIN)
# iterate and send
while True:
try:
s, is_subscriber = func()
hashed_id = hashids.encode(int(s.id))
headers['List-Unsubscribe'] = '%s/usuarios/nlunsubscribe/c/%s/%s/' % (
site_url, category.slug, hashed_id)
msg = Message(html=render_to_string(
'category/newsletter/%s.html' % category.slug, {
'cover_article': cover_article, 'category': category, 'opinion_article': opinion_article,
'datos_article': datos_article, 'site_url': site_url, 'articles': top_articles,
'hashed_id': hashed_id, 'ga_property_id': getattr(settings, 'GA_PROPERTY_ID', None),
'subscriber_id': s.id, 'is_subscriber': is_subscriber,
'cover_article_section': cover_article_section,
}),
mail_to=(s.name, s.user.email), subject=remove_markup(cover_article.headline),
mail_from=(u'la diaria ' + category.name, EMAIL_FROM_ADDR), headers=headers)
# attach ads if any
for f_ad in f_ads:
f_ad_basename = os.path.basename(f_ad)
msg.attach(filename=f_ad_basename, data=open(f_ad, "rb"))
# send using smtp to receive bounces in another mailbox
try:
if not no_deliver:
smtp.sendmail(settings.NOTIFICATIONS_FROM_MX, [s.user.email], msg.as_string())
log.info("%s Email %s to %ssubscriber %s\t%s" % (
today, 'simulated' if no_deliver else 'sent', '' if is_subscriber else 'non-',
s.id, s.user.email))
if is_subscriber:
subscriber_sent += 1
else:
user_sent += 1
except smtplib.SMTPRecipientsRefused:
log.warning("%s Email refused for %ssubscriber %s\t%s" % (
today, '' if is_subscriber else 'non-', s.id, s.user.email))
if is_subscriber:
subscriber_refused += 1
else:
user_refused += 1
except smtplib.SMTPServerDisconnected:
log.error("MTA down, not sending - %s" % s.user.email)
log.info("Trying to reconnect for the next delivery...")
smtp = smtp_connect()
except StopIteration:
# append data processed to global results and quit.
# this can be done because list.append is threadsafe.
counters.append((subscriber_sent, user_sent, subscriber_refused, user_refused))
break
try:
smtp.quit()
except smtplib.SMTPServerDisconnected:
pass
# create threads
subscribers_iter = subscribers()
threads = [Thread(target=send, args=(subscribers_iter.next, )) for i in range(nthreads)]
# start threads
for t in threads:
t.start()
# wait for threads to finish
for t in threads:
t.join()
# sum total counters
subscriber_sent, user_sent, subscriber_refused, user_refused = 0, 0, 0, 0
for counter0, counter1, counter2, counter3 in counters:
subscriber_sent += counter0
user_sent += counter1
subscriber_refused += counter2
user_refused += counter3
# update log stats counters only if subscriber_ids not given
if not subscriber_ids:
try:
# A transaction is needed because autocommit in django is broken in concurrent management processes
cursor = connection.cursor()
cursor.execute('BEGIN')
cursor.execute("""INSERT INTO dashboard_newsletterdelivery(
delivery_date,newsletter_name,user_sent,subscriber_sent,user_refused,subscriber_refused)
VALUES('%s','%s',%d,%d,%d,%d)""" % (
today, category.slug, user_sent, subscriber_sent, user_refused, subscriber_refused))
cursor.execute('COMMIT')
except IntegrityError:
nl_delivery = NewsletterDelivery.objects.get(delivery_date=today, newsletter_name=category.slug)
nl_delivery.user_sent = (nl_delivery.user_sent or 0) + user_sent
nl_delivery.subscriber_sent = (nl_delivery.subscriber_sent or 0) + subscriber_sent
nl_delivery.user_refused = (nl_delivery.user_refused or 0) + user_refused
nl_delivery.subscriber_refused = (nl_delivery.subscriber_refused or 0) + subscriber_refused
nl_delivery.save()
except Exception as e:
log.error(u'%s ERROR: Delivery stats not updated: %s' % (today, e))
log.info((u'%s %s stats: user_sent: %d, subscriber_sent: %s, user_refused: %d, subscriber_refused: %d') % (
today, 'Simulation' if no_deliver else 'Delivery', user_sent, subscriber_sent, user_refused,
subscriber_refused))