def category_detail(request, slug): category, inner_sections = get_object_or_404(Category, slug=slug), [] question_list, questions_topic = [], None category_home = get_object_or_404(CategoryHome, category=category) try: featured_section1 = category.section_set.get(home_order=1) except (Section.DoesNotExist, Section.MultipleObjectsReturned): featured_section1 = None try: featured_section2 = category.section_set.get(home_order=2) except (Section.DoesNotExist, Section.MultipleObjectsReturned): featured_section2 = None try: featured_section3 = category.section_set.get(home_order=3) except (Section.DoesNotExist, Section.MultipleObjectsReturned): featured_section3 = None for inner_section_slug in getattr(settings, 'CORE_CATEGORY_INNER_SECTIONS', {}).get(slug, ()): try: inner_sections.append( category.section_set.get(slug=inner_section_slug)) except Section.DoesNotExist: pass if slug in getattr(settings, 'CORE_CATEGORIES_ENABLE_QUESTIONS', ()): question_list = Question.published.filter(topic__slug=slug) try: questions_topic = Topic.objects.get(slug=slug) except Topic.DoesNotExist: pass return render( request, '%s/%s.html' % (getattr(settings, 'CORE_CATEGORIES_TEMPLATE_DIR', 'core/templates/category'), category.slug if category.slug in getattr(settings, 'CORE_CATEGORIES_CUSTOM_TEMPLATES', ()) else 'detail'), { 'category': category, 'cover_article': category_home.cover(), 'destacados': category_home.non_cover_articles(), 'is_portada': True, 'featured_section1': featured_section1, 'featured_section2': featured_section2, 'featured_section3': featured_section3, 'inner_sections': inner_sections, 'edition': get_latest_edition(), 'question_list': question_list, 'questions_topic': questions_topic, 'big_photo': category.full_width_cover_image, }, )
class HomeArticleViewSet(viewsets.ModelViewSet): try: edition = get_current_edition() or get_latest_edition() except Exception: edition = None pk_list = [a.id for a in edition.top_articles] if edition else [] clauses = ' '.join(['WHEN id=%s THEN %s' % (pk, i) for i, pk in enumerate(pk_list)]) ordering = 'CASE %s END' % clauses queryset = Article.objects.filter(id__in=pk_list).extra(select={'ordering': ordering}, order_by=('ordering', )) serializer_class = ArticleSerializer http_method_names = ['get', 'head'] filter_fields = ('headline', 'sections')
def newsletter_preview(request, slug): # allow only staff members or requests from localhost if not (request.user.is_staff or request.META.get('REMOTE_ADDR') in ( socket.gethostbyname('localhost'), socket.gethostbyname(socket.gethostname()))): return HttpResponseForbidden() # removed or changed categories redirects by settings if slug in getattr(settings, 'CORE_CATEGORY_REDIRECT', {}): return HttpResponsePermanentRedirect( reverse('category-nl-preview', args=(settings.CORE_CATEGORY_REDIRECT[slug], ))) category = get_object_or_404(Category, slug=slug) category_home = get_object_or_404(Home, category=category) cat_modules = category_home.modules.all() try: cover_article = category_home.cover cover_article_section = cover_article.publication_section() top_articles = [ (a, a.publication_section(), 0) for a in (cat_modules[0].articles_as_list if cat_modules else [])] opinion_article = None nl_featured = Article.objects.filter(id=settings.NEWSLETTER_FEATURED_ARTICLE) if \ getattr(settings, 'NEWSLETTER_FEATURED_ARTICLE', False) else \ get_latest_edition().newsletter_featured_articles() if nl_featured: opinion_article = nl_featured[0] # datos_article for elecciones try: datos_article = category.section_set.get(slug='datos-elecciones-2019').latest_article()[0] except (Section.DoesNotExist, Section.MultipleObjectsReturned, IndexError): datos_article = None try: hashed_id = hashids.encode(int(request.user.subscriber.id)) except AttributeError: hashed_id = hashids.encode(0) return 'newsletter/%s.html' % slug, { 'cover_article': cover_article, 'category': category, 'cover_article_section': cover_article_section, 'articles': top_articles, 'hashed_id': hashed_id, 'opinion_article': opinion_article, 'site_url': '%s://%s' % (settings.URL_SCHEME, settings.SITE_DOMAIN), 'datos_article': datos_article if datos_article and datos_article.date_published >= datetime.now() - timedelta(1) else None} except Exception as e: return HttpResponseServerError(u'ERROR: %s' % e)
def category_detail(request, slug): # deporte custom redirect if slug == u'deporte': return redirect(Publication.objects.get(slug=u'garra').get_absolute_url(), permanent=True) category, inner_sections = get_object_or_404(Category, slug=slug), [] question_list, questions_topic = [], None category_home = get_object_or_404(Home, category=category) cat_modules = category_home.modules.all() top_articles = cat_modules[0].articles_as_list if cat_modules else [] try: featured_section1 = category.section_set.get(home_order=1) except (Section.DoesNotExist, Section.MultipleObjectsReturned): featured_section1 = None try: featured_section2 = category.section_set.get(home_order=2) except (Section.DoesNotExist, Section.MultipleObjectsReturned): featured_section2 = None try: featured_section3 = category.section_set.get(home_order=3) except (Section.DoesNotExist, Section.MultipleObjectsReturned): featured_section3 = None # custom inner sections if slug == u'elecciones': # example dropdown menu (elecciones ir disabled now by redirect) inner_sections = category.section_set.filter( Q(slug__startswith='partido-') | Q(slug__in=( 'frente-amplio', 'unidad-popular', 'cabildo-abierto', 'otros'))) if slug == u'coronavirus': question_list = Question.published.filter(topic__slug='coronavirus') try: questions_topic = Topic.objects.get(slug="coronavirus") except Topic.DoesNotExist: pass return render_to_response( '%s/%s.html' % ( getattr(settings, 'CORE_CATEGORIES_TEMPLATE_DIR', 'core/templates/category'), category.slug if category.slug in getattr(settings, 'CORE_CATEGORIES_CUSTOM_TEMPLATES', ()) else 'detail' ), { 'category': category, 'cover_article': category_home.cover, 'destacados': top_articles, 'is_portada': True, 'featured_section1': featured_section1, 'featured_section2': featured_section2, 'featured_section3': featured_section3, 'inner_sections': inner_sections, 'edition': get_latest_edition(), 'question_list': question_list, 'questions_topic': questions_topic, 'big_photo': category.full_width_cover_image}, context_instance=RequestContext(request))
def build_and_send( category, no_deliver, offline, export_subscribers, export_context, site_id, starting_from_s, starting_from_ns, partitions, mod, subscriber_ids, ): site = Site.objects.get( id=site_id) if site_id else Site.objects.get_current() category_slug, export_only = category if offline else category.slug, export_subscribers or export_context context = {'category': category, 'newsletter_campaign': category_slug} export_ctx = {'newsletter_campaign': category_slug} try: offline_ctx_file = join(settings.SENDNEWSLETTER_EXPORT_DIR, '%s_ctx.json' % category_slug) offline_csv_file = join(settings.SENDNEWSLETTER_EXPORT_DIR, '%s_subscribers.csv' % category_slug) if offline: if starting_from_s or starting_from_ns: log.error( '--starting-from* options for offline usage not yet implemented' ) return context = json.loads(open(offline_ctx_file).read()) # de-serialize dates dp_cover = datetime.strptime( context['cover_article']['date_published'], '%Y-%m-%d').date() context['cover_article']['date_published'] = dp_cover featured_article = context['featured_article'] if featured_article: dp_featured = datetime.strptime( featured_article['date_published'], '%Y-%m-%d').date() context['featured_article']['date_published'] = dp_featured dp_articles = [] for a, a_section in context['articles']: dp_article = datetime.strptime(a['date_published'], '%Y-%m-%d').date() a['date_published'] = dp_article dp_articles.append((a, a_section)) context['articles'] = dp_articles elif not export_subscribers or export_context: category_nl = CategoryNewsletter.objects.get( category=category, valid_until__gt=datetime.now()) cover_article, featured_article = category_nl.cover( ), category_nl.featured_article() featured_article_section = featured_article.publication_section( ) if featured_article else None if export_context: export_ctx.update({ 'articles': [(a.nl_serialize(position == 0), { 'name': section.name, 'slug': section.slug }) for position, (a, section) in enumerate( [(a, a.publication_section()) for a in category_nl.non_cover_articles()])], 'featured_article_section': featured_article_section.name if featured_article_section else None, # TODO: check if cover_article_section and featured_articles entries are needed }) else: context.update({ 'cover_article_section': cover_article.publication_section().name if cover_article else None, 'articles': [(a, a.publication_section()) for a in category_nl.non_cover_articles()], 'featured_article_section': featured_article_section, 'featured_articles': [(a, a.publication_section()) for a in category_nl.non_cover_featured_articles()], }) except CategoryNewsletter.DoesNotExist: if not (offline or export_subscribers) or export_context: cover_article = category.home.cover() cover_article_section = cover_article.publication_section( ) if cover_article else None top_articles = [(a, a.publication_section()) for a in category.home.non_cover_articles()] listonly_section = getattr( settings, 'CORE_CATEGORY_NEWSLETTER_LISTONLY_SECTIONS', {}).get(category_slug) if listonly_section: top_articles = [ t for t in top_articles if t[1].slug == listonly_section ] if cover_article_section.slug != listonly_section: cover_article = top_articles.pop( 0)[0] if top_articles else None cover_article_section = cover_article.publication_section( ) if cover_article else None featured_article_id = getattr(settings, 'NEWSLETTER_FEATURED_ARTICLE', False) nl_featured = Article.objects.filter( id=featured_article_id ) if featured_article_id else get_latest_edition( ).newsletter_featured_articles() opinion_article = nl_featured[0] if nl_featured else None # featured_article (a featured section in the category) try: featured_section, days_ago = settings.CORE_CATEGORY_NEWSLETTER_FEATURED_SECTIONS[ category_slug] featured_article = category.section_set.get( slug=featured_section).latest_article()[0] assert (featured_article.date_published >= datetime.now() - timedelta(days_ago)) except (KeyError, Section.DoesNotExist, Section.MultipleObjectsReturned, IndexError, AssertionError): featured_article = None if export_context: export_ctx['articles'] = [(t[0].nl_serialize(position == 0), { 'name': t[1].name, 'slug': t[1].slug }) for position, t in enumerate(top_articles)] else: context.update({ 'opinion_article': opinion_article, 'cover_article_section': cover_article_section, 'articles': top_articles, }) # any custom attached files # TODO: make this a feature in the admin using adzone also make it path-setting instead of absolute # f_ads = ['/srv/ldsocial/portal/media/document.pdf'] f_ads = [] if not offline: receivers = Subscriber.objects.filter(user__is_active=True).exclude( user__email='') if subscriber_ids: receivers = receivers.filter(id__in=subscriber_ids) else: receivers = receivers.filter( category_newsletters__slug=category_slug).exclude( user__email__in=blacklisted) # if both "starting_from" we can filter now with the minimum if starting_from_s and starting_from_ns: receivers = receivers.filter( user__email__gt=min(starting_from_s, starting_from_ns)) if partitions is not None and mod is not None: receivers = receivers.extra(where=[ 'MOD(%s.id,%d)=%d' % (Subscriber._meta.db_table, partitions, mod) ]) if offline: custom_subject = context['custom_subject'] email_subject = context['email_subject'] email_from = context['email_from'] site_url = context['site_url'] list_id = context['list_id'] ga_property_id = context['ga_property_id'] r = reader(open(offline_csv_file)) if subscriber_ids: subscribers_iter = subscribers_nl_iter_filter( r, lambda row: int(row[0]) in subscriber_ids) elif partitions is not None and mod is not None: subscribers_iter = subscribers_nl_iter_filter( r, lambda row: int(row[0]) % partitions == mod) else: subscribers_iter = r elif not export_subscribers or export_context: site_url = '%s://%s' % (settings.URL_SCHEME, settings.SITE_DOMAIN) list_id = '%s <%s.%s>' % (category_slug, __name__, settings.SITE_DOMAIN) ga_property_id = getattr(settings, 'GA_PROPERTY_ID', None) custom_subject = category.newsletter_automatic_subject is False and category.newsletter_subject email_subject = custom_subject or ( getattr(settings, 'CORE_CATEGORY_NL_SUBJECT_PREFIX', {}).get( category_slug, u'') + remove_markup(cover_article.headline)) email_from = ( site.name if category_slug in getattr( settings, 'CORE_CATEGORY_NL_FROM_NAME_SITEONLY', ()) else (u'%s %s' % (site.name, category.name)), settings.NOTIFICATIONS_FROM_ADDR1, ) translation.activate(settings.LANGUAGE_CODE) if not export_subscribers or export_context: common_ctx = { 'site_url': site_url, 'ga_property_id': ga_property_id, 'custom_subject': custom_subject } if export_only: if export_context: export_ctx.update(common_ctx) export_ctx.update({ 'email_subject': email_subject, 'email_from': email_from, 'list_id': list_id, 'cover_article': cover_article.nl_serialize(True), 'featured_article': featured_article.nl_serialize(True) if featured_article else None, }) open(offline_ctx_file, 'w').write(json.dumps(export_ctx)) if export_subscribers: export_subscribers_writer = writer(open(offline_csv_file, 'w')) else: return elif not offline: context.update(common_ctx) context.update({ 'cover_article': cover_article, 'featured_article': featured_article }) if not offline: subscribers_iter = subscribers_nl_iter(receivers, starting_from_s, starting_from_ns) # Connect to the SMTP server and send all emails try: smtp = None if no_deliver or export_only else smtp_connect() except error: log.error("MTA down, '%s %s' was used for partitions and mod" % (partitions, mod)) return subscriber_sent, user_sent, subscriber_refused, user_refused = 0, 0, 0, 0 retry_last_delivery, s_id, is_subscriber = False, None, None email_template = Engine.get_default().get_template( '%s/newsletter/%s.html' % (settings.CORE_CATEGORIES_TEMPLATE_DIR, category_slug)) while True: try: if not retry_last_delivery: if offline: s_id, s_name, s_user_email, hashed_id, is_subscriber, is_subscriber_any, is_subscriber_default = ( subscribers_iter.next()) is_subscriber = eval(is_subscriber) is_subscriber_any = eval(is_subscriber_any) is_subscriber_default = eval(is_subscriber_default) else: s, is_subscriber = subscribers_iter.next() s_id, s_name, s_user_email = s.id, s.name, s.user.email hashed_id = hashids.encode(int(s_id)) is_subscriber_any = s.is_subscriber_any() is_subscriber_default = s.is_subscriber( settings.DEFAULT_PUB) if export_subscribers: export_subscribers_writer.writerow([ s_id, s_name, s_user_email, hashed_id, is_subscriber, is_subscriber_any, is_subscriber_default ]) elif not export_context: headers = {'List-ID': list_id} unsubscribe_url = '%s/usuarios/nlunsubscribe/c/%s/%s/?utm_source=newsletter&utm_medium=email' \ '&utm_campaign=%s&utm_content=unsubscribe' % (site_url, category_slug, hashed_id, category_slug) headers['List-Unsubscribe'] = headers[ 'List-Unsubscribe-Post'] = '<%s>' % unsubscribe_url context.update({ 'hashed_id': hashed_id, 'unsubscribe_url': unsubscribe_url, 'subscriber_id': s_id, 'is_subscriber': is_subscriber, 'is_subscriber_any': is_subscriber_any, 'is_subscriber_default': is_subscriber_default, }) msg = Message( html=email_template.render(Context(context)), mail_to=(s_name, s_user_email), subject=email_subject, mail_from=email_from, headers=headers, ) # attach ads if any for f_ad in f_ads: f_ad_basename = basename(f_ad) msg.attach(filename=f_ad_basename, data=open(f_ad, "rb")) # send using smtp to receive bounces in another mailbox try: if not no_deliver: smtp.sendmail(settings.NOTIFICATIONS_FROM_MX, [s_user_email], msg.as_string()) log.info( "Email %s to %ssubscriber %s\t%s" % ('simulated' if no_deliver else 'sent', '' if is_subscriber else 'non-', s_id, s_user_email)) if is_subscriber: subscriber_sent += 1 else: user_sent += 1 except smtplib.SMTPRecipientsRefused: log.warning( "Email refused for %ssubscriber %s\t%s" % ('' if is_subscriber else 'non-', s_id, s_user_email)) if is_subscriber: subscriber_refused += 1 else: user_refused += 1 except smtplib.SMTPServerDisconnected: # Retries are made only once per iteration to avoid infinite loop if MTA got down at all retry_last_delivery = not retry_last_delivery log_message = "MTA down, email to %s not sent. Reconnecting " % s_user_email if retry_last_delivery: log.warning(log_message + "to retry ...") else: log.error(log_message + "for the next delivery ...") try: smtp = smtp_connect() except error: log.warning('MTA reconnect failed') else: retry_last_delivery = False except (ProgrammingError, OperationalError, StopIteration) as exc: # the connection to databse can be killed, if that is the case print useful log to continue if isinstance(exc, (ProgrammingError, OperationalError)): log.error( 'DB connection error, (%s, %s, %s, %s) was the last delivery attempt' % (s_user_email if s_id else None, is_subscriber, partitions, mod)) break if not export_only: if not no_deliver: try: smtp.quit() except smtplib.SMTPServerDisconnected: pass # update log stats counters only if subscriber_ids not given if not subscriber_ids: try: # close connections because reach this point can take several minutes close_old_connections() # A transaction is needed because autocommit in django is broken in concurrent management processes cursor = connection.cursor() cursor.execute('BEGIN') cursor.execute(""" INSERT INTO dashboard_newsletterdelivery( delivery_date,newsletter_name,user_sent,subscriber_sent,user_refused,subscriber_refused ) VALUES('%s','%s',%d,%d,%d,%d) """ % (today, category_slug, user_sent, subscriber_sent, user_refused, subscriber_refused)) cursor.execute('COMMIT') except IntegrityError: nl_delivery = NewsletterDelivery.objects.get( delivery_date=today, newsletter_name=category_slug) nl_delivery.user_sent = (nl_delivery.user_sent or 0) + user_sent nl_delivery.subscriber_sent = (nl_delivery.subscriber_sent or 0) + subscriber_sent nl_delivery.user_refused = (nl_delivery.user_refused or 0) + user_refused nl_delivery.subscriber_refused = ( nl_delivery.subscriber_refused or 0) + subscriber_refused nl_delivery.save() except Exception as e: log.error(u'Delivery stats not updated: %s' % e) log.info( u'%s stats: user_sent: %d, subscriber_sent: %s, user_refused: %d, subscriber_refused: %d' % ( 'Simulation' if no_deliver else 'Delivery', user_sent, subscriber_sent, user_refused, subscriber_refused, ))
def newsletter_preview(request, slug): # allow only staff members or requests from localhost if not (request.user.is_staff or request.META.get('REMOTE_ADDR') in (socket.gethostbyname('localhost'), socket.gethostbyname(socket.gethostname()))): return HttpResponseForbidden() # removed or changed categories redirects by settings category_redirections = getattr(settings, 'CORE_CATEGORY_REDIRECT', {}) if slug in category_redirections: redirect_slug = category_redirections[slug] if redirect_slug: return HttpResponsePermanentRedirect( reverse('category-nl-preview', args=(settings.CORE_CATEGORY_REDIRECT[slug], ))) site = Site.objects.get_current() category = get_object_or_404(Category, slug=slug) try: context = {'category': category, 'newsletter_campaign': category.slug} try: category_nl = CategoryNewsletter.objects.get( category=category, valid_until__gt=datetime.now()) cover_article, featured_article = category_nl.cover( ), category_nl.featured_article() context.update({ 'cover_article_section': cover_article.publication_section() if cover_article else None, 'articles': [(a, a.publication_section()) for a in category_nl.non_cover_articles()], 'featured_article_section': featured_article.publication_section() if featured_article else None, 'featured_articles': [(a, a.publication_section()) for a in category_nl.non_cover_featured_articles()], }) except CategoryNewsletter.DoesNotExist: category_home = get_object_or_404(CategoryHome, category=category) cover_article = category_home.cover() cover_article_section = cover_article.publication_section( ) if cover_article else None top_articles = [(a, a.publication_section()) for a in category_home.non_cover_articles()] listonly_section = getattr( settings, 'CORE_CATEGORY_NEWSLETTER_LISTONLY_SECTIONS', {}).get(category.slug) if listonly_section: top_articles = [ t for t in top_articles if t[1].slug == listonly_section ] if cover_article_section.slug != listonly_section: cover_article = top_articles.pop( 0)[0] if top_articles else None cover_article_section = cover_article.publication_section( ) if cover_article else None featured_article_id = getattr(settings, 'NEWSLETTER_FEATURED_ARTICLE', False) nl_featured = Article.objects.filter( id=featured_article_id ) if featured_article_id else get_latest_edition( ).newsletter_featured_articles() opinion_article = nl_featured[0] if nl_featured else None # featured_article (a featured section in the category) try: featured_section, days_ago = settings.CORE_CATEGORY_NEWSLETTER_FEATURED_SECTIONS[ category.slug] featured_article = category.section_set.get( slug=featured_section).latest_article()[0] assert (featured_article.date_published >= datetime.now() - timedelta(days_ago)) except (KeyError, Section.DoesNotExist, Section.MultipleObjectsReturned, IndexError, AssertionError): featured_article = None context.update({ 'opinion_article': opinion_article, 'cover_article_section': cover_article_section, 'articles': top_articles, }) try: hashed_id = hashids.encode(int(request.user.subscriber.id)) except AttributeError: hashed_id = hashids.encode(0) custom_subject = category.newsletter_automatic_subject is False and category.newsletter_subject email_subject = custom_subject or ( getattr(settings, 'CORE_CATEGORY_NL_SUBJECT_PREFIX', {}).get( category.slug, u'') + remove_markup(cover_article.headline)) email_from = u'%s <%s>' % ( site.name if category.slug in getattr( settings, 'CORE_CATEGORY_NL_FROM_NAME_SITEONLY', ()) else (u'%s %s' % (site.name, category.name)), settings.NOTIFICATIONS_FROM_ADDR1, ) headers = {'From': email_from, 'Subject': email_subject} site_url = '%s://%s' % (settings.URL_SCHEME, settings.SITE_DOMAIN) unsubscribe_url = '%s/usuarios/nlunsubscribe/c/%s/%s/?utm_source=newsletter&utm_medium=email' \ '&utm_campaign=%s&utm_content=unsubscribe' % (site_url, category.slug, hashed_id, category.slug) headers['List-Unsubscribe'] = headers[ 'List-Unsubscribe-Post'] = '<%s>' % unsubscribe_url context.update({ 'site_url': site_url, 'hashed_id': hashed_id, 'unsubscribe_url': unsubscribe_url, 'custom_subject': custom_subject, 'headers_preview': headers, 'cover_article': cover_article, 'featured_article': featured_article, }) # override is_subscriber_any and _default only to "False" for suffix in ('any', 'default'): is_subscriber_var = 'is_subscriber_' + suffix is_subscriber_val = request.GET.get(is_subscriber_var) if is_subscriber_val and is_subscriber_val.lower() in (u'false', u'0'): context[is_subscriber_var] = False return render( request, '%s/newsletter/%s.html' % (settings.CORE_CATEGORIES_TEMPLATE_DIR, slug), context) except Exception as e: if settings.DEBUG: exc_type, exc_value, exc_traceback = sys.exc_info() print(exc_type) print(exc_value) print(traceback.extract_tb(exc_traceback)) return HttpResponseServerError(u'ERROR: %s' % e)
def build_and_send(category, nthreads, no_deliver, starting_from_s, starting_from_ns, subscriber_ids, even, odd): headers = {} category_home = Home.objects.get(category=category) cat_modules = category_home.modules.all() cover_article = category_home.cover cover_article_section = cover_article.publication_section() top_articles = [(a, a.publication_section(), 0) for a in (cat_modules[0].articles_as_list if cat_modules else [])] opinion_article = None nl_featured = Article.objects.filter(id=settings.NEWSLETTER_FEATURED_ARTICLE) if \ getattr(settings, 'NEWSLETTER_FEATURED_ARTICLE', False) else \ get_latest_edition().newsletter_featured_articles() if nl_featured: opinion_article = nl_featured[0] # datos_article (a featured section in the category) try: datos_article = category.section_set.get(slug='datos').latest_article()[0] assert (datos_article.date_published >= datetime.now() - timedelta(1)) except (Section.DoesNotExist, Section.MultipleObjectsReturned, IndexError, AssertionError): datos_article = None # any custom attached files # TODO: make this a feature in the admin using adzone also make it path-setting instead of absolute # f_ads = ['/srv/ldsocial/portal/media/document.pdf'] f_ads = [] filter_args = {'user__is_active': True} if subscriber_ids: filter_args['id__in'] = subscriber_ids else: filter_args['category_newsletters__slug'] = category.slug if even or odd: filter_args['id__iregex'] = r'^\d*[%s]$' % ('02468' if even else '13579') blacklisted = set([row[0] for row in csv.reader(open(settings.CORE_NEWSLETTER_BLACKLIST))]) # iterate over receivers and yield the subscribers first, saving the # not subscribers ids in a temporal list an then yield them also @threadsafe_generator def subscribers(): receivers2 = [] for s in Subscriber.objects.filter(**filter_args).distinct().order_by('user__email').iterator(): if s.user and s.user.email and s.user.email not in blacklisted: if s.is_subscriber(): if not starting_from_s or (starting_from_s and s.user.email > starting_from_s): yield s, True else: if not starting_from_ns or (starting_from_ns and s.user.email > starting_from_ns): receivers2.append(s.id) for sus_id in receivers2: yield Subscriber.objects.get(id=sus_id), False # create global counters counters = [] # define the function to be executed by each thread def send(func): # Authenticate to smtp (if any auth needed) and send all emails smtp = smtp_connect() subscriber_sent, user_sent, subscriber_refused, user_refused = 0, 0, 0, 0 site_url = '%s://%s' % (settings.URL_SCHEME, settings.SITE_DOMAIN) # iterate and send while True: try: s, is_subscriber = func() hashed_id = hashids.encode(int(s.id)) headers['List-Unsubscribe'] = '%s/usuarios/nlunsubscribe/c/%s/%s/' % ( site_url, category.slug, hashed_id) msg = Message(html=render_to_string( 'category/newsletter/%s.html' % category.slug, { 'cover_article': cover_article, 'category': category, 'opinion_article': opinion_article, 'datos_article': datos_article, 'site_url': site_url, 'articles': top_articles, 'hashed_id': hashed_id, 'ga_property_id': getattr(settings, 'GA_PROPERTY_ID', None), 'subscriber_id': s.id, 'is_subscriber': is_subscriber, 'cover_article_section': cover_article_section, }), mail_to=(s.name, s.user.email), subject=remove_markup(cover_article.headline), mail_from=(u'la diaria ' + category.name, EMAIL_FROM_ADDR), headers=headers) # attach ads if any for f_ad in f_ads: f_ad_basename = os.path.basename(f_ad) msg.attach(filename=f_ad_basename, data=open(f_ad, "rb")) # send using smtp to receive bounces in another mailbox try: if not no_deliver: smtp.sendmail(settings.NOTIFICATIONS_FROM_MX, [s.user.email], msg.as_string()) log.info("%s Email %s to %ssubscriber %s\t%s" % ( today, 'simulated' if no_deliver else 'sent', '' if is_subscriber else 'non-', s.id, s.user.email)) if is_subscriber: subscriber_sent += 1 else: user_sent += 1 except smtplib.SMTPRecipientsRefused: log.warning("%s Email refused for %ssubscriber %s\t%s" % ( today, '' if is_subscriber else 'non-', s.id, s.user.email)) if is_subscriber: subscriber_refused += 1 else: user_refused += 1 except smtplib.SMTPServerDisconnected: log.error("MTA down, not sending - %s" % s.user.email) log.info("Trying to reconnect for the next delivery...") smtp = smtp_connect() except StopIteration: # append data processed to global results and quit. # this can be done because list.append is threadsafe. counters.append((subscriber_sent, user_sent, subscriber_refused, user_refused)) break try: smtp.quit() except smtplib.SMTPServerDisconnected: pass # create threads subscribers_iter = subscribers() threads = [Thread(target=send, args=(subscribers_iter.next, )) for i in range(nthreads)] # start threads for t in threads: t.start() # wait for threads to finish for t in threads: t.join() # sum total counters subscriber_sent, user_sent, subscriber_refused, user_refused = 0, 0, 0, 0 for counter0, counter1, counter2, counter3 in counters: subscriber_sent += counter0 user_sent += counter1 subscriber_refused += counter2 user_refused += counter3 # update log stats counters only if subscriber_ids not given if not subscriber_ids: try: # A transaction is needed because autocommit in django is broken in concurrent management processes cursor = connection.cursor() cursor.execute('BEGIN') cursor.execute("""INSERT INTO dashboard_newsletterdelivery( delivery_date,newsletter_name,user_sent,subscriber_sent,user_refused,subscriber_refused) VALUES('%s','%s',%d,%d,%d,%d)""" % ( today, category.slug, user_sent, subscriber_sent, user_refused, subscriber_refused)) cursor.execute('COMMIT') except IntegrityError: nl_delivery = NewsletterDelivery.objects.get(delivery_date=today, newsletter_name=category.slug) nl_delivery.user_sent = (nl_delivery.user_sent or 0) + user_sent nl_delivery.subscriber_sent = (nl_delivery.subscriber_sent or 0) + subscriber_sent nl_delivery.user_refused = (nl_delivery.user_refused or 0) + user_refused nl_delivery.subscriber_refused = (nl_delivery.subscriber_refused or 0) + subscriber_refused nl_delivery.save() except Exception as e: log.error(u'%s ERROR: Delivery stats not updated: %s' % (today, e)) log.info((u'%s %s stats: user_sent: %d, subscriber_sent: %s, user_refused: %d, subscriber_refused: %d') % ( today, 'Simulation' if no_deliver else 'Delivery', user_sent, subscriber_sent, user_refused, subscriber_refused))