def send_emails(self, recipients): """Send the emails using the templates and contexts requested.""" email_subject = self.options['subject'] email_sender = self.options['sender'] txt_template = loader.get_template(self.options['template']) c = Context(ast.literal_eval(self.options['context'])) email_txt = txt_template.render(c) emails_sent_count = 0 messages = [] for recipient in recipients: messages.append(( email_subject, email_txt, email_sender, [recipient], )) emails_sent_count += 1 if not self.options['simulate']: send_mass_mail(messages, fail_silently=False) tally_stat('pr.emails.sent', inc=emails_sent_count) logger.info("Sent %s pr emails." % emails_sent_count) else: sys.stdout.write('Simulation mode. Imagine that we just sent %s emails!\n' % emails_sent_count)
def serve_static_file(request, file_path=''): """Sends a static file to a user. This serves up the static case files such as the PDFs in a way that can be blocked from search engines if necessary. We do four things: - Look up the document associated with the filepath - Check if it's blocked - If blocked, we set the x-robots-tag HTTP header - Serve up the file using Apache2's xsendfile """ doc = get_object_or_404(Document, local_path=file_path) file_name = file_path.split('/')[-1] file_loc = os.path.join(settings.MEDIA_ROOT, file_path.encode('utf-8')) try: mimetype = magic.from_file(file_loc, mime=True) except IOError: raise Http404 response = HttpResponse() if doc.blocked: response['X-Robots-Tag'] = 'noindex, noodp, noarchive, noimageindex' response['X-Sendfile'] = os.path.join(settings.MEDIA_ROOT, file_path.encode('utf-8')) response['Content-Disposition'] = 'attachment; filename="%s"' % file_name.encode('utf-8') response['Content-Type'] = mimetype if not is_bot(request): tally_stat('case_page.static_file.served') return response
def send_emails(self, recipients): """Send the emails using the templates and contexts requested.""" email_subject = self.options['subject'] email_sender = self.options['sender'] txt_template = loader.get_template(self.options['template']) c = Context(ast.literal_eval(self.options['context'])) email_txt = txt_template.render(c) emails_sent_count = 0 messages = [] for recipient in recipients: messages.append(( email_subject, email_txt, email_sender, [recipient], )) emails_sent_count += 1 if not self.options['simulate']: send_mass_mail(messages, fail_silently=False) tally_stat('pr.emails.sent', inc=emails_sent_count) logger.info("Sent %s pr emails." % emails_sent_count) else: sys.stdout.write( 'Simulation mode. Imagine that we just sent %s emails!\n' % emails_sent_count)
def serve_pagerank_file(request): """Find the pagerank file by interrogating Solr, then serve it up.""" file_loc = settings.BULK_DATA_DIR + "external_pagerank" file_name = file_loc.split('/')[-1] try: mimetype = magic.from_file(file_loc, mime=True) except IOError: raise Http404('Unable to locate external_pagerank file in %s' % settings.BULK_DATA_DIR) response = HttpResponse() response['X-Sendfile'] = os.path.join(file_loc) response['Content-Disposition'] = 'attachment; filename="%s"' % file_name.encode('utf-8') response['Content-Type'] = mimetype tally_stat('bulk_data.pagerank.served') return response
def serve_static_file(request, file_path=''): """Sends a static file to a user. This serves up the static case files such as the PDFs in a way that can be blocked from search engines if necessary. We do four things: - Look up the document or audio file associated with the filepath - Check if it's blocked - If blocked, we set the x-robots-tag HTTP header - Serve up the file using Apache2's xsendfile """ response = HttpResponse() file_loc = os.path.join(settings.MEDIA_ROOT, file_path.encode('utf-8')) if file_path.startswith('mp3'): item = get_object_or_404(Audio, local_path_mp3=file_path) mimetype = 'audio/mpeg' else: item = get_object_or_404(Document, local_path=file_path) try: mimetype = magic.from_file(file_loc, mime=True) except IOError: raise Http404 if item.blocked: response['X-Robots-Tag'] = 'noindex, noodp, noarchive, noimageindex' if settings.DEVELOPMENT: # X-Sendfile will only confuse you in a dev env. response.content = open(file_loc, 'r').read() else: response['X-Sendfile'] = file_loc file_name = file_path.split('/')[-1] response['Content-Disposition'] = 'attachment; filename="%s"' % \ file_name.encode('utf-8') response['Content-Type'] = mimetype if not is_bot(request): tally_stat('case_page.static_file.served') return response
def register(request): """allow only an anonymous user to register""" redirect_to = request.REQUEST.get('next', '') if 'sign-in' in redirect_to: # thus, we don't redirect people back to the sign-in form redirect_to = '' # security checks: # Light security check -- make sure redirect_to isn't garbage. if not redirect_to or ' ' in redirect_to: redirect_to = settings.LOGIN_REDIRECT_URL # Heavier security check -- redirects to http://example.com should # not be allowed, but things like /view/?param=http://example.com # should be allowed. This regex checks if there is a '//' *before* a # question mark. elif '//' in redirect_to and re.match(r'[^\?]*//', redirect_to): redirect_to = settings.LOGIN_REDIRECT_URL if request.user.is_anonymous(): if request.method == 'POST': try: stub_account = User.objects.filter(profile__stub_account=True).\ get(email__iexact=request.POST.get('email')) except User.DoesNotExist: stub_account = False if not stub_account: form = UserCreationFormExtended(request.POST) else: form = UserCreationFormExtended(request.POST, instance=stub_account) if form.is_valid(): cd = form.cleaned_data if not stub_account: # make a new user that is active, but has not confirmed their email address user = User.objects.create_user(cd['username'], cd['email'], cd['password1']) up = UserProfile(user=user) else: user = stub_account user.set_password(cd['password1']) up = stub_account.profile up.stub_account = False if cd['first_name']: user.first_name = cd['first_name'] if cd['last_name']: user.last_name = cd['last_name'] user.save() # Build and assign the activation key salt = hashlib.sha1(str(random.random())).hexdigest()[:5] up.activation_key = hashlib.sha1(salt + user.username).hexdigest() up.key_expires = now() + timedelta(days=5) up.save() # Send an email with the confirmation link to the new user email_subject = 'Confirm your account on CourtListener.com' email_body = ( "Hello, %s, and thanks for signing up for an account!\n\n" "To send you emails, we need you to activate your account with CourtListener. To " "activate your account, click this link within five days:\n\n" "https://www.courtlistener.com/email/confirm/%s\n\n" "Thanks for using our site,\n\n" "The CourtListener Team\n\n" "-------------------\n" "For questions or comments, please see our contact page, " "https://www.courtlistener.com/contact/." % (user.username, up.activation_key)) send_mail(email_subject, email_body, 'CourtListener <*****@*****.**>', [user.email]) # Send an email letting the admins know there's somebody to say hi to email_subject = 'New user confirmed on CourtListener: %s' % up.user.username email_body = ( "A new user has signed up on CourtListener and they'll be automatically welcomed soon!\n\n" " Their name is: %s\n" " Their email address is: %s\n\n" "Sincerely,\n\n" "The CourtListener Bots" % (up.user.get_full_name() or "Not provided", up.user.email)) send_mail(email_subject, email_body, 'CourtListener <*****@*****.**>', [a[1] for a in settings.ADMINS]) tally_stat('user.created') return HttpResponseRedirect('/register/success/?next=%s' % redirect_to) else: form = UserCreationFormExtended() return render_to_response("profile/register.html", { 'form': form, 'private': False }, RequestContext(request)) else: # The user is already logged in. Direct them to their settings page as # a logical fallback return HttpResponseRedirect('/profile/settings/')
class Command(BaseCommand): option_list = BaseCommand.option_list + ( make_option( '--rate', help="The rate to send emails (%s)" % ', '.join(dict(FREQUENCY).keys()), ), make_option( '--simulate', action='store_true', default=False, help='Simulate the emails that would be sent using the console ' 'backend.', ), make_option( '--date', help="The date that you want to send alerts for (for debugging, " "applies simulate mode).", ), make_option( '--user_id', help="A particular user id you want to run the alerts for (debug)." ), ) help = 'Sends the alert emails on a daily, weekly or monthly basis.' args = ('--rate (dly|wly|mly) [--simulate] [--date YYYY-MM-DD] ' '[--user USER]') def send_alert(self, user_profile, hits): email_subject = 'New hits for your CourtListener alerts' email_sender = 'CourtListener Alerts <*****@*****.**>' txt_template = loader.get_template('alerts/email.txt') html_template = loader.get_template('alerts/email.html') c = Context({'hits': hits}) txt = txt_template.render(c) html = html_template.render(c) msg = EmailMultiAlternatives(email_subject, txt, email_sender, [user_profile.user.email]) msg.attach_alternative(html, "text/html") if not self.options['simulate']: msg.send(fail_silently=False) def run_query(self, alert, cut_off_date): results = None error = False try: if self.verbosity >= 1: print "Now running the query: %s" % alert.alertText # Set up the data data = search_utils.get_string_to_dict(alert.alertText) try: del data['filed_before'] except KeyError: pass data['filed_after'] = cut_off_date data['order_by'] = 'score desc' if self.verbosity >= 1: print " Data sent to SearchForm is: %s" % data search_form = SearchForm(data) if search_form.is_valid(): cd = search_form.cleaned_data main_params = search_utils.build_main_query(cd) main_params.update({ 'rows': '20', 'start': '0', 'hl.tag.pre': '<em><strong>', 'hl.tag.post': '</strong></em>', 'caller': 'cl_send_alerts', }) if cd['type'] == 'o': conn = sunburnt.SolrInterface(settings.SOLR_OPINION_URL, mode='r') elif cd['type'] == 'oa': conn = sunburnt.SolrInterface(settings.SOLR_AUDIO_URL, mode='r') results = conn.raw_query(**main_params).execute() else: print " Query for alert %s was invalid" % alert.alertText print " Errors from the SearchForm: %s" % search_form.errors error = True except: traceback.print_exc() print " Search for this alert failed: %s" % alert.alertText error = True if self.verbosity >= 1: if results: print " There were %s results" % len(results) else: print " There were no results" if self.verbosity >= 2: print " The value of results is: %s" % results return error, cd['type'], results, def emailer(self, ups, cut_off_date): """Send out an email to every user whose alert has a new hit for a rate. , and iterate over them. For each of their alerts that has a hit, build up an email that contains all the hits. """ # for each user with a daily, weekly or monthly alert... alerts_sent_count = 0 for up in ups: # ...get their alerts... alerts = up.alert.filter(alertFrequency=self.options['rate']) if self.verbosity >= 1: print "\n\nAlerts for user '%s': %s" % (up.user, alerts) print "*" * 40 hits = [] # ...and iterate over their alerts. for alert in alerts: error, type, results = self.run_query(alert, cut_off_date) if error: continue # hits is a multi-dimensional array. It consists of alerts, # paired with a list of document dicts, of the form: # [[alert1, [{hit1}, {hit2}, {hit3}]], [alert2, ...]] try: if len(results) > 0: hits.append([alert, type, results]) alert.lastHitDate = now() alert.save() elif len(results) == 0 and alert.sendNegativeAlert: # if they want an alert even when no hits. hits.append([alert, type, None]) if self.verbosity >= 1: print " Sending results for negative alert '%s'" % \ alert.alertName except Exception, e: traceback.print_exc() print " Search failed on this alert: %s" % alert.alertText print e if len(hits) > 0: alerts_sent_count += 1 self.send_alert(up, hits) elif self.verbosity >= 1: print " No hits, thus not sending mail for this alert." if not self.options['simulate']: tally_stat('alerts.sent.%s' % self.options['rate'], inc=alerts_sent_count) logger.info("Sent %s %s email alerts." % (alerts_sent_count, self.options['rate']))
def show_results(request): """ This view can vary significantly, depending on how it is called: - In its most simple form, it is called via GET and without any parameters. --> This loads the homepage. - It might also be called with GET *with* parameters. --> This loads search results. - It might be called with a POST. --> This attempts to save an alert. It also has a few failure modes it needs to support: - It must react properly to an invalid alert form. - It must react properly to an invalid or failing search form. All of these paths have tests. """ # Create a search string that does not contain the page numbers get_string = search_utils.make_get_string(request) get_string_sans_alert = search_utils.make_get_string(request, ['page', 'edit_alert']) render_dict = { 'private': True, 'get_string': get_string, 'get_string_sans_alert': get_string_sans_alert, } if request.method == 'POST': # The user is trying to save an alert. alert_form = CreateAlertForm(request.POST, user=request.user) if alert_form.is_valid(): cd = alert_form.cleaned_data # save the alert if request.POST.get('edit_alert'): # check if the user can edit this, or if they are url hacking alert = get_object_or_404( Alert, pk=request.POST.get('edit_alert'), userprofile=request.user.profile ) alert_form = CreateAlertForm(cd, instance=alert, user=request.user) alert_form.save() action = "edited" else: alert_form = CreateAlertForm(cd, user=request.user) alert = alert_form.save() # associate the user with the alert up = request.user.profile up.alert.add(alert) action = "created" messages.add_message(request, messages.SUCCESS, 'Your alert was %s successfully.' % action) # and redirect to the alerts page return HttpResponseRedirect('/profile/alerts/') else: # Invalid form. Do the search again and show them the alert form # with the errors render_dict.update(do_search(request)) render_dict.update({'alert_form': alert_form}) return render_to_response( 'search/search.html', render_dict, RequestContext(request), ) else: # Either a search or the homepage if len(request.GET) == 0: # No parameters --> Homepage. if not is_bot(request): tally_stat('search.homepage_loaded') # Load the render_dict with good results that can be shown in the # "Latest Cases" section render_dict.update(do_search(request, rows=5, order_by='dateFiled desc')) # Get the results from the oral arguments as well oa_dict = do_search(request, rows=5, order_by='dateArgued desc', type='oa') render_dict.update({'results_oa': oa_dict['results']}) # But give it a fresh form for the advanced search section render_dict.update({'search_form': SearchForm(request.GET)}) ten_days_ago = make_aware(datetime.today() - timedelta(days=10), utc) alerts_in_last_ten = Stat.objects\ .filter( name__contains='alerts.sent', date_logged__gte=ten_days_ago)\ .aggregate(Sum('count'))['count__sum'] queries_in_last_ten = Stat.objects\ .filter( name='search.results', date_logged__gte=ten_days_ago) \ .aggregate(Sum('count'))['count__sum'] bulk_in_last_ten = Stat.objects\ .filter( name__contains='bulk_data', date_logged__gte=ten_days_ago)\ .aggregate(Sum('count'))['count__sum'] api_in_last_ten = Stat.objects \ .filter( name__contains='api', date_logged__gte=ten_days_ago) \ .aggregate(Sum('count'))['count__sum'] users_in_last_ten = User.objects\ .filter(date_joined__gte=ten_days_ago).count() opinions_in_last_ten = Document.objects\ .filter(time_retrieved__gte=ten_days_ago).count() oral_arguments_in_last_ten = Audio.objects\ .filter(time_retrieved__gte=ten_days_ago).count() days_of_oa = naturalduration( Audio.objects.aggregate(Sum('duration'))['duration__sum'], as_dict=True, )['d'] render_dict.update({ 'alerts_in_last_ten': alerts_in_last_ten, 'queries_in_last_ten': queries_in_last_ten, 'opinions_in_last_ten': opinions_in_last_ten, 'oral_arguments_in_last_ten': oral_arguments_in_last_ten, 'bulk_in_last_ten': bulk_in_last_ten, 'api_in_last_ten': api_in_last_ten, 'users_in_last_ten': users_in_last_ten, 'days_of_oa': days_of_oa, 'private': False }) return render_to_response( 'homepage.html', render_dict, RequestContext(request) ) else: # User placed a search or is trying to edit an alert if request.GET.get('edit_alert'): # They're editing an alert alert = get_object_or_404( Alert, pk=request.GET.get('edit_alert'), userprofile=request.user.profile ) alert_form = CreateAlertForm( instance=alert, initial={'query': get_string_sans_alert}, user=request.user, ) else: # Just a regular search if not is_bot(request): tally_stat('search.results') # Create bare-bones alert form. alert_form = CreateAlertForm( initial={'query': get_string, 'rate': "dly"}, user=request.user ) render_dict.update(do_search(request)) render_dict.update({'alert_form': alert_form}) return render_to_response( 'search/search.html', render_dict, RequestContext(request), )
def show_results(request): """ This view can vary significantly, depending on how it is called: - In its most simple form, it is called via GET and without any parameters. --> This loads the homepage. - It might also be called with GET *with* parameters. --> This loads search results. - It might be called with a POST. --> This attempts to save an alert. It also has a few failure modes it needs to support: - It must react properly to an invalid alert form. - It must react properly to an invalid or failing search form. All of these paths have tests. """ # Create a search string that does not contain the page numbers get_string = search_utils.make_get_string(request) get_string_sans_alert = search_utils.make_get_string( request, ['page', 'edit_alert']) render_dict = { 'private': True, 'get_string': get_string, 'get_string_sans_alert': get_string_sans_alert, } if request.method == 'POST': # The user is trying to save an alert. alert_form = CreateAlertForm(request.POST, user=request.user) if alert_form.is_valid(): cd = alert_form.cleaned_data # save the alert if request.POST.get('edit_alert'): # check if the user can edit this, or if they are url hacking alert = get_object_or_404(Alert, pk=request.POST.get('edit_alert'), userprofile=request.user.profile) alert_form = CreateAlertForm(cd, instance=alert, user=request.user) alert_form.save() action = "edited" else: alert_form = CreateAlertForm(cd, user=request.user) alert = alert_form.save() # associate the user with the alert up = request.user.profile up.alert.add(alert) action = "created" messages.add_message(request, messages.SUCCESS, 'Your alert was %s successfully.' % action) # and redirect to the alerts page return HttpResponseRedirect('/profile/alerts/') else: # Invalid form. Do the search again and show them the alert form # with the errors render_dict.update(do_search(request)) render_dict.update({'alert_form': alert_form}) return render_to_response( 'search/search.html', render_dict, RequestContext(request), ) else: # Either a search or the homepage if len(request.GET) == 0: # No parameters --> Homepage. if not is_bot(request): tally_stat('search.homepage_loaded') # Load the render_dict with good results that can be shown in the # "Latest Cases" section render_dict.update( do_search(request, rows=5, order_by='dateFiled desc')) # Get the results from the oral arguments as well oa_dict = do_search(request, rows=5, order_by='dateArgued desc', type='oa') render_dict.update({'results_oa': oa_dict['results']}) # But give it a fresh form for the advanced search section render_dict.update({'search_form': SearchForm(request.GET)}) ten_days_ago = make_aware(datetime.today() - timedelta(days=10), utc) alerts_in_last_ten = Stat.objects\ .filter( name__contains='alerts.sent', date_logged__gte=ten_days_ago)\ .aggregate(Sum('count'))['count__sum'] queries_in_last_ten = Stat.objects\ .filter( name='search.results', date_logged__gte=ten_days_ago) \ .aggregate(Sum('count'))['count__sum'] bulk_in_last_ten = Stat.objects\ .filter( name__contains='bulk_data', date_logged__gte=ten_days_ago)\ .aggregate(Sum('count'))['count__sum'] api_in_last_ten = Stat.objects \ .filter( name__contains='api', date_logged__gte=ten_days_ago) \ .aggregate(Sum('count'))['count__sum'] users_in_last_ten = User.objects\ .filter(date_joined__gte=ten_days_ago).count() opinions_in_last_ten = Document.objects\ .filter(time_retrieved__gte=ten_days_ago).count() oral_arguments_in_last_ten = Audio.objects\ .filter(time_retrieved__gte=ten_days_ago).count() days_of_oa = naturalduration( Audio.objects.aggregate(Sum('duration'))['duration__sum'], as_dict=True, )['d'] render_dict.update({ 'alerts_in_last_ten': alerts_in_last_ten, 'queries_in_last_ten': queries_in_last_ten, 'opinions_in_last_ten': opinions_in_last_ten, 'oral_arguments_in_last_ten': oral_arguments_in_last_ten, 'bulk_in_last_ten': bulk_in_last_ten, 'api_in_last_ten': api_in_last_ten, 'users_in_last_ten': users_in_last_ten, 'days_of_oa': days_of_oa, 'private': False }) return render_to_response('homepage.html', render_dict, RequestContext(request)) else: # User placed a search or is trying to edit an alert if request.GET.get('edit_alert'): # They're editing an alert alert = get_object_or_404(Alert, pk=request.GET.get('edit_alert'), userprofile=request.user.profile) alert_form = CreateAlertForm( instance=alert, initial={'query': get_string_sans_alert}, user=request.user, ) else: # Just a regular search if not is_bot(request): tally_stat('search.results') # Create bare-bones alert form. alert_form = CreateAlertForm(initial={ 'query': get_string, 'rate': "dly" }, user=request.user) render_dict.update(do_search(request)) render_dict.update({'alert_form': alert_form}) return render_to_response( 'search/search.html', render_dict, RequestContext(request), )
def register(request): """allow only an anonymous user to register""" redirect_to = request.REQUEST.get('next', '') if 'sign-in' in redirect_to: # thus, we don't redirect people back to the sign-in form redirect_to = '' # security checks: # Light security check -- make sure redirect_to isn't garbage. if not redirect_to or ' ' in redirect_to: redirect_to = settings.LOGIN_REDIRECT_URL # Heavier security check -- redirects to http://example.com should # not be allowed, but things like /view/?param=http://example.com # should be allowed. This regex checks if there is a '//' *before* a # question mark. elif '//' in redirect_to and re.match(r'[^\?]*//', redirect_to): redirect_to = settings.LOGIN_REDIRECT_URL if request.user.is_anonymous(): if request.method == 'POST': try: stub_account = User.objects.filter(profile__stub_account=True).\ get(email__iexact=request.POST.get('email')) except User.DoesNotExist: stub_account = False if not stub_account: form = UserCreationFormExtended(request.POST) else: form = UserCreationFormExtended(request.POST, instance=stub_account) if form.is_valid(): cd = form.cleaned_data if not stub_account: # make a new user that is active, but has not confirmed their email address user = User.objects.create_user(cd['username'], cd['email'], cd['password1']) up = UserProfile(user=user) else: user = stub_account user.set_password(cd['password1']) up = stub_account.profile up.stub_account = False if cd['first_name']: user.first_name = cd['first_name'] if cd['last_name']: user.last_name = cd['last_name'] user.save() # Build and assign the activation key salt = hashlib.sha1(str(random.random())).hexdigest()[:5] up.activation_key = hashlib.sha1(salt + user.username).hexdigest() up.key_expires = now() + timedelta(days=5) up.save() # Send an email with the confirmation link to the new user email_subject = 'Confirm your account on CourtListener.com' email_body = ("Hello, %s, and thanks for signing up for an account!\n\n" "To send you emails, we need you to activate your account with CourtListener. To " "activate your account, click this link within five days:\n\n" "https://www.courtlistener.com/email/confirm/%s\n\n" "Thanks for using our site,\n\n" "The CourtListener Team\n\n" "-------------------\n" "For questions or comments, please see our contact page, " "https://www.courtlistener.com/contact/." % (user.username, up.activation_key)) send_mail( email_subject, email_body, 'CourtListener <*****@*****.**>', [user.email] ) # Send an email letting the admins know there's somebody to say hi to email_subject = 'New user confirmed on CourtListener: %s' % up.user.username email_body = ("A new user has signed up on CourtListener and they'll be automatically welcomed soon!\n\n" " Their name is: %s\n" " Their email address is: %s\n\n" "Sincerely,\n\n" "The CourtListener Bots" % (up.user.get_full_name() or "Not provided", up.user.email)) send_mail(email_subject, email_body, 'CourtListener <*****@*****.**>', [a[1] for a in settings.ADMINS]) tally_stat('user.created') return HttpResponseRedirect('/register/success/?next=%s' % redirect_to) else: form = UserCreationFormExtended() return render_to_response("profile/register.html", {'form': form, 'private': False}, RequestContext(request)) else: # The user is already logged in. Direct them to their settings page as # a logical fallback return HttpResponseRedirect('/profile/settings/')
def dispatch(self, request_type, request, **kwargs): """Simple override here to tally stats before sending off the results.""" tally_stat(self.tally_name) return super(ModelResourceWithFieldsFilter, self).dispatch(request_type, request, **kwargs)
class Command(BaseCommand): option_list = BaseCommand.option_list + ( make_option( '--rate', help="The rate to send emails (%s)" % ', '.join(dict(FREQUENCY).keys()), ), make_option( '--simulate', action='store_true', default=False, help='Simulate the emails that would be sent using the console ' 'backend.', ), ) help = 'Sends the alert emails on a daily, weekly or monthly basis.' args = ('--rate (dly|wly|mly) [--simulate] [--date YYYY-MM-DD] ' '[--user USER]') def run_query(self, alert): results = [] error = False try: logger.info("Now running the query: %s\n" % alert.query) # Set up the data data = search_utils.get_string_to_dict(alert.query) try: del data['filed_before'] except KeyError: pass data['order_by'] = 'score desc' logger.info(" Data sent to SearchForm is: %s\n" % data) search_form = SearchForm(data) if search_form.is_valid(): cd = search_form.cleaned_data if self.rate == 'rt' and len(self.valid_ids[cd['type']]) == 0: # Bail out. No results will be found if no valid_ids. return error, cd['type'], results cut_off_date = get_cut_off_date(self.rate) if cd['type'] == 'o': cd['filed_after'] = cut_off_date elif cd['type'] == 'oa': cd['argued_after'] = cut_off_date main_params = search_utils.build_main_query(cd) main_params.update({ 'rows': '20', 'start': '0', 'hl.tag.pre': '<em><strong>', 'hl.tag.post': '</strong></em>', 'caller': 'cl_send_alerts', }) if self.rate == 'rt': main_params['fq'].append( 'id:(%s)' % ' OR '.join( [str(i) for i in self.valid_ids[cd['type']]] ), ) results = self.connections[ cd['type'] ].raw_query( **main_params ).execute() else: logger.info(" Query for alert %s was invalid\n" " Errors from the SearchForm: %s\n" % (alert.query, search_form.errors)) error = True except: traceback.print_exc() logger.info(" Search for this alert failed: %s\n" % alert.query) error = True logger.info(" There were %s results\n" % len(results)) return error, cd['type'], results def send_emails(self): """Send out an email to every user whose alert has a new hit for a rate. """ ups = UserProfile.objects.filter( alert__rate=self.rate, ).distinct() alerts_sent_count = 0 for up in ups: not_donated_enough = up.total_donated_last_year < \ settings.MIN_DONATION['rt_alerts'] if not_donated_enough and self.rate == 'rt': logger.info('\n\nUser: %s has not donated enough for their %s RT ' 'alerts to be sent.\n' % (up.user, len(alerts))) continue alerts = up.alert.filter(rate=self.rate) logger.info("\n\nAlerts for user '%s': %s\n" "%s\n" % (up.user, alerts, '*' * 40)) hits = [] for alert in alerts: error, type, results = self.run_query(alert) if error: continue # hits is a multi-dimensional array. It consists of alerts, # paired with a list of document dicts, of the form: # [[alert1, [{hit1}, {hit2}, {hit3}]], [alert2, ...]] try: if len(results) > 0: hits.append([alert, type, results]) alert.date_last_hit = now() alert.save() elif len(results) == 0 and alert.always_send_email: hits.append([alert, type, None]) logger.info(" Sending results for negative alert " "'%s'\n" % alert.name) except Exception, e: traceback.print_exc() logger.info(" Search failed on this alert: %s\n%s\n" % (alert.query, e)) if len(hits) > 0: alerts_sent_count += 1 send_alert(up, hits, self.options['simulate']) elif self.verbosity >= 1: logger.info(" No hits. Not sending mail for this alert.\n") if not self.options['simulate']: tally_stat('alerts.sent.%s' % self.rate, inc=alerts_sent_count) logger.info("Sent %s %s email alerts." % (alerts_sent_count, self.rate))
def serve_or_gen_dump(request, court, year=None, month=None, day=None): """Serves the dump file to the user, generating it if needed.""" if year is None: if court != 'all': # Sanity check return HttpResponseBadRequest('<h2>Error 400: Complete dumps are ' 'not available for individual courts. Try using "all" for ' 'your court ID instead.</h2>') else: # Serve the dump for all cases. tally_stat('bulk_data.served.all') return HttpResponseRedirect('/dumps/all.xml.gz') else: # Date-based dump start_date, end_date, annual, monthly, daily = get_date_range(year, month, day) # Ensure that it's a valid request. if (now() < end_date) and (now() < start_date): # It's the future. They fail. return HttpResponseBadRequest('<h2>Error 400: Requested date is in the future. Please try again then.</h2>') elif now() <= end_date: # Some of the data is in the past, some could be in the future. return HttpResponseBadRequest('<h2>Error 400: Requested date is partially in the future. Please try again ' 'then.</h2>') filename = court + '.xml' if daily: filepath = os.path.join(year, month, day) elif monthly: filepath = os.path.join(year, month) elif annual: filepath = os.path.join(year) path_from_root = os.path.join(settings.DUMP_DIR, filepath) # See if we already have it on disk. try: _ = open(os.path.join(path_from_root, filename + '.gz'), 'rb') tally_stat('bulk_data.served.by_date') return HttpResponseRedirect(os.path.join('/dumps', filepath, filename + '.gz')) except IOError: # Time-based dump if court == 'all': # dump everything; disable default ordering qs = Document.objects.all().order_by() else: # dump just the requested court; disable default ordering qs = Document.objects.filter(court=court).order_by() # check if there are any documents at all dump_has_docs = qs.filter(date_filed__gte=start_date, date_filed__lte=end_date).exists() if dump_has_docs: docs_to_dump = queryset_generator_by_date(qs, 'date_filed', start_date, end_date) make_dump_file(docs_to_dump, path_from_root, filename) else: print "Bad request!" return HttpResponseBadRequest('<h2>Error 400: We do not have any data for this time period.</h2>', status=404) tally_stat('bulk_data.served.by_date') return HttpResponseRedirect('%s.gz' % os.path.join('/dumps', filepath, filename))