Ejemplo n.º 1
0
    def process_request(self, request):
        """Adds is_human field to the request object.  This is used to filter non-human
           activity from the usage logs"""

        user_agent = request.META.get('HTTP_USER_AGENT', None)
        request.is_human = True

        if user_agent is None or robot_detection.is_robot(user_agent):
            request.is_human = False
Ejemplo n.º 2
0
    def process_request(self, request):
        """Add is_human field to the request object.

        This is used to filter non-human activity from the usage logs
        """
        user_agent = request.META.get('HTTP_USER_AGENT', None)
        request.is_human = True

        if user_agent is None or robot_detection.is_robot(user_agent):
            request.is_human = False
Ejemplo n.º 3
0
def thread_index(request, mlist_fqdn, threadid, month=None, year=None):
    ''' Displays all the email for a given thread identifier '''
    store = get_store(request)
    thread = store.get_thread(mlist_fqdn, threadid)
    if not thread:
        raise Http404
    prev_thread, next_thread = store.get_thread_neighbors(mlist_fqdn, threadid)

    sort_mode = request.GET.get("sort", "thread")
    set_message_votes(thread.starting_email, request.user)

    # Tags
    tag_form = AddTagForm()
    try:
        tags = Tag.objects.filter(threadid=threadid, list_address=mlist_fqdn)
    except Tag.DoesNotExist:
        tags = []

    # Favorites
    fav_action = "add"
    if request.user.is_authenticated():
        try:
            Favorite.objects.get(list_address=mlist_fqdn, threadid=threadid,
                                 user=request.user)
        except Favorite.DoesNotExist:
            pass
        else:
            fav_action = "rm"

    # Category
    category, category_form = get_category_widget(request, thread.category)

    # Extract relative dates
    today = datetime.date.today()
    days_old = today - thread.starting_email.date.date()
    days_inactive = today - thread.last_email.date.date()

    mlist = store.get_list(mlist_fqdn)
    subject = stripped_subject(mlist, thread.starting_email.subject)

    # Last view
    last_view = None
    if request.user.is_authenticated():
        last_view_obj, created = LastView.objects.get_or_create(
                list_address=mlist_fqdn, threadid=threadid, user=request.user)
        if not created:
            last_view = last_view_obj.view_date
            last_view_obj.save() # update timestamp
    # get the number of unread messages
    if last_view is None:
        if request.user.is_authenticated():
            unread_count = len(thread)
        else:
            unread_count = 0
    else:
        # XXX: Storm-specific
        unread_count = thread.replies_after(last_view).count()

    # Flash messages
    flash_messages = []
    flash_msg = request.GET.get("msg")
    if flash_msg:
        flash_msg = { "type": FLASH_MESSAGES[flash_msg][0],
                      "msg": FLASH_MESSAGES[flash_msg][1] }
        flash_messages.append(flash_msg)

    # TODO: eventually move to a middleware ?
    # http://djangosnippets.org/snippets/1865/
    is_bot = True
    user_agent = request.META.get('HTTP_USER_AGENT', None)
    if user_agent:
        is_bot = robot_detection.is_robot(user_agent)

    context = {
        'mlist': mlist,
        'threadid': threadid,
        'subject': subject,
        'tags': tags,
        'addtag_form': tag_form,
        'month': thread.date_active,
        'first_mail': thread.starting_email,
        'neighbors': (prev_thread, next_thread),
        'months_list': get_months(store, mlist.name),
        'days_inactive': days_inactive.days,
        'days_old': days_old.days,
        'sort_mode': sort_mode,
        'fav_action': fav_action,
        'reply_form': ReplyForm(),
        'is_bot': is_bot,
        'num_comments': len(thread),
        'participants': thread.participants,
        'last_view': last_view,
        'unread_count': unread_count,
        'category_form': category_form,
        'category': category,
        'flash_messages': flash_messages,
    }
    context["participants"].sort(key=lambda x: x[0].lower())

    if is_bot:
        # Don't rely on AJAX to load the replies
        # The limit is a safety measure, don't let a bot kill the DB
        context["replies"] = _get_thread_replies(request, thread, limit=1000)

    return render(request, "thread.html", context)
Ejemplo n.º 4
0
 def testFunc(self):
     self.assertTrue(robot_detection.is_robot(user_agent))
Ejemplo n.º 5
0
def thread_index(request, mlist_fqdn, threadid, month=None, year=None):
    ''' Displays all the email for a given thread identifier '''
    mlist = get_object_or_404(MailingList, name=mlist_fqdn)
    thread = get_object_or_404(Thread, mailinglist=mlist, thread_id=threadid)
    starting_email = thread.starting_email

    sort_mode = request.GET.get("sort", "thread")
    if request.user.is_authenticated():
        starting_email.myvote = starting_email.votes.filter(
            user=request.user).first()
    else:
        starting_email.myvote = None

    # Tags
    tag_form = AddTagForm()

    # Favorites
    fav_action = "add"
    if request.user.is_authenticated() and Favorite.objects.filter(
            thread=thread, user=request.user).exists():
        fav_action = "rm"

    # Category
    category, category_form = get_category_widget(request, thread.category)

    # Extract relative dates
    today = datetime.date.today()
    days_old = today - starting_email.date.date()
    days_inactive = today - thread.date_active.date()

    subject = stripped_subject(mlist, starting_email.subject)

    # Last view
    last_view = None
    if request.user.is_authenticated():
        last_view_obj, created = LastView.objects.get_or_create(
            thread=thread, user=request.user)
        if not created:
            last_view = last_view_obj.view_date
            last_view_obj.save()  # update timestamp
    # get the number of unread messages
    if last_view is None:
        if request.user.is_authenticated():
            unread_count = thread.emails_count
        else:
            unread_count = 0
    else:
        unread_count = thread.emails.filter(date__gt=last_view).count()

    # TODO: eventually move to a middleware ?
    # http://djangosnippets.org/snippets/1865/
    user_agent = request.META.get('HTTP_USER_AGENT', None)
    if user_agent:
        is_bot = robot_detection.is_robot(user_agent)
    else:
        is_bot = True

    # Export button
    export = {
        "url":
        "%s?thread=%s" %
        (reverse("hk_list_export_mbox",
                 kwargs={
                     "mlist_fqdn": mlist.name,
                     "filename": "%s-%s" % (mlist.name, thread.thread_id)
                 }), thread.thread_id),
        "message":
        _("Download"),
        "title":
        _("This thread in gzipped mbox format"),
    }

    context = {
        'mlist': mlist,
        'thread': thread,
        'starting_email': starting_email,
        'subject': subject,
        'addtag_form': tag_form,
        'month': thread.date_active,
        'months_list': get_months(mlist),
        'days_inactive': days_inactive.days,
        'days_old': days_old.days,
        'sort_mode': sort_mode,
        'fav_action': fav_action,
        'reply_form': get_posting_form(ReplyForm, request, mlist),
        'is_bot': is_bot,
        'num_comments': thread.emails_count - 1,
        'last_view': last_view,
        'unread_count': unread_count,
        'category_form': category_form,
        'category': category,
        'export': export,
    }

    if is_bot:
        # Don't rely on AJAX to load the replies
        # The limit is a safety measure, don't let a bot kill the DB
        context["replies"] = _get_thread_replies(request, thread, limit=1000)

    return render(request, "hyperkitty/thread.html", context)
Ejemplo n.º 6
0
 def testFunc(self):
     self.assertTrue(robot_detection.is_robot(user_agent))
Ejemplo n.º 7
0
def is_human(user_agent):
    if robot_detection.is_robot(user_agent):
        return False
    return True
Ejemplo n.º 8
0
def thread_index(request, mlist_fqdn, threadid, month=None, year=None):
    ''' Displays all the email for a given thread identifier '''
    # pylint: disable=unused-argument
    mlist = get_object_or_404(MailingList, name=mlist_fqdn)
    thread = get_object_or_404(Thread, mailinglist=mlist, thread_id=threadid)
    starting_email = thread.starting_email

    sort_mode = request.GET.get("sort", "thread")
    if request.user.is_authenticated():
        starting_email.myvote = starting_email.votes.filter(
            user=request.user).first()
    else:
        starting_email.myvote = None

    # Tags
    tag_form = AddTagForm()

    # Favorites
    fav_action = "add"
    if request.user.is_authenticated() and Favorite.objects.filter(
            thread=thread, user=request.user).exists():
        fav_action = "rm"

    # Category
    category, category_form = get_category_widget(request, thread.category)

    # Extract relative dates
    today = datetime.date.today()
    days_old = today - starting_email.date.date()
    days_inactive = today - thread.date_active.date()

    subject = stripped_subject(mlist, starting_email.subject)

    # Last view
    last_view = None
    if request.user.is_authenticated():
        last_view_obj, created = LastView.objects.get_or_create(
                thread=thread, user=request.user)
        if not created:
            last_view = last_view_obj.view_date
            last_view_obj.save() # update timestamp
    # get the number of unread messages
    if last_view is None:
        if request.user.is_authenticated():
            unread_count = thread.emails_count
        else:
            unread_count = 0
    else:
        unread_count = thread.emails.filter(date__gt=last_view).count()

    # TODO: eventually move to a middleware ?
    # http://djangosnippets.org/snippets/1865/
    user_agent = request.META.get('HTTP_USER_AGENT', None)
    if user_agent:
        is_bot = robot_detection.is_robot(user_agent)
    else:
        is_bot = True

    # Export button
    export = {
        "url": "%s?thread=%s" % (
            reverse("hk_list_export_mbox", kwargs={
                    "mlist_fqdn": mlist.name,
                    "filename": "%s-%s" % (mlist.name, thread.thread_id)}),
            thread.thread_id),
        "message": _("Download"),
        "title": _("This thread in gzipped mbox format"),
    }

    context = {
        'mlist': mlist,
        'thread': thread,
        'starting_email': starting_email,
        'subject': subject,
        'addtag_form': tag_form,
        'month': thread.date_active,
        'months_list': get_months(mlist),
        'days_inactive': days_inactive.days,
        'days_old': days_old.days,
        'sort_mode': sort_mode,
        'fav_action': fav_action,
        'reply_form': get_posting_form(ReplyForm, request, mlist),
        'is_bot': is_bot,
        'num_comments': thread.emails_count - 1,
        'last_view': last_view,
        'unread_count': unread_count,
        'category_form': category_form,
        'category': category,
        'export': export,
    }

    if is_bot:
        # Don't rely on AJAX to load the replies
        # The limit is a safety measure, don't let a bot kill the DB
        context["replies"] = _get_thread_replies(request, thread, limit=1000)

    return render(request, "hyperkitty/thread.html", context)
Ejemplo n.º 9
0
def thread_index(request, mlist_fqdn, threadid, month=None, year=None):
    ''' Displays all the email for a given thread identifier '''
    # pylint: disable=unused-argument
    mlist = get_object_or_404(MailingList, name=mlist_fqdn)
    thread = get_object_or_404(Thread, mailinglist=mlist, thread_id=threadid)
    starting_email = thread.starting_email

    sort_mode = request.GET.get("sort", "thread")
    if request.user.is_authenticated():
        starting_email.myvote = starting_email.votes.filter(
            user=request.user).first()
    else:
        starting_email.myvote = None

    # Tags
    tag_form = AddTagForm()

    # Favorites
    fav_action = "add"
    if request.user.is_authenticated() and Favorite.objects.filter(
            thread=thread, user=request.user).exists():
        fav_action = "rm"

    # Category
    categories = [ (c.name, c.name.upper())
                   for c in ThreadCategory.objects.all() ] \
                 + [("", "no category")]
    category, category_form = get_category_widget(request, thread.category,
                                                  categories)

    # Extract relative dates
    today = datetime.date.today()
    days_old = today - starting_email.date.date()
    days_inactive = today - thread.date_active.date()

    subject = stripped_subject(mlist, starting_email.subject)

    # Last view
    last_view = None
    if request.user.is_authenticated():
        last_view_obj, created = LastView.objects.get_or_create(
            thread=thread, user=request.user)
        if not created:
            last_view = last_view_obj.view_date
            last_view_obj.save()  # update timestamp
    # get the number of unread messages
    if last_view is None:
        if request.user.is_authenticated():
            unread_count = thread.emails_count
        else:
            unread_count = 0
    else:
        unread_count = thread.emails.filter(date__gt=last_view).count()

    # Flash messages
    flash_messages = []
    flash_msg = request.GET.get("msg")
    if flash_msg:
        flash_msg = {
            "type": FLASH_MESSAGES[flash_msg][0],
            "msg": FLASH_MESSAGES[flash_msg][1]
        }
        flash_messages.append(flash_msg)

    # TODO: eventually move to a middleware ?
    # http://djangosnippets.org/snippets/1865/
    user_agent = request.META.get('HTTP_USER_AGENT', None)
    if user_agent:
        is_bot = robot_detection.is_robot(user_agent)
    else:
        is_bot = True

    context = {
        'mlist': mlist,
        'thread': thread,
        'starting_email': starting_email,
        'subject': subject,
        'addtag_form': tag_form,
        'month': thread.date_active,
        'months_list': get_months(mlist),
        'days_inactive': days_inactive.days,
        'days_old': days_old.days,
        'sort_mode': sort_mode,
        'fav_action': fav_action,
        'reply_form': ReplyForm(),
        'is_bot': is_bot,
        'num_comments': thread.emails_count - 1,
        'last_view': last_view,
        'unread_count': unread_count,
        'category_form': category_form,
        'category': category,
        'flash_messages': flash_messages,
    }

    if is_bot:
        # Don't rely on AJAX to load the replies
        # The limit is a safety measure, don't let a bot kill the DB
        context["replies"] = _get_thread_replies(request, thread, limit=1000)

    return render(request, "hyperkitty/thread.html", context)
Ejemplo n.º 10
0
def thread_index(request, mlist_fqdn, threadid, month=None, year=None):
    ''' Displays all the email for a given thread identifier '''
    search_form = SearchForm(auto_id=False)
    store = get_store(request)
    thread = store.get_thread(mlist_fqdn, threadid)
    if not thread:
        raise Http404
    prev_thread, next_thread = store.get_thread_neighbors(mlist_fqdn, threadid)

    sort_mode = request.GET.get("sort", "thread")
    set_message_votes(thread.starting_email, request.user)

    from_url = reverse("thread", kwargs={"mlist_fqdn":mlist_fqdn,
                                         "threadid":threadid})
    # Tags
    tag_form = AddTagForm(initial={'from_url' : from_url})
    try:
        tags = Tag.objects.filter(threadid=threadid, list_address=mlist_fqdn)
    except Tag.DoesNotExist:
        tags = []

    # Favorites
    fav_action = "add"
    if request.user.is_authenticated():
        try:
            Favorite.objects.get(list_address=mlist_fqdn, threadid=threadid,
                                 user=request.user)
        except Favorite.DoesNotExist:
            pass
        else:
            fav_action = "rm"

    # Extract relative dates
    today = datetime.date.today()
    days_old = today - thread.starting_email.date.date()
    days_inactive = today - thread.last_email.date.date()

    mlist = store.get_list(mlist_fqdn)
    subject = stripped_subject(mlist, thread.starting_email.subject)

    # TODO: eventually move to a middleware ?
    # http://djangosnippets.org/snippets/1865/
    is_bot = True
    user_agent = request.META.get('HTTP_USER_AGENT', None)
    if user_agent:
        is_bot = robot_detection.is_robot(user_agent)

    context = {
        'mlist': mlist,
        'threadid': threadid,
        'subject': subject,
        'tags': tags,
        'search_form': search_form,
        'addtag_form': tag_form,
        'month': thread.date_active,
        'first_mail': thread.starting_email,
        'neighbors': (prev_thread, next_thread),
        'months_list': get_months(store, mlist.name),
        'days_inactive': days_inactive.days,
        'days_old': days_old.days,
        'sort_mode': sort_mode,
        'fav_action': fav_action,
        'reply_form': ReplyForm(),
        'is_bot': is_bot,
        'participants': thread.participants,
    }
    context["participants"].sort(key=lambda x: x[0].lower())

    if is_bot:
        # Don't rely on AJAX to load the replies
        context["replies"] = _get_thread_replies(request, thread)

    return render(request, "thread.html", context)
Ejemplo n.º 11
0
def thread_index(request, mlist_fqdn, threadid, month=None, year=None):
    ''' Displays all the email for a given thread identifier '''
    store = get_store(request)
    thread = store.get_thread(mlist_fqdn, threadid)
    if not thread:
        raise Http404
    prev_thread, next_thread = store.get_thread_neighbors(mlist_fqdn, threadid)

    sort_mode = request.GET.get("sort", "thread")
    set_message_votes(thread.starting_email, request.user)

    # Tags
    tag_form = AddTagForm()
    try:
        tags = Tag.objects.filter(threadid=threadid, list_address=mlist_fqdn)
    except Tag.DoesNotExist:
        tags = []

    # Favorites
    fav_action = "add"
    if request.user.is_authenticated():
        try:
            Favorite.objects.get(list_address=mlist_fqdn,
                                 threadid=threadid,
                                 user=request.user)
        except Favorite.DoesNotExist:
            pass
        else:
            fav_action = "rm"

    # Category
    category, category_form = get_category_widget(request, thread.category)

    # Extract relative dates
    today = datetime.date.today()
    days_old = today - thread.starting_email.date.date()
    days_inactive = today - thread.last_email.date.date()

    mlist = store.get_list(mlist_fqdn)
    subject = stripped_subject(mlist, thread.starting_email.subject)

    # Last view
    last_view = None
    if request.user.is_authenticated():
        last_view_obj, created = LastView.objects.get_or_create(
            list_address=mlist_fqdn, threadid=threadid, user=request.user)
        if not created:
            last_view = last_view_obj.view_date
            last_view_obj.save()  # update timestamp
    # get the number of unread messages
    if last_view is None:
        if request.user.is_authenticated():
            unread_count = len(thread)
        else:
            unread_count = 0
    else:
        # XXX: Storm-specific
        unread_count = thread.replies_after(last_view).count()

    # Flash messages
    flash_messages = []
    flash_msg = request.GET.get("msg")
    if flash_msg:
        flash_msg = {
            "type": FLASH_MESSAGES[flash_msg][0],
            "msg": FLASH_MESSAGES[flash_msg][1]
        }
        flash_messages.append(flash_msg)

    # TODO: eventually move to a middleware ?
    # http://djangosnippets.org/snippets/1865/
    is_bot = True
    user_agent = request.META.get('HTTP_USER_AGENT', None)
    if user_agent:
        is_bot = robot_detection.is_robot(user_agent)

    context = {
        'mlist': mlist,
        'threadid': threadid,
        'subject': subject,
        'tags': tags,
        'addtag_form': tag_form,
        'month': thread.date_active,
        'first_mail': thread.starting_email,
        'neighbors': (prev_thread, next_thread),
        'months_list': get_months(store, mlist.name),
        'days_inactive': days_inactive.days,
        'days_old': days_old.days,
        'sort_mode': sort_mode,
        'fav_action': fav_action,
        'reply_form': ReplyForm(),
        'is_bot': is_bot,
        'num_comments': len(thread),
        'participants': thread.participants,
        'last_view': last_view,
        'unread_count': unread_count,
        'category_form': category_form,
        'category': category,
        'flash_messages': flash_messages,
    }
    context["participants"].sort(key=lambda x: x[0].lower())

    if is_bot:
        # Don't rely on AJAX to load the replies
        # The limit is a safety measure, don't let a bot kill the DB
        context["replies"] = _get_thread_replies(request, thread, limit=1000)

    return render(request, "thread.html", context)
Ejemplo n.º 12
0
def thread_index(request, mlist_fqdn, threadid, month=None, year=None):
    ''' Displays all the email for a given thread identifier '''
    search_form = SearchForm(auto_id=False)
    store = get_store(request)
    thread = store.get_thread(mlist_fqdn, threadid)
    if not thread:
        raise Http404
    prev_thread, next_thread = store.get_thread_neighbors(mlist_fqdn, threadid)

    sort_mode = request.GET.get("sort", "thread")
    set_message_votes(thread.starting_email, request.user)

    from_url = reverse("thread",
                       kwargs={
                           "mlist_fqdn": mlist_fqdn,
                           "threadid": threadid
                       })
    # Tags
    tag_form = AddTagForm(initial={'from_url': from_url})
    try:
        tags = Tag.objects.filter(threadid=threadid, list_address=mlist_fqdn)
    except Tag.DoesNotExist:
        tags = []

    # Favorites
    fav_action = "add"
    if request.user.is_authenticated():
        try:
            Favorite.objects.get(list_address=mlist_fqdn,
                                 threadid=threadid,
                                 user=request.user)
        except Favorite.DoesNotExist:
            pass
        else:
            fav_action = "rm"

    # Extract relative dates
    today = datetime.date.today()
    days_old = today - thread.starting_email.date.date()
    days_inactive = today - thread.last_email.date.date()

    mlist = store.get_list(mlist_fqdn)
    subject = stripped_subject(mlist, thread.starting_email.subject)

    # TODO: eventually move to a middleware ?
    # http://djangosnippets.org/snippets/1865/
    is_bot = True
    user_agent = request.META.get('HTTP_USER_AGENT', None)
    if user_agent:
        is_bot = robot_detection.is_robot(user_agent)

    context = {
        'mlist': mlist,
        'threadid': threadid,
        'subject': subject,
        'tags': tags,
        'search_form': search_form,
        'addtag_form': tag_form,
        'month': thread.date_active,
        'first_mail': thread.starting_email,
        'neighbors': (prev_thread, next_thread),
        'months_list': get_months(store, mlist.name),
        'days_inactive': days_inactive.days,
        'days_old': days_old.days,
        'sort_mode': sort_mode,
        'fav_action': fav_action,
        'reply_form': ReplyForm(),
        'is_bot': is_bot,
        'participants': thread.participants,
    }
    context["participants"].sort(key=lambda x: x[0].lower())

    if is_bot:
        # Don't rely on AJAX to load the replies
        context["replies"] = _get_thread_replies(request, thread)

    return render(request, "thread.html", context)
Ejemplo n.º 13
0
def thread_index(request, mlist_fqdn, threadid, month=None, year=None):
    ''' Displays all the email for a given thread identifier '''
    # pylint: disable=unused-argument
    mlist = get_object_or_404(MailingList, name=mlist_fqdn)
    thread = get_object_or_404(Thread, mailinglist=mlist, thread_id=threadid)
    starting_email = thread.starting_email

    sort_mode = request.GET.get("sort", "thread")
    if request.user.is_authenticated():
        starting_email.myvote = starting_email.votes.filter(
            user=request.user).first()
    else:
        starting_email.myvote = None

    # Tags
    tag_form = AddTagForm()

    # Favorites
    fav_action = "add"
    if request.user.is_authenticated() and Favorite.objects.filter(
            thread=thread, user=request.user).exists():
        fav_action = "rm"

    # Category
    categories = [ (c.name, c.name.upper())
                   for c in ThreadCategory.objects.all() ] \
                 + [("", "no category")]
    category, category_form = get_category_widget(
        request, thread.category, categories)

    # Extract relative dates
    today = datetime.date.today()
    days_old = today - starting_email.date.date()
    days_inactive = today - thread.date_active.date()

    subject = stripped_subject(mlist, starting_email.subject)

    # Last view
    last_view = None
    if request.user.is_authenticated():
        last_view_obj, created = LastView.objects.get_or_create(
                thread=thread, user=request.user)
        if not created:
            last_view = last_view_obj.view_date
            last_view_obj.save() # update timestamp
    # get the number of unread messages
    if last_view is None:
        if request.user.is_authenticated():
            unread_count = thread.emails_count
        else:
            unread_count = 0
    else:
        unread_count = thread.emails.filter(date__gt=last_view).count()

    # Flash messages
    flash_messages = []
    flash_msg = request.GET.get("msg")
    if flash_msg:
        flash_msg = { "type": FLASH_MESSAGES[flash_msg][0],
                      "msg": FLASH_MESSAGES[flash_msg][1] }
        flash_messages.append(flash_msg)

    # TODO: eventually move to a middleware ?
    # http://djangosnippets.org/snippets/1865/
    user_agent = request.META.get('HTTP_USER_AGENT', None)
    if user_agent:
        is_bot = robot_detection.is_robot(user_agent)
    else:
        is_bot = True

    context = {
        'mlist': mlist,
        'thread': thread,
        'starting_email': starting_email,
        'subject': subject,
        'addtag_form': tag_form,
        'month': thread.date_active,
        'months_list': get_months(mlist),
        'days_inactive': days_inactive.days,
        'days_old': days_old.days,
        'sort_mode': sort_mode,
        'fav_action': fav_action,
        'reply_form': ReplyForm(),
        'is_bot': is_bot,
        'num_comments': thread.emails_count - 1,
        'last_view': last_view,
        'unread_count': unread_count,
        'category_form': category_form,
        'category': category,
        'flash_messages': flash_messages,
    }

    if is_bot:
        # Don't rely on AJAX to load the replies
        # The limit is a safety measure, don't let a bot kill the DB
        context["replies"] = _get_thread_replies(request, thread, limit=1000)

    return render(request, "hyperkitty/thread.html", context)
Ejemplo n.º 14
0
def flag_robots(doc):
    """Flag events which are created by robots."""
    doc['is_robot'] = 'user_agent' in doc and is_robot(doc['user_agent'])
    return doc
Ejemplo n.º 15
0
def create_training_data(event, context):
    logging.info(json.dumps({'event': event}))
    correlation_id = get_correlation_id(event=event)

    s3_resource = boto3.resource('s3')
    data = open('./data.log')
    reader = geoip2.database.Reader('./GeoLite2-City.mmdb')
    parsed = cloudfront_log_parser.parse(data)
    output = io.StringIO()
    fieldnames = [
        'ip_address', 'day_of_week', 'hour_of_day', 'minute_of_hour', 'edge',
        'response_size', 'http_method', 'cloudfront_host', 'path',
        'status_code', 'status_code_group', 'aborted', 'referrer',
        'user_agent', 'browser_family', 'browser_version', 'os_family',
        'os_version', 'device', 'is_mobile', 'is_tablet', 'is_pc',
        'is_touch_capable', 'is_bot', 'querystring', 'edge_result_type',
        'request_host', 'request_protocol', 'request_size',
        'response_duration', 'ssl_protocol', 'ssl_cypher',
        'edge_response_result_type', 'country', 'city', 'latitude',
        'longitude', 'is_malicious_bot'
    ]
    writer = csv.DictWriter(output,
                            fieldnames=fieldnames,
                            quoting=csv.QUOTE_NONNUMERIC)
    writer.writeheader()

    return_object = []
    pool = ThreadPool(12)
    return_object = pool.starmap(format_data,
                                 zip(parsed, itertools.repeat(reader)))
    pool.close()
    pool.join()
    reader.close()

    #    with ThreadPoolExecutor(max_workers=12) as executor:
    #        for arg in zip(parsed, itertools.repeat(reader)):
    #            future = executor.submit(format_data, arg)
    #    return_object = future.result()

    whitelisted_ips = ['']
    bad_ips = ['111.88.139.9', '111.88.139.9']
    bad_agents = [
        'curl', 'wget', 'Python', 'python', 'ruby', '-', 'Java', 'PhantomJS'
    ]
    logging.debug(
        json.dumps({
            'message': 'multithreaded return object',
            'object': '{}'.format(return_object)
        }))
    for row in return_object:
        is_malicious_bot = True
        for i in bad_agents:
            if i in row['user_agent']:
                is_malicious_bot = True
        if robot_detection.is_robot(row['ip_address']):
            is_malicious_bot = False
        if row['ip_address'] in whitelisted_ips:
            is_malicious_bot = False
        if row['ip_address'] in bad_ips:
            is_malicious_bot = True
        if 'health' in row['path']:
            is_malicious_bot = False
        row['is_malicious_bot'] = is_malicious_bot

        # Ignore this metadata when training
        row['ip_address'] = ''
        row['request_host'] = ''
        writer.writerow(row)
    s3_resource.Object(os.environ['TRANSFORMED_BUCKET'],
                       "training-data.csv").put(Body=output.getvalue())
    logging.info(json.dumps({'message': 'Done!'}))
Ejemplo n.º 16
0
def is_human(user_agent):
    if robot_detection.is_robot(user_agent):
        return False
    return True