def register(name, loader):
"""
Register a loader function for the given column.
"""
if isinstance(loader, str):
loader = import_later(loader)
DATA_LOADERS[name] = loader
from collections import Iterable
from random import randrange
from boogie import db
from boogie.models import QuerySet
from django.contrib.auth import get_user_model
from django.core.exceptions import ImproperlyConfigured
from django.utils.timezone import now
from sidekick import import_later
from ej_profiles.enums import Gender, Race
from ej_profiles.utils import years_from
from .math import user_statistics
db = db.ej_conversations
np = import_later('numpy')
NOT_GIVEN = object()
class ConversationMixin:
"""
Implements an interface with a predictable route to fetch conversations,
comments and votes related to the current queryset.
Different models may interpret this relation slightly different, and this
mixin just implements sane defaults.
"""
def _votes_from_comments(self, comments):
return comments.votes()
def random(self, default=NOT_GIVEN):
from django.utils.translation import ugettext_lazy as _
from model_utils.models import TimeStampedModel
import sidekick as sk
from boogie import rules
from boogie.fields import EnumField
from boogie.rest import rest_api
from ej_conversations.managers import BoogieManager
from ej_conversations.models import Choice, Conversation
from ej_conversations.models.vote import normalize_choice
from sidekick import delegate_to, alias
from .manager import ClusterManager
from .types import ClusterStatus
log = logging.getLogger('ej')
math = sk.import_later('.math', package=__package__)
@rest_api()
class Clusterization(TimeStampedModel):
"""
Manages clusterization tasks for a given conversation.
"""
conversation = models.OneToOneField(
'ej_conversations.Conversation',
on_delete=models.CASCADE,
related_name='clusterization',
)
cluster_status = EnumField(
ClusterStatus,
default=ClusterStatus.PENDING_DATA,
from random import random
import numpy as np
from django.db import IntegrityError
from sidekick import import_later
global_id = 0
get_user_model = import_later("django.contrib.auth:get_user_model")
def create_colors_experiment(skip_prob=0.25,
n_times=1,
n_users=21,
has_yellow=False):
"""
The "colors" experiment creates a few groups tagged by color names.
Each comment has a baseline probability of around 25% for agree/disagree in
the general population, but has some different probability for each user.
"""
from ej_conversations import create_conversation
from ej_clusters.models import Clusterization
from ej_clusters.enums import ClusterStatus
# Conversation
rgb = ("red", "green", "blue")
author = create_user("color", "admin")
conversation = create_conversation("What is your favorite color?",
"Color",
author,
is_promoted=True)
from typing import Union, TYPE_CHECKING
import sidekick as _sk
import streamlit as _st
from . import sidebar
from .. import components as _components
if TYPE_CHECKING:
import numpy as np
import pandas as pd
import streamlit as st
_self = _sk.import_later("pydemic_ui.st")
#
# Utilities
#
asset = _components.asset
#
# Generic
#
card = _components.card.bind(_st)
cards = _components.cards.bind(_st)
css = _components.css.bind(_st)
dataframe_download = _components.dataframe_download.bind(_st)
data_anchor = _components.data_anchor.bind(_st)
footnote_disclaimer = _components.footnote_disclaimer.bind(_st)
footnotes = _components.footnotes.bind(_st)
html = _components.html.bind(_st)
from boogie import models
from django.conf import settings
from django.utils.translation import ugettext as __, ugettext_lazy as _
from sidekick import delegate_to, lazy, import_later, placeholder as this
from ..enums import (
CommenterLevel,
HostLevel,
ProfileLevel,
ConversationLevel,
VoterLevel,
)
signals = import_later("..signals", package=__package__)
class ProgressBase(models.Model):
"""
Common features of all Progress models.
"""
score = models.PositiveSmallIntegerField(default=0)
score_bias = models.SmallIntegerField(default=0)
@classmethod
def level_fields(cls):
try:
return cls.__dict__["_level_fields"]
except KeyError:
fields = {}
for field in cls._meta.fields:
from ..mixins import (
Meta,
WithParamsMixin,
WithDataModelMixin,
WithInfoMixin,
WithResultsMixin,
WithRegionDemography,
)
from ..packages import plt
from ..utils import today, not_implemented, extract_keys, param_property
T = TypeVar("T")
NOW = datetime.datetime.now()
TODAY = datetime.date(NOW.year, NOW.month, NOW.day)
DAY = datetime.timedelta(days=1)
pplt = sk.import_later("..plot", package=__package__)
if TYPE_CHECKING:
from ..model_group import ModelGroup
from pydemic_ui.model import UIProperty
class Model(
WithDataModelMixin,
WithInfoMixin,
WithResultsMixin,
WithParamsMixin,
WithRegionDemography,
metaclass=ModelMeta,
):
"""
from itertools import chain
from logging import getLogger
from boogie.models import QuerySet, F, Manager, Value, IntegerField
from django.contrib.auth import get_user_model
from django.db import transaction
from sidekick import import_later
from ej_conversations.math import imputation
from ej_conversations.models import Conversation
from ..mixins import ClusterizationBaseMixin
pd = import_later("pandas")
np = import_later("numpy")
clusterization_pipeline = import_later(
"..math:clusterization_pipeline", package=__package__
)
models = import_later(".models", package=__package__)
log = getLogger("ej")
class ClusterQuerySet(ClusterizationBaseMixin, QuerySet):
"""
Represents a table of Cluster objects.
"""
clusters = lambda self: self
def users(self, by_comment=False):
"""
Queryset of users explicitly clusterized in the current cluster set.
import logging
import os
from pprint import pformat
from boogie.router import Router
from django.conf import settings
from django.contrib.auth import get_user_model
from django.contrib.flatpages.models import FlatPage
from django.http import Http404
from django.shortcuts import render, redirect
from django.utils.translation import ugettext as _
from sidekick import import_later, once
from ej.utils.flatpages import flat_page_route
conversations = import_later("ej_conversations.models:Conversation")
log = logging.getLogger("ej")
urlpatterns = Router(template="pages/{name}.jinja2")
#
# Views
#
@urlpatterns.route("")
def index(request):
if request.user.is_authenticated:
return redirect(settings.EJ_USER_HOME_PATH)
else:
return redirect(settings.EJ_ANONYMOUS_HOME_PATH)
@urlpatterns.route("start/")
import os
import bs4
import django
import sidekick as sk
from django.apps import apps
from django.test.client import Client
from hyperpython import Blob # noqa: F401
from sidekick import deferred, import_later, namespace
# Pydata
pd = import_later("pandas")
np = import_later("numpy")
plt = import_later("matplotlib.pyplot")
# Scikit learn
pca = import_later("sklearn.decomposition.pca")
svm = import_later("sklearn.svm")
decomposition = import_later("sklearn.decomposition")
model_selection = import_later("sklearn.model_selection")
preprocessing = import_later("sklearn.preprocessing")
impute = import_later("sklearn.impute")
# Start django
os.environ.setdefault("DJANGO_SETTINGS_MODEL", "ej.settings")
django.setup()
#
# Django imports
#
from boogie.models import F, Q, Sum, Max, Min # noqa: E402
import sidekick as sk
from .queryset import QuerySet
wc = sk.import_later("wordcloud")
class WordCloudQuerySet(QuerySet):
"""
A queryset that exposes a word_cloud() method to display a word cloud from
some fields extracted from a queryset.
"""
def word_cloud(self, *fields, **kwargs):
"""
Return an word cloud from the given fields spanning all values in the
queryset.
See Also:
see :func:`word_cloud` for additional options.
"""
return word_cloud(self, fields=fields, **kwargs)
def word_cloud(qs, fields=(), lang="en", sep="\n", stop_words=None, **kwargs):
"""
Creates a word-cloud object from queryset.
Args:
qs:
Queryset
fields:
import sidekick as sk
models = sk.import_later('.models', package=__package__)
powers = sk.deferred(lambda: models.ConversationPowers.objects)
def promote_comment(comment, user=None):
"""
Promotes a comment.
If user is given, register that the given user contributed with the
comment promotion.
"""
conversation = comment.conversation
if user:
powers.incr_by(user, conversation, promote_actions=1)
if comment.author != user:
powers.incr_by(comment.author, conversation, promoted_comments=1)
if not comment.is_promoted:
comment.is_promoted = True
comment.save(update_fields=['is_promoted'])
models.CommentPromotion.objects.get_or_create(user=user, comment=comment)
import functools
from django.contrib.contenttypes.models import ContentType
from django.db import models
from django.utils.translation import ugettext_lazy as _
from jsonfield import JSONField
from model_utils import Choices
from model_utils.models import StatusModel, TimeStampedModel
from sidekick import import_later
from time import sleep
from boogie.apps.tasks.manager import TaskQuerySet
from boogie.apps.tasks.settings import REGISTERED_FUNCTIONS, USE_CELERY, log
pd = import_later("pandas")
class TaskMeta(type(models.Model)):
"""
Metaclass that register @task-decorated methods as tasks.
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if not self._meta.abstract:
for attr in dir(self):
method = getattr(self, attr, None)
if getattr(method, "is_task", False):
self.register(name=attr, function=method)
from django.test import Client as DjangoClient
from sidekick import import_later
bs4 = import_later("bs4")
Text = import_later("hyperpython:Text")
class Client(DjangoClient):
def get_data(self, *args, fix_links=False, **kwargs):
"""
Makes a get request and return result as a raw string of data instead
of the usual response object.
"""
response = self.get(*args, **kwargs)
if getattr(response, "url", None):
return self.get_data(response.url, fix_links=fix_links)
if fix_links:
soup = self.get_soup(*args, fix_links=True, **kwargs)
return str(soup)
return response.content.decode(response.charset)
def get_html(self, *args, **kwargs):
"""
Like .get_data(), but converts the result into an hyperpython Text()
instance.
Text instances are rendered as HTML in Jupyter Notebooks.
"""
return Text(self.get_data(*args, **kwargs), escape=False)
from django.contrib.auth import get_user_model
from django.db.models import Count, Q
from sidekick import import_later
from sidekick import property as property
from .comment import Comment
models = import_later("ej_conversations.models")
def make_clean(cls, commit=True, **kwargs):
"""
Create an instance of the cls model (or execute the cls callable) and
call the resulting object .full_clean() method and later decide to save it
(if commit=True) or not.
"""
obj = cls(**kwargs)
obj.full_clean()
if commit:
obj.save()
return obj
def patch_user_model(model):
def conversations_with_votes(user):
return models.Conversation.objects.filter(comments__votes__author=user).distinct()
model.conversations_with_votes = property(conversations_with_votes)
from sidekick import import_later
from boogie.models import F
pd = import_later('pandas')
class TestQuerySetIndexing:
def test_django_slicing_still_works(self, authors):
assert authors[0] == authors.first()
assert authors[-1] == authors.last()
assert list(authors[0:2]) == [authors[0], authors[1]]
def test_simple_2d_slicing(self, authors):
author = authors.get(pk=1)
assert authors[1, :] == author
assert authors[1, 'name'] == author.name
assert authors[1, ['name', 'age']] == [author.name, author.age]
def test_2d_slicing_of_querysets(self, authors):
assert_equal(authors[:, :], authors.all())
assert_equal(authors[:, 'name'], authors.values_list('name',
flat=True))
assert_equal(authors[:, ['name', 'age']],
authors.values_list('name', 'age'))
def test_F_expression_slicing(self, authors):
assert_equal(authors[F.age > 25, 'name'],
authors.filter(age__gt=25).values_list('name', flat=True))
assert_equal(authors[F.age > 25, F.name],
authors.filter(age__gt=25).values_list('name', flat=True))
import re
from markupsafe import Markup
from sidekick import import_later
lxml_html = import_later("lxml.html")
lxml_etree = import_later("lxml.etree")
head_children = re.compile(r"^<(?:title|meta|script|style|link)")
def render_pretty(source, raw=False):
"""
Pretty prints HTML source or element.
Returns a Markup strings.
"""
if not isinstance(source, str):
source = source.render()
root = lxml_html.fromstring(source)
pretty = lxml_etree.tostring(root, encoding="unicode", pretty_print=True)
# lxml adds <head> and <html> tags if the root tag should be
if source.startswith("<head"):
pretty = dedent(pretty[7:-8], 2)
elif head_children.match(source):
pretty = dedent(pretty[16:-18], 4)
if raw:
return pretty
from sidekick import import_later
from boogie.rest import rest_api
from ej_conversations.models import Conversation
from . import models
from .utils import cluster_shapes
math = import_later(".math", package=__package__)
@rest_api.link(Conversation, name="clusterization")
def clusterization_link(request, conversation):
try:
clusterization = conversation.clusterization
except models.Clusterization.DoesNotExist:
return None
else:
return rest_api.get_hyperlink(clusterization, request)
#
# Cluster info
#
@rest_api.detail_action(models.Clusterization)
def clusters(clusterization):
return clusterization.clusters.all()
@rest_api.detail_action(models.Clusterization)
def affinities(request, clusterization):
return cluster_shapes(clusterization, user=request.user)
import sidekick as _sk
default_app_config = 'ej_clusters.apps.EjClustersConfig'
update_clusters = _sk.import_later('.factories:update_clusters',
package=__package__)
set_clusters_from_comments = _sk.import_later(
'.factories:set_clusters_from_comments', package=__package__)
from sidekick import import_later
__version__ = '0.1.0b'
_db = import_later('.models', package=__package__)
default_app_config = 'ej_conversations.apps.EjConversationsConfig'
def create_conversation(text,
title,
author,
*,
is_promoted=False,
tags=(),
commit=True):
"""
Creates a new conversation object and saves it in the database.
"""
conversation = _db.Conversation(text=text,
title=title,
author=author,
is_promoted=is_promoted)
conversation.clean()
if commit:
conversation.save()
return conversation
from logging import getLogger
from django.core.exceptions import ValidationError
from django.http import Http404
from django.utils.translation import ugettext_lazy as _
from hyperpython import a
from sidekick import import_later
from ej.components.builtins import toast
log = getLogger("ej")
models = import_later(".models", package=__package__)
forms = import_later(".forms", package=__package__)
#
# Functions
#
def request_comes_from_ej_bot(request):
return request_promoted_conversations(request) and (request.GET.get("participation_source") == "bot")
def request_promoted_conversations(request):
return request.GET.get("is_promoted") == "true"
def check_promoted(conversation, request):
"""
Raise a Http404 if conversation is not promoted
from boogie.fields import EnumField
from boogie.rest import rest_api
from django.apps import apps
from django.conf import settings
from django.contrib.auth import get_user_model
from django.contrib.staticfiles.storage import staticfiles_storage
from django.db import models
from django.urls import reverse
from django.utils.translation import ugettext_lazy as _, ugettext as __
from rest_framework.authtoken.models import Token
from sidekick import delegate_to, import_later
from .enums import Race, Gender
from .utils import years_from
SocialAccount = import_later("allauth.socialaccount.models:SocialAccount")
User = get_user_model()
@rest_api(exclude=["user"])
class Profile(models.Model):
"""
User profile
"""
user = models.OneToOneField(User,
on_delete=models.CASCADE,
related_name="profile")
race = EnumField(Race, _("Race"), default=Race.NOT_FILLED)
ethnicity = models.CharField(_("Ethnicity"), blank=True, max_length=50)
education = models.CharField(_("Education"), blank=True, max_length=140)
from hyperpython import a
from sidekick import import_later
urls = import_later("django.urls")
class Url(str):
"""
A specialized string class that represents urls.
"""
def __init__(self, url):
self.url = str(url)
def __truediv__(self, other):
try:
if self.endswith("/") or other.startswith("/"):
sep = ""
else:
sep = "/"
return Url(self.url + sep + str(other))
except AttributeError:
return NotImplementedError
def __repr__(self):
return f"Url({self.url!r})"
def anchor(self, name, **kwargs):
"""
Return an Hyperpython anchor object using url as href.
import logging
import os
from pprint import pformat
from boogie.router import Router
from django.conf import settings
from django.contrib.auth import get_user_model
from django.contrib.flatpages.models import FlatPage
from django.http import Http404
from django.shortcuts import render, redirect
from django.utils.translation import ugettext as _
from sidekick import import_later, Proxy
from ej.utils.flatpages import flat_page_route
conversations = import_later('ej_conversations.models:Conversation')
log = logging.getLogger("ej")
urlpatterns = Router(template="pages/{name}.jinja2")
#
# Views
#
@urlpatterns.route("")
def index(request):
if request.user.is_authenticated:
return redirect(settings.EJ_USER_HOME_PATH)
else:
return redirect(settings.EJ_ANONYMOUS_HOME_PATH)
def loader():
reload(mod)
return sk.import_later(mod)
import re
from html import unescape
from random import choice
from string import ascii_letters, digits
from markupsafe import Markup, escape, escape_silent
from sidekick import import_later
# qa: used imports
bleach = import_later("bleach")
escape = escape
escape_silent = escape_silent
unescape = unescape
VALID_ID_CHARS = ascii_letters + digits + "_-"
STR_TYPES = (str, bytes, Markup)
SAFE_ATTRIBUTE_NAME = re.compile(r"^[^\s=<>&\"\']+$")
def dash_case(name):
"""
Convert a camel case string to dash case.
Example:
>>> dash_case('SomeName')
'some-name'
"""
letters = []
for c in name:
if c.isupper() and letters and letters[-1] != "-":
letters.append("-" + c.lower())
from typing import Callable, Sequence
from boogie.router import Router
from django.apps import apps
from django.conf import settings
from django.contrib.auth import get_user_model
from django.http import JsonResponse
from django.utils.translation import ugettext_lazy as _, ugettext as __
from sidekick import import_later
from ej_conversations.models import Conversation
from ej_conversations.routes import conversation_url
from ej_conversations.utils import check_promoted
from ej_profiles.enums import Gender, Race, STATE_CHOICES
np = import_later("numpy")
wordcloud = import_later("wordcloud")
stop_words = import_later("stop_words")
pd = import_later("pandas")
log = getLogger("ej")
urlpatterns = Router(
base_path=conversation_url,
template="ej_dataviz/{name}.jinja2",
models={"conversation": Conversation},
login=True,
)
app_name = "ej_dataviz"
User = get_user_model()
import logging
from collections import defaultdict, Counter
from sidekick import import_later
log = logging.getLogger("ej")
np = import_later("numpy")
models = import_later("..models", package=__name__)
#
# Cluster belonging fractions
#
def compute_cluster_affinities(votes,
distance=lambda x, y: np.sum(np.abs(x - y))):
"""
Returns a dictionary mapping clusters to a list of affinities.
Each affinity is another dictionary mapping clusters to the degree of
affinity for the user in each other cluster.
Args:
votes (dataframe):
A votes dataframe with users as rows and comments as columns. It
must contain an extra column named 'cluster' indicating in which
cluster each user is classified.
Usually this data will come from a call to ``clusterization.clusters.votes_table()``
distance (callable):
Distance function.
"""
import pytest
from numpy.testing import assert_almost_equal, assert_equal
from sidekick import import_later
from ej_clusters.math import kmeans
np = import_later("numpy")
# A very easy dataset with k=2
STEREOTYPES = np.array([[1, 1, 1], [-1, -1, -1]], dtype=float)
DATA = np.array(
[[1, 0, 1], [1, 1, 1], [0, 1, 1], [-1, 0, -1], [-1, -1, 0], [-1, -1, -1]],
dtype=float,
)
class TestAuxiliaryMathFunctions:
@pytest.fixture
def a(self):
return STEREOTYPES[0].copy()
@pytest.fixture
def b(self):
return STEREOTYPES[1].copy()
def test_euclidean_distance(self, a, b):
assert kmeans.euclidean_distance(a, a) == 0
assert kmeans.euclidean_distance(b, b) == 0
assert_almost_equal(kmeans.euclidean_distance(a, b), np.sqrt(12))
def test_l1_distance(self, a, b):
def profile_class():
"""
Return a profile instance for user.
"""
return import_later('ej_profiles.models:Profile')
