def get_dates(decks: QuerySet) -> List[date]:
"""
Gets the unique list of dates for the given list of decks
:param decks: The list of decks to get te date for
:return: The list of dates
"""
return list(decks.values_list("date_created", flat=True).distinct())
def breakdown_by_name(queryset: QuerySet) -> Dict[str, int]:
"""Convert logs queryset into {name: count} dictionary."""
return {
x[0]: x[1]
for x in queryset.values_list("command_name")
.annotate(count=Count("id"))
.order_by("command_name")
}
def _where_node_leaf_callback(self, node, negated, new_parent, connection, model, compiler):
new_node = WhereNode()
def convert_rhs_op(node):
db_rhs = getattr(node.rhs, '_db', None)
if db_rhs is not None and db_rhs != connection.alias:
raise ValueError(
"Subqueries aren't allowed across different databases. Force "
"the inner query to be evaluated using `list(inner_query)`."
)
value = node.get_rhs_op(connection, node.rhs)
operator = value.split()[0].lower().strip()
if operator == 'between':
operator = 'range'
return operator
if not hasattr(node, "lhs"):
raise NotSupportedError("Attempted probable subquery, these aren't supported on the Datastore")
# Don't call on querysets
if not hasattr(node.rhs, "_as_sql") and not isinstance(node.rhs, DjangoQuery):
try:
# Although we do nothing with this. We need to call it as many lookups
# perform validation etc.
node.process_rhs(compiler, connection)
except EmptyResultSet:
if node.lookup_name == 'in':
node.rhs = []
else:
raise
# Leaf
if hasattr(node.lhs, 'target'):
# from Django 1.9, some node.lhs might not have a target attribute
# as they might be wrapping date fields
field = node.lhs.target
operator = convert_rhs_op(node)
elif isinstance(node.lhs, Aggregate):
raise NotSupportedError("Aggregate filters are not supported on the Datastore")
else:
field = node.lhs.lhs.target
operator = convert_rhs_op(node)
# This deals with things like datefield__month__gt=X which means from this point
# on, operator will have two parts in that particular case and will probably need to
# be dealt with by a special indexer
if node.lookup_name != node.lhs.lookup_name:
operator = "{}__{}".format(node.lhs.lookup_name, node.lookup_name)
if get_top_concrete_parent(field.model) != get_top_concrete_parent(model):
raise NotSupportedError("Cross-join where filters are not supported on the Datastore")
# Make sure we don't let people try to filter on a text field, otherwise they just won't
# get any results!
if field.db_type(connection) in ("bytes", "text"):
raise NotSupportedError("You can't filter on text or blob fields on the Datastore")
if operator == "isnull" and field.model._meta.parents.values():
raise NotSupportedError("isnull lookups on inherited relations aren't supported on the Datastore")
lhs = field.column
if hasattr(node.rhs, "get_compiler"):
if len(node.rhs.select) == 1:
# In Django >= 1.11 this is a values list type query, which we explicitly handle
# because of the common case of pk__in=Something.objects.values_list("pk", flat=True)
qs = QuerySet(query=node.rhs, using=self.connection.alias)
# We make the query for the values, but wrap in a list to trick the
# was_iter code below. This whole set of if/elif statements needs rethinking!
rhs = [list(qs.values_list("pk", flat=True))]
else:
# This is a subquery
raise NotSupportedError("Attempted to run a subquery on the Datastore")
elif isinstance(node.rhs, ValuesListQuerySet):
# We explicitly handle ValuesListQuerySet because of the
# common case of pk__in=Something.objects.values_list("pk", flat=True)
# this WILL execute another query, but that is to be expected on a
# non-relational database.
rhs = [x for x in node.rhs] # Evaluate the queryset
elif isinstance(node.rhs, QuerySet):
# In Django 1.9, ValuesListQuerySet doesn't exist anymore, and instead
# values_list returns a QuerySet
if node.rhs._iterable_class == FlatValuesListIterable:
# if the queryset has FlatValuesListIterable as iterable class
# then it's a flat list, and we just need to evaluate the
# queryset converting it into a list
rhs = [x for x in node.rhs]
else:
# otherwise, we try to get the PK from the queryset
rhs = list(node.rhs.values_list('pk', flat=True))
else:
rhs = node.rhs
was_iter = hasattr(node.rhs, "__iter__")
rhs = node.get_db_prep_lookup(rhs, connection)[-1]
if rhs and not was_iter and hasattr(rhs, "__iter__"):
rhs = rhs[0]
new_node.set_leaf(
lhs,
operator,
rhs,
is_pk_field=field==model._meta.pk,
negated=negated,
lookup_name=node.lookup_name,
namespace=connection.ops.connection.settings_dict.get("NAMESPACE"),
target_field=field,
)
# For some reason, this test:
# test_update_with_related_manager (get_or_create.tests.UpdateOrCreateTests)
# ends up with duplicate nodes in the where tree. I don't know why. But this
# weirdly causes the Datastore query to return nothing.
# so here we don't add duplicate nodes, I can't think of a case where that would
# change the query if it's under the same parent.
if new_node in new_parent.children:
return
new_parent.children.append(new_node)
def _where_node_leaf_callback(self, node, negated, new_parent, connection, model, compiler):
new_node = WhereNode(new_parent.using)
def convert_rhs_op(node):
db_rhs = getattr(node.rhs, "_db", None)
if db_rhs is not None and db_rhs != connection.alias:
raise ValueError(
"Subqueries aren't allowed across different databases. Force "
"the inner query to be evaluated using `list(inner_query)`."
)
value = node.get_rhs_op(connection, node.rhs)
operator = value.split()[0].lower().strip()
if operator == "between":
operator = "range"
return operator
if not hasattr(node, "lhs"):
raise NotSupportedError("Attempted probable subquery, these aren't supported on the Datastore")
# Don't call on querysets
if not hasattr(node.rhs, "_as_sql") and not isinstance(node.rhs, DjangoQuery):
try:
# Although we do nothing with this. We need to call it as many lookups
# perform validation etc.
node.process_rhs(compiler, connection)
except EmptyResultSet:
if node.lookup_name == "in":
node.rhs = []
else:
raise
# Leaf
if hasattr(node.lhs, "target"):
# from Django 1.9, some node.lhs might not have a target attribute
# as they might be wrapping date fields
field = node.lhs.target
operator = convert_rhs_op(node)
elif isinstance(node.lhs, Aggregate):
raise NotSupportedError("Aggregate filters are not supported on the Datastore")
else:
field = node.lhs.lhs.target
operator = convert_rhs_op(node)
# This deals with things like datefield__month__gt=X which means from this point
# on, operator will have two parts in that particular case and will probably need to
# be dealt with by a special indexer
if node.lookup_name != node.lhs.lookup_name:
operator = "{}__{}".format(node.lhs.lookup_name, node.lookup_name)
if get_top_concrete_parent(field.model) != get_top_concrete_parent(model):
raise NotSupportedError("Cross-join where filters are not supported on the Datastore")
# Make sure we don't let people try to filter on a text field, otherwise they just won't
# get any results!
lookup_supports_text = getattr(node, "lookup_supports_text", False)
if field.db_type(connection) in ("bytes", "text") and not lookup_supports_text:
raise NotSupportedError("You can't filter on text or blob fields on the Datastore")
if operator == "isnull" and field.model._meta.parents.values():
raise NotSupportedError("isnull lookups on inherited relations aren't supported on the Datastore")
lhs = field.column
if hasattr(node.rhs, "get_compiler"):
if len(node.rhs.select) == 1:
# In Django >= 1.11 this is a values list type query, which we explicitly handle
# because of the common case of pk__in=Something.objects.values_list("pk", flat=True)
qs = QuerySet(query=node.rhs, using=self.connection.alias)
# We make the query for the values, but wrap in a list to trick the
# was_iter code below. This whole set of if/elif statements needs rethinking!
rhs = [list(qs.values_list("pk", flat=True))]
else:
# This is a subquery
raise NotSupportedError("Attempted to run a subquery on the Datastore")
elif isinstance(node.rhs, ValuesListQuerySet):
# We explicitly handle ValuesListQuerySet because of the
# common case of pk__in=Something.objects.values_list("pk", flat=True)
# this WILL execute another query, but that is to be expected on a
# non-relational database.
rhs = [x for x in node.rhs] # Evaluate the queryset
elif isinstance(node.rhs, QuerySet):
# In Django 1.9, ValuesListQuerySet doesn't exist anymore, and instead
# values_list returns a QuerySet
if node.rhs._iterable_class == FlatValuesListIterable:
# if the queryset has FlatValuesListIterable as iterable class
# then it's a flat list, and we just need to evaluate the
# queryset converting it into a list
rhs = [x for x in node.rhs]
else:
# otherwise, we try to get the PK from the queryset
rhs = list(node.rhs.values_list("pk", flat=True))
else:
rhs = node.rhs
was_iter = _iterable(node.rhs)
rhs = node.get_db_prep_lookup(rhs, connection)[-1]
if rhs and not was_iter and _iterable(rhs):
rhs = rhs[0]
new_node.set_leaf(
lhs,
operator,
rhs,
is_pk_field=field == model._meta.pk,
negated=negated,
lookup_name=node.lookup_name,
namespace=connection.ops.connection.settings_dict.get("NAMESPACE"),
target_field=field,
)
# For some reason, this test:
# test_update_with_related_manager (get_or_create.tests.UpdateOrCreateTests)
# ends up with duplicate nodes in the where tree. I don't know why. But this
# weirdly causes the Datastore query to return nothing.
# so here we don't add duplicate nodes, I can't think of a case where that would
# change the query if it's under the same parent.
if new_node in new_parent.children:
return
new_parent.children.append(new_node)
def _fetch_all(self):
"""
Fetch all uses the standard iterator but sorts the values on the
way out, this maintains the lazy evaluation of querysets
"""
def locate_pk_column(query):
pk_name = self.model._meta.pk.name
if django.VERSION >= (1, 9):
if "pk" in query.values_select:
return query.values_select.index("pk")
elif pk_name in query.values_select:
return query.values_select.index(pk_name)
else:
for i, col in enumerate(query.select):
if hasattr(col, "field") and col.field.name == pk_name:
return i
if self._result_cache is None:
# Making this work efficiently is tricky. We never want to fetch more than self.ordered_pks
# and the __getitem__ implementation sets this to a single item, so in that case we only want
# to fetch one. So we do a few things here:
# 1. We clone the Queryset as a normal Queryset, then we do a pk__in on the ordered_pks.
# 2. We add the primary key if this was a values_list query without it being specified, otherwise
# we can't match up the ordering
# 3. We execute the clone, and then remove the additional PK column from the result set
# There are various combinations to handle depending on whether it's a "flat" values_list or
# whether or not we added the PK manually to the result set
if django.VERSION >= (1, 9):
clone = QuerySet(model=self.model, query=self.query, using=self._db)
clone._iterable_class = self._iterable_class
else:
if isinstance(self, ValuesListQuerySet):
clone = ValuesListQuerySet(model=self.model, query=self.query, using=self._db)
clone._fields = self._fields
clone.field_names = self.field_names
clone.extra_names = self.extra_names
clone.annotation_names = self.annotation_names
clone.flat = self.flat
else:
clone = QuerySet(model=self.model, query=self.query, using=self._db)
clone = clone.filter(pk__in=self.ordered_pks)
pk_col = 0
pk_added = False
if django.VERSION >= (1, 9):
values_select = clone.query.values_select
else:
values_select = [x.field.name for x in clone.query.select]
if values_select:
pk_col = locate_pk_column(clone.query)
if pk_col is None:
# Manually add the PK to the result set
clone = clone.values_list(*(["pk"] + values_select))
values_select = [x.field.name for x in clone.query.select]
pk_col = 0
pk_added = True
# Hit the database
results = list(clone)
ordered_results = []
pk_hash = {}
if django.VERSION >= (1, 9):
flat = self._iterable_class == FlatValuesListIterable
else:
# On Django 1.8 things are different
flat = getattr(self, "flat", False)
for x in results:
if isinstance(x, models.Model):
# standard query case
pk_hash[x.pk] = x
elif len(values_select) == 1:
# Only PK case
pk_hash[x if flat else x[pk_col]] = x
else:
# Multiple columns (either passed in, or as a result of the PK being added)
if flat:
pk_hash[x[pk_col]] = x[1] if pk_added else x
else:
pk_hash[x[pk_col]] = x[1:] if pk_added else x
for pk in self.ordered_pks:
obj = pk_hash.get(pk)
if obj:
ordered_results.append(obj)
self._result_cache = ordered_results
if self._prefetch_related_lookups and not self._prefetch_done:
self._prefetch_related_objects()
def _fetch_all(self):
"""
Fetch all uses the standard iterator but sorts the values on the
way out, this maintains the lazy evaluation of querysets
"""
def locate_pk_column(query):
pk_name = self.model._meta.pk.name
if django.VERSION >= (1, 9):
if "pk" in query.values_select:
return query.values_select.index("pk")
elif pk_name in query.values_select:
return query.values_select.index(pk_name)
else:
for i, col in enumerate(query.select):
if hasattr(col, "field") and col.field.name == pk_name:
return i
if self._result_cache is None:
# Making this work efficiently is tricky. We never want to fetch more than self.ordered_pks
# and the __getitem__ implementation sets this to a single item, so in that case we only want
# to fetch one. So we do a few things here:
# 1. We clone the Queryset as a normal Queryset, then we do a pk__in on the ordered_pks.
# 2. We add the primary key if this was a values_list query without it being specified, otherwise
# we can't match up the ordering
# 3. We execute the clone, and then remove the additional PK column from the result set
# There are various combinations to handle depending on whether it's a "flat" values_list or
# whether or not we added the PK manually to the result set
if django.VERSION >= (1, 9):
clone = QuerySet(model=self.model,
query=self.query,
using=self._db)
clone._iterable_class = self._iterable_class
else:
if isinstance(self, ValuesListQuerySet):
clone = ValuesListQuerySet(model=self.model,
query=self.query,
using=self._db)
clone._fields = self._fields
clone.field_names = self.field_names
clone.extra_names = self.extra_names
clone.annotation_names = self.annotation_names
clone.flat = self.flat
else:
clone = QuerySet(model=self.model,
query=self.query,
using=self._db)
clone = clone.filter(pk__in=self.ordered_pks)
pk_col = 0
pk_added = False
if django.VERSION >= (1, 9):
values_select = clone.query.values_select
else:
values_select = [x.field.name for x in clone.query.select]
if values_select:
pk_col = locate_pk_column(clone.query)
if pk_col is None:
# Manually add the PK to the result set
clone = clone.values_list(*(["pk"] + values_select))
values_select = [x.field.name for x in clone.query.select]
pk_col = 0
pk_added = True
# Hit the database
results = list(clone)
ordered_results = []
pk_hash = {}
if django.VERSION >= (1, 9):
flat = self._iterable_class == FlatValuesListIterable
else:
# On Django 1.8 things are different
flat = getattr(self, "flat", False)
for x in results:
if isinstance(x, models.Model):
# standard query case
pk_hash[x.pk] = x
elif len(values_select) == 1:
# Only PK case
pk_hash[x if flat else x[pk_col]] = x
else:
# Multiple columns (either passed in, or as a result of the PK being added)
if flat:
pk_hash[x[pk_col]] = x[1] if pk_added else x
else:
pk_hash[x[pk_col]] = x[1:] if pk_added else x
for pk in self.ordered_pks:
obj = pk_hash.get(pk)
if obj:
ordered_results.append(obj)
self._result_cache = ordered_results
if self._prefetch_related_lookups and not self._prefetch_done:
self._prefetch_related_objects()
def calculateStats(searched_term: str, unprocessed_stats: QuerySet,
source: str, sentimentable_model: Model):
try:
term = search_term.objects.get(term=searched_term)
if unprocessed_stats.count() == 0:
return True
processed_weight = stat_processor.processDataWeight(
searched_term_id=term.id,
data_source=source,
new_record_count=unprocessed_stats.count(),
new_data=False)
unprocessed_weight = stat_processor.processDataWeight(
searched_term_id=term.id,
data_source=source,
new_record_count=unprocessed_stats.count())
new_neutral_avg = round(
unprocessed_stats.aggregate(Avg('nlp_neutral_sentiment'))
['nlp_neutral_sentiment__avg'], 5)
new_mixed_avg = round(
unprocessed_stats.aggregate(
Avg('nlp_mixed_sentiment'))['nlp_mixed_sentiment__avg'], 5)
new_positive_avg = round(
unprocessed_stats.aggregate(Avg('nlp_positive_sentiment'))
['nlp_positive_sentiment__avg'], 5)
new_negative_avg = round(
unprocessed_stats.aggregate(Avg('nlp_negative_sentiment'))
['nlp_negative_sentiment__avg'], 5)
if processed_weight == 0:
with transaction.atomic():
sentiment_stat = pie_chart_sentiment_stat(
neutral_sentiment_aggregate=new_neutral_avg,
mixed_sentiment_aggregate=new_mixed_avg,
positive_sentiment_aggregate=new_positive_avg,
negative_sentiment_aggregate=new_negative_avg,
processed_records_count=unprocessed_stats.count(),
term_id=term.id,
data_source=source)
sentiment_stat.save()
sentimentable_model.objects.filter(id__in=list(
unprocessed_stats.values_list(
'id', flat=True))).update(pie_stat_processed=True)
return True
else:
try:
with transaction.atomic():
old_stats = pie_chart_sentiment_stat.objects.get(
term_id=term.id, data_source=source)
old_stats.neutral_sentiment_aggregate = (
processed_weight *
old_stats.neutral_sentiment_aggregate) + (
unprocessed_weight * new_neutral_avg)
old_stats.mixed_sentiment_aggregate = (
processed_weight *
old_stats.mixed_sentiment_aggregate) + (
unprocessed_weight * new_mixed_avg)
old_stats.positive_sentiment_aggregate = (
processed_weight *
old_stats.positive_sentiment_aggregate) + (
unprocessed_weight * new_positive_avg)
old_stats.negative_sentiment_aggregate = (
processed_weight *
old_stats.negative_sentiment_aggregate) + (
unprocessed_weight * new_negative_avg)
old_stats.processed_records_count = old_stats.processed_records_count + unprocessed_stats.count(
)
old_stats.save()
sentimentable_model.objects.filter(id__in=list(
unprocessed_stats.values_list('id', flat=True)
)).update(pie_stat_processed=True)
return True
except ObjectDoesNotExist as ex_stat:
print(
'Error! The term ' + searched_term +
' does not exist when looking for existing aggregate stats! Exception: '
+ str(ex_stat))
logging.error(
'Error! The term ' + searched_term +
' does not exist when looking for existing aggregate stats! Exception: '
+ str(ex_stat))
return False
except ObjectDoesNotExist as ex_term:
print(
'Error! The term ' + searched_term +
' does not exist when searching for terms in the term table! Exception: '
+ str(ex_term))
logging.error(
'Error! The term ' + searched_term +
' does not exist when searching for terms in the term table! Exception: '
+ str(ex_term))
return False
def send_test_emails(
self, request: HttpRequest, queryset: QuerySet
) -> HttpResponseRedirect:
selected = ",".join([str(s) for s in queryset.values_list("id", flat=True)])
url = "{}?templates={}".format(reverse("appmail:send_test_email"), selected)
return HttpResponseRedirect(url)
def queryset_to_list(queryset: QuerySet):
if isinstance(queryset, QuerySet):
list_as_string = sorted(queryset.values_list("slug", flat=True))
if "unknown" in list_as_string:
list_as_string.remove("unknown")
return list_as_string
