def add_special_index(model_class, field_name, index_type, value=None):
from djangae.utils import on_production, in_testing
from django.conf import settings
indexer = REQUIRES_SPECIAL_INDEXES[index_type]
index_type = indexer.prepare_index_type(index_type, value)
field_name = field_name.encode(
"utf-8") # Make sure we are working with strings
load_special_indexes()
if special_index_exists(model_class, field_name, index_type):
return
if on_production() or (in_testing() and not getattr(
settings, "GENERATE_SPECIAL_INDEXES_DURING_TESTING", False)):
raise RuntimeError(
"There is a missing index in your djangaeidx.yaml - \n\n{0}:\n\t{1}: [{2}]"
.format(_get_table_from_model(model_class), field_name,
index_type))
_special_indexes.setdefault(_get_table_from_model(model_class),
{}).setdefault(field_name,
[]).append(str(index_type))
write_special_indexes()
def fix_sandbox(self):
"""
This is the nastiest thing in the world...
This WSGI middleware is the first chance we get to hook into anything. On the dev_appserver
at this point the Python sandbox will have already been initialized. The sandbox replaces stuff
like the subprocess module, and the select module. As well as disallows _sqlite3. These things
are really REALLY useful for development.
So here we dismantle parts of the sandbox. Firstly we add _sqlite3 to the allowed C modules.
Next, we manipulate the sandbox to allow the default python subprocess and select modules to function.
This only happens on the dev_appserver, it would only die on live. Everything is checked so that
changes are only made if they haven't been made already.
"""
if on_production():
return
from google.appengine.tools.devappserver2.python import sandbox
if '_sqlite3' not in sandbox._WHITE_LIST_C_MODULES:
sandbox._WHITE_LIST_C_MODULES.extend([
'_sqlite3'
])
#Fix up the subprocess module
self.fix_subprocess_module()
def fix_sandbox(self):
"""
This is the nastiest thing in the world...
This WSGI middleware is the first chance we get to hook into anything. On the dev_appserver
at this point the Python sandbox will have already been initialized. The sandbox replaces stuff
like the subprocess module, and the select module. As well as disallows _sqlite3. These things
are really REALLY useful for development.
So here we dismantle parts of the sandbox. Firstly we add _sqlite3 to the allowed C modules.
Next, we manipulate the sandbox to allow the default python subprocess and select modules to function.
This only happens on the dev_appserver, it would only die on live. Everything is checked so that
changes are only made if they haven't been made already.
"""
if on_production():
return
from google.appengine.tools.devappserver2.python import sandbox
if '_sqlite3' not in sandbox._WHITE_LIST_C_MODULES:
sandbox._WHITE_LIST_C_MODULES.extend([
'_sqlite3'
])
#Fix up the subprocess module
self.fix_subprocess_module()
def deferred(request):
from google.appengine.ext.deferred.deferred import (
run,
SingularTaskFailure,
PermanentTaskFailure
)
response = HttpResponse()
if 'HTTP_X_APPENGINE_TASKEXECUTIONCOUNT' in request.META:
logging.debug("[DEFERRED] Retry %s of deferred task", request.META['HTTP_X_APPENGINE_TASKEXECUTIONCOUNT'])
if 'HTTP_X_APPENGINE_TASKNAME' not in request.META:
logging.critical('Detected an attempted XSRF attack. The header "X-AppEngine-Taskname" was not set.')
response.status_code = 403
return response
in_prod = on_production()
if in_prod and os.environ.get("REMOTE_ADDR") != "0.1.0.2":
logging.critical('Detected an attempted XSRF attack. This request did not originate from Task Queue.')
response.status_code = 403
return response
try:
run(request.body)
except SingularTaskFailure:
logging.debug("Failure executing task, task retry forced")
response.status_code = 408
except PermanentTaskFailure:
logging.exception("Permanent failure attempting to execute task")
return response
def fix_sandbox():
"""
This is the nastiest thing in the world...
This WSGI middleware is the first chance we get to hook into anything. On the dev_appserver
at this point the Python sandbox will have already been initialized. The sandbox replaces stuff
like the subprocess module, and the select module. As well as disallows _sqlite3. These things
are really REALLY useful for development.
So here we dismantle parts of the sandbox. Firstly we add _sqlite3 to the allowed C modules.
This only happens on the dev_appserver, it would only die on live. Everything is checked so that
changes are only made if they haven't been made already.
"""
if on_production():
return
from google.appengine.tools.devappserver2.python import sandbox
if '_sqlite3' not in sandbox._WHITE_LIST_C_MODULES:
fix_c_whitelist()
# Reload the system socket.py, because of bug #9246
import imp
import os
import ast
psocket = os.path.join(os.path.dirname(ast.__file__), 'socket.py')
imp.load_source('socket', psocket)
def fix_sandbox():
"""
This is the nastiest thing in the world...
This WSGI middleware is the first chance we get to hook into anything. On the dev_appserver
at this point the Python sandbox will have already been initialized. The sandbox replaces stuff
like the subprocess module, and the select module. As well as disallows _sqlite3. These things
are really REALLY useful for development.
So here we dismantle parts of the sandbox. Firstly we add _sqlite3 to the allowed C modules.
This only happens on the dev_appserver, it would only die on live. Everything is checked so that
changes are only made if they haven't been made already.
"""
if on_production():
return
from google.appengine.tools.devappserver2.python import sandbox
if '_sqlite3' not in sandbox._WHITE_LIST_C_MODULES:
fix_c_whitelist()
# Reload the system socket.py, because of bug #9246
import imp
import os
import ast
psocket = os.path.join(os.path.dirname(ast.__file__), 'socket.py')
imp.load_source('socket', psocket)
def add_special_index(model_class, field_name, index_type, value=None):
from djangae.utils import on_production, in_testing
from django.conf import settings
indexer = REQUIRES_SPECIAL_INDEXES[index_type]
index_type = indexer.prepare_index_type(index_type, value)
field_name = field_name.encode("utf-8") # Make sure we are working with strings
load_special_indexes()
if special_index_exists(model_class, field_name, index_type):
return
if on_production() or (in_testing() and not getattr(settings, "GENERATE_SPECIAL_INDEXES_DURING_TESTING", False)):
raise RuntimeError(
"There is a missing index in your djangaeidx.yaml - \n\n{0}:\n\t{1}: [{2}]".format(
_get_table_from_model(model_class), field_name, index_type
)
)
_special_indexes.setdefault(
_get_table_from_model(model_class), {}
).setdefault(field_name, []).append(str(index_type))
write_special_indexes()
def fix_subprocess_module(self):
"""
Making the subprocess module work on the dev_appserver is hard work
so I've isolated it all here
"""
if on_production():
return
# TODO: Remove support for the subprocess module.
warnings.warn(
(
"Subprocess support on dev_appserver will be removed soon. "
"Mediagenerator/assetpipe users should find "
"alternatives which run in a separate process."
),
DeprecationWarning
)
import sys
from google.appengine import dist27
from google.appengine.tools.devappserver2.python import sandbox
if 'fcntl' not in sandbox._WHITE_LIST_C_MODULES:
sandbox._WHITE_LIST_C_MODULES.extend([
'fcntl'
])
if "subprocess" in dist27.MODULE_OVERRIDES:
dist27.MODULE_OVERRIDES.remove("subprocess")
if "subprocess" in sys.modules:
del sys.modules["subprocess"]
for finder in sys.meta_path:
if isinstance(finder, sandbox.ModuleOverrideImportHook):
del finder.policies['os']
if "os" in sys.modules:
del sys.modules["os"]
request_environment.PatchOsEnviron(__import__("os"))
if "select" in dist27.MODULE_OVERRIDES:
dist27.MODULE_OVERRIDES.remove("select")
if "select" not in sandbox._WHITE_LIST_C_MODULES:
sandbox._WHITE_LIST_C_MODULES.extend(["select"])
if "select" in sys.modules:
del sys.modules["select"]
def _convert_ordering(query):
if not query.default_ordering:
result = query.order_by
else:
result = query.order_by or query.get_meta().ordering
if result:
# We factor out cross-table orderings (rather than raising NotSupportedError) otherwise we'll break
# the admin which uses them. We log a warning when this happens though
try:
ordering = []
for name in result:
if name == "?":
raise NotSupportedError(
"Random ordering is not supported on the datastore")
if not (isinstance(name, basestring) and "__" in name):
if isinstance(name, basestring):
if name.lstrip("-") == "pk":
field_column = query.model._meta.pk.column
else:
field_column = query.model._meta.get_field(
name.lstrip("-")).column
ordering.append(field_column if not name.startswith(
"-") else "-{}".format(field_column))
else:
ordering.append(name)
except FieldDoesNotExist:
opts = query.model._meta
available = opts.get_all_field_names()
raise FieldError("Cannot resolve keyword %r into field. "
"Choices are: %s" % (name, ", ".join(available)))
if len(ordering) < len(result):
diff = set(result) - set(ordering)
log_once(
DJANGAE_LOG.warning
if not on_production() else DJANGAE_LOG.debug,
"The following orderings were ignored as cross-table orderings are not supported on the datastore: %s",
diff)
result = ordering
return result
def add_special_index(model_class, field_name, index_type):
from djangae.utils import on_production, in_testing
from django.conf import settings
load_special_indexes()
if special_index_exists(model_class, field_name, index_type):
return
if on_production() or (in_testing() and not getattr(
settings, "GENERATE_SPECIAL_INDEXES_DURING_TESTING", False)):
raise RuntimeError(
"There is a missing index in your djangaeidx.yaml - \n\n{0}:\n\t{1}: [{2}]"
.format(_get_table_from_model(model_class), field_name,
index_type))
_special_indexes.setdefault(_get_table_from_model(model_class),
{}).setdefault(field_name,
[]).append(str(index_type))
write_special_indexes()
def add_special_index(model_class, field_name, index_type):
from djangae.utils import on_production, in_testing
from django.conf import settings
load_special_indexes()
if special_index_exists(model_class, field_name, index_type):
return
if on_production() or (in_testing() and not getattr(settings, "GENERATE_SPECIAL_INDEXES_DURING_TESTING", False)):
raise RuntimeError(
"There is a missing index in your djangaeidx.yaml - \n\n{0}:\n\t{1}: [{2}]".format(
_get_table_from_model(model_class), field_name, index_type
)
)
_special_indexes.setdefault(
_get_table_from_model(model_class), {}
).setdefault(field_name, []).append(str(index_type))
write_special_indexes()
def _convert_ordering(query):
if not query.default_ordering:
result = query.order_by
else:
result = query.order_by or query.get_meta().ordering
if result:
# We factor out cross-table orderings (rather than raising NotSupportedError) otherwise we'll break
# the admin which uses them. We log a warning when this happens though
ordering = [ x for x in result if not (isinstance(x, basestring) and "__" in x) ]
if len(ordering) < len(result):
diff = set(result) - set(ordering)
log_once(
DJANGAE_LOG.warning if not on_production() else DJANGAE_LOG.debug,
"The following orderings were ignored as cross-table orderings are not supported on the datastore: %s", diff
)
result = ordering
if "?" in result:
raise NotSupportedError("Random ordering is not supported on the datastore")
return result
def fix_subprocess_module(self):
"""
Making the subprocess module work on the dev_appserver is hard work
so I've isloated it all here
"""
if on_production():
return
import sys
from google.appengine import dist27
from google.appengine.tools.devappserver2.python import sandbox
if 'fcntl' not in sandbox._WHITE_LIST_C_MODULES:
sandbox._WHITE_LIST_C_MODULES.extend([
'fcntl'
])
if "subprocess" in dist27.MODULE_OVERRIDES:
dist27.MODULE_OVERRIDES.remove("subprocess")
if "subprocess" in sys.modules:
del sys.modules["subprocess"]
for finder in sys.meta_path:
if isinstance(finder, sandbox.ModuleOverrideImportHook):
del finder.policies['os']
if "os" in sys.modules:
del sys.modules["os"]
request_environment.PatchOsEnviron(__import__("os"))
if "select" in dist27.MODULE_OVERRIDES:
dist27.MODULE_OVERRIDES.remove("select")
if "select" not in sandbox._WHITE_LIST_C_MODULES:
sandbox._WHITE_LIST_C_MODULES.extend(["select"])
if "select" in sys.modules:
del sys.modules["select"]
def deferred(request):
from google.appengine.ext.deferred.deferred import (run,
SingularTaskFailure,
PermanentTaskFailure)
response = HttpResponse()
if 'HTTP_X_APPENGINE_TASKEXECUTIONCOUNT' in request.META:
logging.debug("[DEFERRED] Retry %s of deferred task",
request.META['HTTP_X_APPENGINE_TASKEXECUTIONCOUNT'])
if 'HTTP_X_APPENGINE_TASKNAME' not in request.META:
logging.critical(
'Detected an attempted XSRF attack. The header "X-AppEngine-Taskname" was not set.'
)
response.status_code = 403
return response
in_prod = on_production()
if in_prod and os.environ.get("REMOTE_ADDR") != "0.1.0.2":
logging.critical(
'Detected an attempted XSRF attack. This request did not originate from Task Queue.'
)
response.status_code = 403
return response
try:
run(request.body)
except SingularTaskFailure:
logging.debug("Failure executing task, task retry forced")
response.status_code = 408
except PermanentTaskFailure:
logging.exception("Permanent failure attempting to execute task")
return response
"""
WSGI config for scaffold project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/
"""
from scaffold.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "scaffold.settings.production" if on_production() else "scaffold.settings.dev"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
"""
WSGI config for feedapp project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/
"""
from feedapp.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "feedapp.settings_live" if on_production() else "feedapp.settings"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
def __init__(self, connection, query, keys_only=False):
self.original_query = query
self.connection = connection
self.limits = (query.low_mark, query.high_mark)
opts = query.get_meta()
self.distinct = query.distinct
self.distinct_values = set()
self.distinct_on_field = None
self.distinct_field_convertor = None
self.queried_fields = []
self.model = query.model
self.pk_col = opts.pk.column
self.is_count = query.aggregates
self.extra_select = query.extra_select
self._set_db_table()
self._validate_query_is_possible(query)
if not query.default_ordering:
self.ordering = query.order_by
else:
self.ordering = query.order_by or opts.ordering
if self.ordering:
ordering = [ x for x in self.ordering if not (isinstance(x, basestring) and "__" in x) ]
if len(ordering) < len(self.ordering):
if not on_production() and not in_testing():
diff = set(self.ordering) - set(ordering)
log_once(DJANGAE_LOG.warning, "The following orderings were ignored as cross-table orderings are not supported on the datastore: %s", diff)
self.ordering = ordering
#If the query uses defer()/only() then we need to process deferred. We have to get all deferred columns
# for all (concrete) inherited models and then only include columns if they appear in that list
deferred_columns = {}
query.deferred_to_data(deferred_columns, query.deferred_to_columns_cb)
inherited_db_tables = [ x._meta.db_table for x in get_concrete_parents(self.model) ]
only_load = list(chain(*[ list(deferred_columns.get(x, [])) for x in inherited_db_tables ]))
if query.select:
for x in query.select:
if hasattr(x, "field"):
#In Django 1.6+ 'x' above is a SelectInfo (which is a tuple subclass), whereas in 1.5 it's a tuple
# in 1.6 x[1] == Field, but 1.5 x[1] == unicode (column name)
if x.field is None:
column = x.col.col[1] #This is the column we are getting
lookup_type = x.col.lookup_type
self.distinct_on_field = column
#This whole section of code is weird, and is probably better implemented as a custom Query type (like QueryByKeys)
# basically, appengine gives back dates as a time since the epoch, we convert it to a date, then floor it, then convert it back
# in our transform function. The transform is applied when the results are read back so that only distinct values are returned.
# this is very hacky...
if lookup_type in DATE_TRANSFORMS:
self.distinct_field_convertor = lambda value: DATE_TRANSFORMS[lookup_type](self.connection, value)
else:
raise CouldBeSupportedError("Unhandled lookup_type %s" % lookup_type)
else:
column = x.field.column
else:
column = x[1]
if only_load and column not in only_load:
continue
self.queried_fields.append(column)
else:
self.queried_fields = [ x.column for x in opts.fields if (not only_load) or (x.column in only_load) ]
self.keys_only = keys_only or self.queried_fields == [ opts.pk.column ]
assert self.queried_fields
#Projection queries don't return results unless all projected fields are
#indexed on the model. This means if you add a field, and all fields on the model
#are projectable, you will never get any results until you've resaved all of them.
#Because it's not possible to detect this situation, we only try a projection query if a
#subset of fields was specified (e.g. values_list('bananas')) which makes the behaviour a
#bit more predictable. It would be nice at some point to add some kind of force_projection()
#thing on a queryset that would do this whenever possible, but that's for the future, maybe.
try_projection = (self.keys_only is False) and bool(self.queried_fields)
if not self.queried_fields:
self.queried_fields = [ x.column for x in opts.fields ]
self.excluded_pks = set()
self.has_inequality_filter = False
self.all_filters = []
self.results = None
self.gae_query = None
projection_fields = []
if try_projection:
for field in self.queried_fields:
#We don't include the primary key in projection queries...
if field == self.pk_col:
continue
#Text and byte fields aren't indexed, so we can't do a
#projection query
f = get_field_from_column(self.model, field)
if not f:
raise CouldBeSupportedError("Attemping a cross-table select or dates query, or something?!")
assert f #If this happens, we have a cross-table select going on! #FIXME
db_type = f.db_type(connection)
if db_type in ("bytes", "text"):
projection_fields = []
break
projection_fields.append(field)
self.projection = list(set(projection_fields)) or None
if opts.parents:
self.projection = None
columns = set()
if isinstance(query.where, EmptyWhere):
#Empty where means return nothing!
raise EmptyResultSet()
else:
self.where = normalize_query(query.where, self.connection, filtered_columns=columns)
#DISABLE PROJECTION IF WE ARE FILTERING ON ONE OF THE PROJECTION_FIELDS
for field in self.projection or []:
if field in columns:
self.projection = None
break
try:
#If the PK was queried, we switch it in our queried
#fields store with __key__
pk_index = self.queried_fields.index(self.pk_col)
self.queried_fields[pk_index] = "__key__"
except ValueError:
pass
"""
WSGI config for djangaemoddemo project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/
"""
from djangaemoddemo.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "djangaemoddemo.settings_live" if on_production() else "djangaemoddemo.settings"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
def home(request):
context = {
'on_production': on_production(),
}
return render(request, 'index.html', context)
def _extract_ordering_from_query_18(query):
from djangae.db.backends.appengine.commands import log_once
from django.db.models.expressions import OrderBy, F
# Add any orderings
if not query.default_ordering:
result = list(query.order_by)
else:
result = list(query.order_by or query.get_meta().ordering or [])
if query.extra_order_by:
result = list(query.extra_order_by)
# we need some extra logic to handle dot seperated ordering
new_result = []
cross_table_ordering = set()
for ordering in result:
if "." in ordering:
dot_based_ordering = ordering.split(".")
if dot_based_ordering[0] == query.model._meta.db_table:
ordering = dot_based_ordering[1]
elif dot_based_ordering[0].lstrip(
'-') == query.model._meta.db_table:
ordering = '-{}'.format(dot_based_ordering[1])
else:
cross_table_ordering.add(ordering)
continue # we don't want to add this ordering value
new_result.append(ordering)
if len(cross_table_ordering):
log_once(
DJANGAE_LOG.warning
if not on_production() else DJANGAE_LOG.debug,
"The following orderings were ignored as cross-table orderings are not supported on the datastore: %s",
cross_table_ordering)
result = new_result
final = []
opts = query.model._meta
# Apparently expression ordering is absolute and so shouldn't be flipped
# if the standard_ordering is False. This keeps track of which columns
# were expressions and so don't need flipping
expressions = set()
for col in result:
if isinstance(col, OrderBy):
descending = col.descending
col = col.expression.name
if descending:
col = "-" + col
expressions.add(col)
elif isinstance(col, F):
col = col.name
if isinstance(col, (int, long)):
# If you do a Dates query, the ordering is set to [1] or [-1]... which is weird
# I think it's to select the column number but then there is only 1 column so
# unless the ordinal is one-based I have no idea. So basically if it's an integer
# subtract 1 from the absolute value and look up in the select for the column (guessing)
idx = abs(col) - 1
try:
field_name = query.select[idx].col.col[-1]
field = query.model._meta.get_field_by_name(field_name)[0]
final.append("-" + field.column if col < 0 else field.column)
except IndexError:
raise NotSupportedError("Unsupported order_by %s" % col)
elif col.lstrip("-") == "pk":
pk_col = "__key__"
final.append("-" + pk_col if col.startswith("-") else pk_col)
elif col == "?":
raise NotSupportedError(
"Random ordering is not supported on the datastore")
elif "__" in col:
continue
else:
try:
column = col.lstrip("-")
# This is really 1.8 only, but I didn't want to duplicate this function
# just for this. Suggestions for doing this more cleanly welcome!
if column in getattr(query, "annotation_select", {}):
# It's an annotation, if it's a supported one, return the
# original column
annotation = query.annotation_select[column]
name = annotation.__class__.__name__
# We only support a few expressions
if name not in ("Col", "Date", "DateTime"):
raise NotSupportedError(
"Tried to order by unsupported expression")
else:
# Retrieve the original column and use that for ordering
if name == "Col":
column = annotation.output_field.column
else:
column = annotation.col.output_field.column
field = query.model._meta.get_field_by_name(column)[0]
column = "__key__" if field.primary_key else field.column
final.append("-" + column if col.startswith("-") else column)
except FieldDoesNotExist:
if col in query.extra_select:
# If the column is in the extra select we transform to the original
# column
try:
field = opts.get_field_by_name(
query.extra_select[col][0])[0]
column = "__key__" if field.primary_key else field.column
final.append("-" +
column if col.startswith("-") else column)
continue
except FieldDoesNotExist:
# Just pass through to the exception below
pass
available = opts.get_all_field_names()
raise FieldError("Cannot resolve keyword %r into field. "
"Choices are: %s" %
(col, ", ".join(available)))
# Reverse if not using standard ordering
def swap(col):
if col.startswith("-"):
return col.lstrip("-")
else:
return "-{}".format(col)
if not query.standard_ordering:
final = [x if x in expressions else swap(x) for x in final]
if len(final) != len(result):
diff = set(result) - set(final)
log_once(
DJANGAE_LOG.warning if not on_production() else DJANGAE_LOG.debug,
"The following orderings were ignored as cross-table and random orderings are not supported on the datastore: %s",
diff)
return final
"""
WSGI config for scaffold project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/
"""
from scaffold.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "scaffold.settings_live" if on_production() else "scaffold.settings"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
from ntkc.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "ntkc.settings_live" if on_production() else "ntkc.settings"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
def __init__(self, connection, query, keys_only=False, all_fields=False):
self.original_query = query
self.connection = connection
self.limits = (query.low_mark, query.high_mark)
opts = query.get_meta()
if not query.default_ordering:
self.ordering = query.order_by
else:
self.ordering = query.order_by or opts.ordering
if self.ordering:
ordering = [ x for x in self.ordering if not (isinstance(x, basestring) and "__" in x) ]
if len(ordering) < len(self.ordering):
if not on_production() and not in_testing():
diff = set(self.ordering) - set(ordering)
log_once(DJANGAE_LOG.warning, "The following orderings were ignored as cross-table orderings are not supported on the datastore: %s", diff)
self.ordering = ordering
self.distinct = query.distinct
self.distinct_values = set()
self.distinct_on_field = None
self.distinct_field_convertor = None
self.queried_fields = []
if keys_only:
self.queried_fields = [ opts.pk.column ]
elif not all_fields:
for x in query.select:
if isinstance(x, tuple):
#Django < 1.6 compatibility
self.queried_fields.append(x[1])
else:
self.queried_fields.append(x.col[1])
if x.lookup_type == 'year':
assert self.distinct_on_field is None
self.distinct_on_field = x.col[1]
self.distinct_field_convertor = field_conv_year_only
elif x.lookup_type == 'month':
assert self.distinct_on_field is None
self.distinct_on_field = x.col[1]
self.distinct_field_convertor = field_conv_month_only
elif x.lookup_type == 'day':
assert self.distinct_on_field is None
self.distinct_on_field = x.col[1]
self.distinct_field_convertor = field_conv_day_only
else:
raise NotSupportedError("Unhandled lookup type: {0}".format(x.lookup_type))
#Projection queries don't return results unless all projected fields are
#indexed on the model. This means if you add a field, and all fields on the model
#are projectable, you will never get any results until you've resaved all of them.
#Because it's not possible to detect this situation, we only try a projection query if a
#subset of fields was specified (e.g. values_list('bananas')) which makes the behaviour a
#bit more predictable. It would be nice at some point to add some kind of force_projection()
#thing on a queryset that would do this whenever possible, but that's for the future, maybe.
try_projection = bool(self.queried_fields)
if not self.queried_fields:
self.queried_fields = [ x.column for x in opts.fields ]
self.connection = connection
self.pk_col = opts.pk.column
self.model = query.model
self.is_count = query.aggregates
self.keys_only = False #FIXME: This should be used where possible
self.exact_pk = None
self.included_pks = []
self.excluded_pks = set()
self.has_inequality_filter = False
self.all_filters = []
self.results = None
self.extra_select = query.extra_select
self.gae_query = None
self._set_db_table()
self._validate_query_is_possible(query)
projection_fields = []
if try_projection:
for field in self.queried_fields:
#We don't include the primary key in projection queries...
if field == self.pk_col:
continue
#Text and byte fields aren't indexed, so we can't do a
#projection query
f = get_field_from_column(self.model, field)
if not f:
raise NotSupportedError("Attemping a cross-table select. Maybe? #FIXME")
assert f #If this happens, we have a cross-table select going on! #FIXME
db_type = f.db_type(connection)
if db_type in ("bytes", "text"):
projection_fields = []
break
projection_fields.append(field)
self.projection = list(set(projection_fields)) or None
if opts.parents:
self.projection = None
self.where = self.parse_where_and_check_projection(query.where)
try:
#If the PK was queried, we switch it in our queried
#fields store with __key__
pk_index = self.queried_fields.index(self.pk_col)
self.queried_fields[pk_index] = "__key__"
#If the only field queried was the key, then we can do a keys_only
#query
self.keys_only = len(self.queried_fields) == 1
except ValueError:
pass
"""
WSGI config for scaffold project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/
"""
from scaffold.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "scaffold.settings_live" if on_production() else "scaffold.settings"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
"""
WSGI config for scaffold project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/
"""
from tracker.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "tracker.settings_live" if on_production() else "tracker.settings"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
"""
WSGI config for betas project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/
"""
from betas.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "betas.settings_live" if on_production() else "betas.settings"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
"""
WSGI config for scaffold project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/
"""
from core.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "core.settings.live" if on_production() else "core.settings.dev"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
"""
WSGI config for myblog project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/
"""
from myblog.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "myblog.settings_live" if on_production() else "myblog.settings"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
"""WSGI config for scaffold project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/
"""
from tracker.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "tracker.settings_live" if on_production() else "tracker.settings"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
"""
WSGI config for urgentgramme project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/
"""
from urgentgramme.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "urgentgramme.settings_live" if on_production() else "urgentgramme.settings"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
"""
WSGI config for the project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/
"""
from project.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "project.settings_live" if on_production() else "project.settings"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
"""
WSGI config for kirberichuk project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/
"""
from kirberichuk.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "kirberichuk.settings_live" if on_production(
) else "kirberichuk.settings"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
def _extract_ordering_from_query_17(query):
from djangae.db.backends.appengine.commands import log_once
# Add any orderings
if not query.default_ordering:
result = list(query.order_by)
else:
result = list(query.order_by or query.get_meta().ordering or [])
if query.extra_order_by:
result = list(query.extra_order_by)
final = []
opts = query.model._meta
for col in result:
if isinstance(col, (int, long)):
# If you do a Dates query, the ordering is set to [1] or [-1]... which is weird
# I think it's to select the column number but then there is only 1 column so
# unless the ordinal is one-based I have no idea. So basically if it's an integer
# subtract 1 from the absolute value and look up in the select for the column (guessing)
idx = abs(col) - 1
try:
field_name = query.select[idx].col.col[-1]
field = query.model._meta.get_field_by_name(field_name)[0]
final.append("-" + field.column if col < 0 else field.column)
except IndexError:
raise NotSupportedError("Unsupported order_by %s" % col)
elif col.lstrip("-") == "pk":
pk_col = "__key__"
final.append("-" + pk_col if col.startswith("-") else pk_col)
elif col == "?":
raise NotSupportedError(
"Random ordering is not supported on the datastore")
elif "__" in col:
continue
else:
try:
column = col.lstrip("-")
field = query.model._meta.get_field_by_name(column)[0]
column = "__key__" if field.primary_key else field.column
final.append("-" + column if col.startswith("-") else column)
except FieldDoesNotExist:
if col in query.extra_select:
# If the column is in the extra select we transform to the original
# column
try:
field = opts.get_field_by_name(
query.extra_select[col][0])[0]
column = "__key__" if field.primary_key else field.column
final.append("-" +
column if col.startswith("-") else column)
continue
except FieldDoesNotExist:
# Just pass through to the exception below
pass
available = opts.get_all_field_names()
raise FieldError("Cannot resolve keyword %r into field. "
"Choices are: %s" %
(col, ", ".join(available)))
# Reverse if not using standard ordering
def swap(col):
if col.startswith("-"):
return col.lstrip("-")
else:
return "-{}".format(col)
if not query.standard_ordering:
final = [swap(x) for x in final]
if len(final) != len(result):
diff = set(result) - set(final)
log_once(
DJANGAE_LOG.warning if not on_production() else DJANGAE_LOG.debug,
"The following orderings were ignored as cross-table and random orderings are not supported on the datastore: %s",
diff)
return final
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
"""
"""
WSGI config for udon project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/
"""
from udon.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "udon.conf.production" if on_production() else "udon.conf.local"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
# Modify thinkAmi: djangae settings
# DATABASES = {
# 'default': {
# 'ENGINE': 'django.db.backends.sqlite3',
# 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
# }
# }
from djangae.utils import on_production
DATABASES = {
'default': {
'ENGINE': 'djangae.db.backends.appengine'
}
}
if on_production():
pass
else:
DATABASES['sql'] = {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': 'development.sqlite3'
}
# Internationalization
# https://docs.djangoproject.com/en/1.8/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
def _extract_ordering_from_query_18(query):
from djangae.db.backends.appengine.commands import log_once
from django.db.models.expressions import OrderBy, F
# Add any orderings
if not query.default_ordering:
result = list(query.order_by)
else:
result = list(query.order_by or query.get_meta().ordering or [])
if query.extra_order_by:
result = list(query.extra_order_by)
# we need some extra logic to handle dot seperated ordering
new_result = []
cross_table_ordering = set()
for ordering in result:
if "." in ordering:
dot_based_ordering = ordering.split(".")
if dot_based_ordering[0] == query.model._meta.db_table:
ordering = dot_based_ordering[1]
elif dot_based_ordering[0].lstrip('-') == query.model._meta.db_table:
ordering = '-{}'.format(dot_based_ordering[1])
else:
cross_table_ordering.add(ordering)
continue # we don't want to add this ordering value
new_result.append(ordering)
if len(cross_table_ordering):
log_once(
DJANGAE_LOG.warning if not on_production() else DJANGAE_LOG.debug,
"The following orderings were ignored as cross-table orderings are not supported on the datastore: %s", cross_table_ordering
)
result = new_result
final = []
opts = query.model._meta
# Apparently expression ordering is absolute and so shouldn't be flipped
# if the standard_ordering is False. This keeps track of which columns
# were expressions and so don't need flipping
expressions = set()
for col in result:
if isinstance(col, OrderBy):
descending = col.descending
col = col.expression.name
if descending:
col = "-" + col
expressions.add(col)
elif isinstance(col, F):
col = col.name
if isinstance(col, (int, long)):
# If you do a Dates query, the ordering is set to [1] or [-1]... which is weird
# I think it's to select the column number but then there is only 1 column so
# unless the ordinal is one-based I have no idea. So basically if it's an integer
# subtract 1 from the absolute value and look up in the select for the column (guessing)
idx = abs(col) - 1
try:
field_name = query.select[idx].col.col[-1]
field = query.model._meta.get_field(field_name)
final.append("-" + field.column if col < 0 else field.column)
except IndexError:
raise NotSupportedError("Unsupported order_by %s" % col)
elif col.lstrip("-") == "pk":
pk_col = "__key__"
final.append("-" + pk_col if col.startswith("-") else pk_col)
elif col == "?":
raise NotSupportedError("Random ordering is not supported on the datastore")
elif col.lstrip("-").startswith("__") and col.endswith("__"):
# Allow stuff like __scatter__
final.append(col)
elif "__" in col:
continue
else:
try:
column = col.lstrip("-")
# This is really 1.8 only, but I didn't want to duplicate this function
# just for this. Suggestions for doing this more cleanly welcome!
if column in getattr(query, "annotation_select", {}):
# It's an annotation, if it's a supported one, return the
# original column
annotation = query.annotation_select[column]
name = annotation.__class__.__name__
# We only support a few expressions
if name not in ("Col", "Date", "DateTime"):
raise NotSupportedError("Tried to order by unsupported expression")
else:
# Retrieve the original column and use that for ordering
if name == "Col":
column = annotation.output_field.column
else:
column = annotation.col.output_field.column
field = query.model._meta.get_field(column)
if field.get_internal_type() in (u"TextField", u"BinaryField"):
raise NotSupportedError(INVALID_ORDERING_FIELD_MESSAGE)
# If someone orders by 'fk' rather than 'fk_id' this complains as that should take
# into account the related model ordering. Note the difference between field.name == column
# and field.attname (X_id)
if field.related_model and field.name == column and field.related_model._meta.ordering:
raise NotSupportedError("Related ordering is not supported on the datastore")
column = "__key__" if field.primary_key else field.column
final.append("-" + column if col.startswith("-") else column)
except FieldDoesNotExist:
if col in query.extra_select:
# If the column is in the extra select we transform to the original
# column
try:
field = opts.get_field(query.extra_select[col][0])
column = "__key__" if field.primary_key else field.column
final.append("-" + column if col.startswith("-") else column)
continue
except FieldDoesNotExist:
# Just pass through to the exception below
pass
available = opts.get_all_field_names()
raise FieldError("Cannot resolve keyword %r into field. "
"Choices are: %s" % (col, ", ".join(available))
)
# Reverse if not using standard ordering
def swap(col):
if col.startswith("-"):
return col.lstrip("-")
else:
return "-{}".format(col)
if not query.standard_ordering:
final = [ x if x in expressions else swap(x) for x in final ]
if len(final) != len(result):
diff = set(result) - set(final)
log_once(
DJANGAE_LOG.warning if not on_production() else DJANGAE_LOG.debug,
"The following orderings were ignored as cross-table and random orderings are not supported on the datastore: %s", diff
)
return final
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
"""
"""
WSGI config for udon project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/
"""
from udon.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "udon.conf.production" if on_production() else "udon.conf.local"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
"""
WSGI config for potato-assignment project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/
"""
from potato_assignment.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "potato_assignment.settings_live" if on_production() else "potato_assignment.settings"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
"""
WSGI config for blogit project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/
"""
from blogit.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "blogit.settings_live" if on_production() else "blogit.settings"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
"""
WSGI config for WriteBunny project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/
"""
from main.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "main.settings_live" if on_production() else "main.settings_dev"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
"""
WSGI config for Parliament Tree project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/
"""
from pt.site.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "pt.site.settings_live" if on_production() else "pt.site.settings"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
"""
WSGI config for main project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/
"""
from main.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "main.settings_live" if on_production() else "main.settings"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
"""
WSGI config for foodles project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/
"""
from foodles.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "foodles.settings_live" if on_production() else "foodles.settings"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
"""
WSGI config for twistpass project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/
"""
from twistpass.site.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "twistpass.site.settings_live" if on_production() else "twistpass.site.settings"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
"""
WSGI config for app project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/
"""
from app.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "app.settings_live" if on_production() else "app.settings"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
"""
WSGI config for betas project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/
"""
from betas.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "betas.settings_live" if on_production() else "betas.settings"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
from djangae.utils import on_production
def fix_c_whitelist():
from google.appengine.tools.devappserver2.python import sandbox
if '_sqlite3' not in sandbox._WHITE_LIST_C_MODULES:
sandbox._WHITE_LIST_C_MODULES.extend([
'_sqlite3',
'_ssl', # Workaround for App Engine bug #9246
'_socket'
])
# We do this globally for the local environment outside of dev_appserver
if not on_production():
fix_c_whitelist()
def fix_sandbox():
"""
This is the nastiest thing in the world...
This WSGI middleware is the first chance we get to hook into anything. On the dev_appserver
at this point the Python sandbox will have already been initialized. The sandbox replaces stuff
like the subprocess module, and the select module. As well as disallows _sqlite3. These things
are really REALLY useful for development.
So here we dismantle parts of the sandbox. Firstly we add _sqlite3 to the allowed C modules.
This only happens on the dev_appserver, it would only die on live. Everything is checked so that
changes are only made if they haven't been made already.
def __init__(self, connection, query, keys_only=False):
self.original_query = query
self.connection = connection
self.limits = (query.low_mark, query.high_mark)
opts = query.get_meta()
self.distinct = query.distinct
self.distinct_values = set()
self.distinct_on_field = None
self.distinct_field_convertor = None
self.queried_fields = []
self.model = query.model
self.pk_col = opts.pk.column
self.is_count = query.aggregates
self.extra_select = query.extra_select
self._set_db_table()
self._validate_query_is_possible(query)
if not query.default_ordering:
self.ordering = query.order_by
else:
self.ordering = query.order_by or opts.ordering
if self.ordering:
ordering = [ x for x in self.ordering if not (isinstance(x, basestring) and "__" in x) ]
if len(ordering) < len(self.ordering):
if not on_production() and not in_testing():
diff = set(self.ordering) - set(ordering)
log_once(DJANGAE_LOG.warning, "The following orderings were ignored as cross-table orderings are not supported on the datastore: %s", diff)
self.ordering = ordering
#If the query uses defer()/only() then we need to process deferred. We have to get all deferred columns
# for all (concrete) inherited models and then only include columns if they appear in that list
deferred_columns = {}
query.deferred_to_data(deferred_columns, query.deferred_to_columns_cb)
inherited_db_tables = [ x._meta.db_table for x in get_concrete_parents(self.model) ]
only_load = list(chain(*[ list(deferred_columns.get(x, [])) for x in inherited_db_tables ]))
if query.select:
for x in query.select:
if hasattr(x, "field"):
#In Django 1.6+ 'x' above is a SelectInfo (which is a tuple subclass), whereas in 1.5 it's a tuple
# in 1.6 x[1] == Field, but 1.5 x[1] == unicode (column name)
if x.field is None:
column = x.col.col[1] #This is the column we are getting
lookup_type = x.col.lookup_type
self.distinct_on_field = column
#This whole section of code is weird, and is probably better implemented as a custom Query type (like QueryByKeys)
# basically, appengine gives back dates as a time since the epoch, we convert it to a date, then floor it, then convert it back
# in our transform function. The transform is applied when the results are read back so that only distinct values are returned.
# this is very hacky...
if lookup_type in DATE_TRANSFORMS:
self.distinct_field_convertor = lambda value: DATE_TRANSFORMS[lookup_type](self.connection, value)
else:
raise CouldBeSupportedError("Unhandled lookup_type %s" % lookup_type)
else:
column = x.field.column
else:
column = x[1]
if only_load and column not in only_load:
continue
self.queried_fields.append(column)
else:
self.queried_fields = [ x.column for x in opts.fields if (not only_load) or (x.column in only_load) ]
self.keys_only = keys_only or self.queried_fields == [ opts.pk.column ]
assert self.queried_fields
#Projection queries don't return results unless all projected fields are
#indexed on the model. This means if you add a field, and all fields on the model
#are projectable, you will never get any results until you've resaved all of them.
#Because it's not possible to detect this situation, we only try a projection query if a
#subset of fields was specified (e.g. values_list('bananas')) which makes the behaviour a
#bit more predictable. It would be nice at some point to add some kind of force_projection()
#thing on a queryset that would do this whenever possible, but that's for the future, maybe.
try_projection = (self.keys_only is False) and bool(self.queried_fields)
if not self.queried_fields:
self.queried_fields = [ x.column for x in opts.fields ]
self.excluded_pks = set()
self.has_inequality_filter = False
self.all_filters = []
self.results = None
self.gae_query = None
projection_fields = []
if try_projection:
for field in self.queried_fields:
##We don't include the primary key in projection queries...
if field == self.pk_col:
order_fields = set([ x.strip("-") for x in self.ordering])
if self.pk_col in order_fields or "pk" in order_fields:
#If we were ordering on __key__ we can't do a projection at all
self.projection_fields = []
break
continue
#Text and byte fields aren't indexed, so we can't do a
#projection query
f = get_field_from_column(self.model, field)
if not f:
raise CouldBeSupportedError("Attemping a cross-table select or dates query, or something?!")
assert f #If this happens, we have a cross-table select going on! #FIXME
db_type = f.db_type(connection)
if db_type in ("bytes", "text"):
projection_fields = []
break
projection_fields.append(field)
self.projection = list(set(projection_fields)) or None
if opts.parents:
self.projection = None
if isinstance(query.where, EmptyWhere):
#Empty where means return nothing!
raise EmptyResultSet()
else:
from dnf import parse_dnf
self.where, columns = parse_dnf(query.where, self.connection)
#DISABLE PROJECTION IF WE ARE FILTERING ON ONE OF THE PROJECTION_FIELDS
for field in self.projection or []:
if field in columns:
self.projection = None
break
try:
#If the PK was queried, we switch it in our queried
#fields store with __key__
pk_index = self.queried_fields.index(self.pk_col)
self.queried_fields[pk_index] = "__key__"
except ValueError:
pass
"""
WSGI config for core project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/
"""
from core.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "core.settings_live" if on_production() else "core.settings"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
"""
WSGI config for javiman project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/
"""
from javiman.boot import fix_path
fix_path()
import os
from django.core.wsgi import get_wsgi_application
from djangae.wsgi import DjangaeApplication
from djangae.utils import on_production
settings = "javiman.settings_live" if on_production() else "javiman.settings"
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings)
application = DjangaeApplication(get_wsgi_application())
def _convert_ordering(query):
if not query.default_ordering:
result = query.order_by
else:
result = query.order_by or query.get_meta().ordering
if query.extra_order_by:
# This is a best attempt at ordering by extra select, it covers the cases
# in the Django tests, but use this functionality with care
all_fields = query.get_meta().get_all_field_names()
new_ordering = []
for col in query.extra_order_by:
# If the query in the extra order by is part of the extra select
# and the extra select is just an alias, then use the original column
if col in query.extra_select:
if query.extra_select[col][0] in all_fields:
new_ordering.append(query.extra_select[col][0])
else:
# It wasn't an alias, probably can't support it
raise NotSupportedError("Unsupported extra_order_by: {}".format(query.extra_order_by))
else:
# Not in the extra select, probably just a column so use it if it is
if col in all_fields:
new_ordering.append(col)
else:
raise NotSupportedError("Unsupported extra_order_by: {}".format(query.extra_order_by))
result = tuple(new_ordering)
if result:
# We factor out cross-table orderings (rather than raising NotSupportedError) otherwise we'll break
# the admin which uses them. We log a warning when this happens though
try:
ordering = []
for name in result:
if name == "?":
raise NotSupportedError("Random ordering is not supported on the datastore")
if not (isinstance(name, basestring) and "__" in name):
if isinstance(name, basestring):
if name.lstrip("-") == "pk":
field_column = query.model._meta.pk.column
else:
field = query.model._meta.get_field_by_name(name.lstrip("-"))[0]
field_column = field.column
ordering.append(field_column if not name.startswith("-") else "-{}".format(field_column))
else:
ordering.append(name)
except FieldDoesNotExist:
opts = query.model._meta
available = opts.get_all_field_names()
raise FieldError("Cannot resolve keyword %r into field. "
"Choices are: %s" % (name, ", ".join(available))
)
if len(ordering) < len(result):
diff = set(result) - set(ordering)
log_once(
DJANGAE_LOG.warning if not on_production() else DJANGAE_LOG.debug,
"The following orderings were ignored as cross-table orderings are not supported on the datastore: %s", diff
)
result = ordering
return result
from djangae.utils import on_production
def fix_c_whitelist():
from google.appengine.tools.devappserver2.python import sandbox
if '_sqlite3' not in sandbox._WHITE_LIST_C_MODULES:
sandbox._WHITE_LIST_C_MODULES.extend([
'_sqlite3',
'_ssl', # Workaround for App Engine bug #9246
'_socket'
])
# We do this globally for the local environment outside of dev_appserver
if not on_production():
fix_c_whitelist()
def fix_sandbox():
"""
This is the nastiest thing in the world...
This WSGI middleware is the first chance we get to hook into anything. On the dev_appserver
at this point the Python sandbox will have already been initialized. The sandbox replaces stuff
like the subprocess module, and the select module. As well as disallows _sqlite3. These things
are really REALLY useful for development.
So here we dismantle parts of the sandbox. Firstly we add _sqlite3 to the allowed C modules.
This only happens on the dev_appserver, it would only die on live. Everything is checked so that
changes are only made if they haven't been made already.
