def get_extracted_ordering(self):
from djangae.db.backends.appengine.commands import log_once
from django.db.models.expressions import OrderBy, F
query = self.django_query
# Add any orderings
if not query.default_ordering:
result = list(query.order_by)
else:
result = list(query.order_by or query.get_meta().ordering or [])
if query.extra_order_by:
result = list(query.extra_order_by)
# we need some extra logic to handle dot seperated ordering
new_result = []
cross_table_ordering = set()
for ordering in result:
if "." in ordering:
dot_based_ordering = ordering.split(".")
if dot_based_ordering[0] == query.model._meta.db_table:
ordering = dot_based_ordering[1]
elif dot_based_ordering[0].lstrip(
'-') == query.model._meta.db_table:
ordering = '-{}'.format(dot_based_ordering[1])
else:
cross_table_ordering.add(ordering)
continue # we don't want to add this ordering value
new_result.append(ordering)
if len(cross_table_ordering):
log_once(
logger.warning if environment.is_development_environment()
else logger.debug,
"The following orderings were ignored as cross-table orderings are not supported on the datastore: %s",
cross_table_ordering)
result = new_result
final = []
opts = query.model._meta
# Apparently expression ordering is absolute and so shouldn't be flipped
# if the standard_ordering is False. This keeps track of which columns
# were expressions and so don't need flipping
expressions = set()
for col in result:
if isinstance(col, OrderBy):
descending = col.descending
col = col.expression.name
if descending:
col = "-" + col
expressions.add(col)
elif isinstance(col, F):
col = col.name
if isinstance(col, (int, long)):
# If you do a Dates query, the ordering is set to [1] or [-1]... which is weird
# I think it's to select the column number but then there is only 1 column so
# unless the ordinal is one-based I have no idea. So basically if it's an integer
# subtract 1 from the absolute value and look up in the select for the column (guessing)
idx = abs(col) - 1
try:
field_name = query.select[idx].col.col[-1]
field = query.model._meta.get_field(field_name)
final.append("-" +
field.column if col < 0 else field.column)
except IndexError:
raise NotSupportedError("Unsupported order_by %s" % col)
elif col.lstrip("-") == "pk":
pk_col = "__key__"
final.append("-" + pk_col if col.startswith("-") else pk_col)
elif col == "?":
raise NotSupportedError(
"Random ordering is not supported on the datastore")
elif col.lstrip("-").startswith("__") and col.endswith("__"):
# Allow stuff like __scatter__
final.append(col)
elif "__" in col:
continue
else:
try:
column = col.lstrip("-")
# This is really 1.8 only, but I didn't want to duplicate this function
# just for this. Suggestions for doing this more cleanly welcome!
if column in getattr(query, "annotation_select", {}):
# It's an annotation, if it's a supported one, return the
# original column
annotation = query.annotation_select[column]
name = annotation.__class__.__name__
# We only support a few expressions
if name not in ("Trunc", "Col", "Date", "DateTime"):
raise NotSupportedError(
"Tried to order by unsupported expression")
elif name == "Trunc":
column = annotation.lhs.output_field.column
else:
# Retrieve the original column and use that for ordering
if name == "Col":
column = annotation.output_field.column
else:
column = annotation.col.output_field.column
field = query.model._meta.get_field(column)
if field.get_internal_type() in (u"TextField",
u"BinaryField"):
raise NotSupportedError(INVALID_ORDERING_FIELD_MESSAGE)
# If someone orders by 'fk' rather than 'fk_id' this complains as that should take
# into account the related model ordering. Note the difference between field.name == column
# and field.attname (X_id)
if field.related_model and field.name == column and field.related_model._meta.ordering:
raise NotSupportedError(
"Related ordering is not supported on the datastore"
)
column = "__key__" if field.primary_key else field.column
final.append("-" +
column if col.startswith("-") else column)
except FieldDoesNotExist:
if col in query.extra_select:
# If the column is in the extra select we transform to the original
# column
try:
field = opts.get_field(query.extra_select[col][0])
column = "__key__" if field.primary_key else field.column
final.append(
"-" +
column if col.startswith("-") else column)
continue
except FieldDoesNotExist:
# Just pass through to the exception below
pass
available = opts.get_all_field_names()
raise FieldError("Cannot resolve keyword %r into field. "
"Choices are: %s" %
(col, ", ".join(available)))
# Reverse if not using standard ordering
def swap(col):
if col.startswith("-"):
return col.lstrip("-")
else:
return "-{}".format(col)
if not query.standard_ordering:
final = [x if x in expressions else swap(x) for x in final]
if len(final) != len(result):
diff = set(result) - set(final)
log_once(
logger.warning
if environment.is_development_environment() else logger.debug,
"The following orderings were ignored as cross-table and random orderings are not supported on the datastore: %s",
diff)
return final
def _extract_ordering_from_query_18(query):
from djangae.db.backends.appengine.commands import log_once
from django.db.models.expressions import OrderBy, F
# Add any orderings
if not query.default_ordering:
result = list(query.order_by)
else:
result = list(query.order_by or query.get_meta().ordering or [])
if query.extra_order_by:
result = list(query.extra_order_by)
# we need some extra logic to handle dot seperated ordering
new_result = []
cross_table_ordering = set()
for ordering in result:
if "." in ordering:
dot_based_ordering = ordering.split(".")
if dot_based_ordering[0] == query.model._meta.db_table:
ordering = dot_based_ordering[1]
elif dot_based_ordering[0].lstrip('-') == query.model._meta.db_table:
ordering = '-{}'.format(dot_based_ordering[1])
else:
cross_table_ordering.add(ordering)
continue # we don't want to add this ordering value
new_result.append(ordering)
if len(cross_table_ordering):
log_once(
DJANGAE_LOG.warning if environment.is_development_environment() else DJANGAE_LOG.debug,
"The following orderings were ignored as cross-table orderings are not supported on the datastore: %s", cross_table_ordering
)
result = new_result
final = []
opts = query.model._meta
# Apparently expression ordering is absolute and so shouldn't be flipped
# if the standard_ordering is False. This keeps track of which columns
# were expressions and so don't need flipping
expressions = set()
for col in result:
if isinstance(col, OrderBy):
descending = col.descending
col = col.expression.name
if descending:
col = "-" + col
expressions.add(col)
elif isinstance(col, F):
col = col.name
if isinstance(col, (int, long)):
# If you do a Dates query, the ordering is set to [1] or [-1]... which is weird
# I think it's to select the column number but then there is only 1 column so
# unless the ordinal is one-based I have no idea. So basically if it's an integer
# subtract 1 from the absolute value and look up in the select for the column (guessing)
idx = abs(col) - 1
try:
field_name = query.select[idx].col.col[-1]
field = query.model._meta.get_field(field_name)
final.append("-" + field.column if col < 0 else field.column)
except IndexError:
raise NotSupportedError("Unsupported order_by %s" % col)
elif col.lstrip("-") == "pk":
pk_col = "__key__"
final.append("-" + pk_col if col.startswith("-") else pk_col)
elif col == "?":
raise NotSupportedError("Random ordering is not supported on the datastore")
elif col.lstrip("-").startswith("__") and col.endswith("__"):
# Allow stuff like __scatter__
final.append(col)
elif "__" in col:
continue
else:
try:
column = col.lstrip("-")
# This is really 1.8 only, but I didn't want to duplicate this function
# just for this. Suggestions for doing this more cleanly welcome!
if column in getattr(query, "annotation_select", {}):
# It's an annotation, if it's a supported one, return the
# original column
annotation = query.annotation_select[column]
name = annotation.__class__.__name__
# We only support a few expressions
if name not in ("Col", "Date", "DateTime"):
raise NotSupportedError("Tried to order by unsupported expression")
else:
# Retrieve the original column and use that for ordering
if name == "Col":
column = annotation.output_field.column
else:
column = annotation.col.output_field.column
field = query.model._meta.get_field(column)
if field.get_internal_type() in (u"TextField", u"BinaryField"):
raise NotSupportedError(INVALID_ORDERING_FIELD_MESSAGE)
# If someone orders by 'fk' rather than 'fk_id' this complains as that should take
# into account the related model ordering. Note the difference between field.name == column
# and field.attname (X_id)
if field.related_model and field.name == column and field.related_model._meta.ordering:
raise NotSupportedError("Related ordering is not supported on the datastore")
column = "__key__" if field.primary_key else field.column
final.append("-" + column if col.startswith("-") else column)
except FieldDoesNotExist:
if col in query.extra_select:
# If the column is in the extra select we transform to the original
# column
try:
field = opts.get_field(query.extra_select[col][0])
column = "__key__" if field.primary_key else field.column
final.append("-" + column if col.startswith("-") else column)
continue
except FieldDoesNotExist:
# Just pass through to the exception below
pass
available = opts.get_all_field_names()
raise FieldError("Cannot resolve keyword %r into field. "
"Choices are: %s" % (col, ", ".join(available))
)
# Reverse if not using standard ordering
def swap(col):
if col.startswith("-"):
return col.lstrip("-")
else:
return "-{}".format(col)
if not query.standard_ordering:
final = [ x if x in expressions else swap(x) for x in final ]
if len(final) != len(result):
diff = set(result) - set(final)
log_once(
DJANGAE_LOG.warning if environment.is_development_environment() else DJANGAE_LOG.debug,
"The following orderings were ignored as cross-table and random orderings are not supported on the datastore: %s", diff
)
return final
def oauth2callback(request):
if environment.is_development_environment():
# hack required for login to work when running the app locally;
# the required env var `OAUTHLIB_INSECURE_TRANSPORT` cannot be set in the shell or
# `manage.py` because dev_appserver ignores env vars not set in `app.yaml`'s
# `env_variables` section;
# see https://oauthlib.readthedocs.io/en/latest/oauth2/security.html#environment-variables
os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1'
try:
encoded_state = request.GET['state']
except KeyError:
msg = 'Missing state'
logging.exception(msg)
return HttpResponseBadRequest(msg)
try:
state = json.loads(encoded_state)
original_hostname = state['hostname']
except (ValueError, KeyError):
msg = 'Invalid state'
logging.exception(msg)
return HttpResponseBadRequest(msg)
# If we began the auth flow on a non-default version then (optionaly) redirect
# back to the version we started on. This avoids having to add authorized
# redirect URIs to the console for every deployed version.
if _get_oauth_redirect_host_from_settings(
) and original_hostname != request.META['HTTP_HOST']:
logging.debug('Redirect to version %s', original_hostname)
return shortcuts.redirect('{}://{}{}'.format(request.scheme,
original_hostname,
request.get_full_path()))
if STATE_SESSION_KEY not in request.session:
logging.exception("Missing oauth state from session")
return HttpResponseBadRequest()
client_id = getattr(settings, _CLIENT_ID_SETTING)
client_secret = getattr(settings, _CLIENT_SECRET_SETTING)
assert client_id and client_secret
google = _google_oauth2_session(request,
with_scope=False,
state=request.session[STATE_SESSION_KEY])
# If we have a next_url, then on error we can redirect there
# as that will likely restart the flow, if not, we'll raise
# a bad request on error
has_next_url = auth.REDIRECT_FIELD_NAME in request.session
next_url = (request.session[auth.REDIRECT_FIELD_NAME]
if has_next_url else settings.LOGIN_REDIRECT_URL)
failed = False
try:
token = google.fetch_token(
TOKEN_URL,
client_secret=client_secret,
authorization_response=request.build_absolute_uri())
except MismatchingStateError:
logging.exception("Mismatched state error in oauth handling")
failed = True
if google.authorized:
try:
profile = id_token.verify_oauth2_token(token['id_token'],
requests.Request(),
client_id)
except ValueError:
logging.exception("Error verifying OAuth2 token")
failed = True
else:
pk = profile["sub"]
defaults = dict(access_token=token['access_token'],
token_type=token['token_type'],
expires_at=_calc_expires_at(token['expires_in']),
profile=profile,
scopes=token['scope'])
# Refresh tokens only exist on the first authorization
# or, if you've specified the access_type as "offline"
if 'refresh_token' in token:
defaults['refresh_token'] = token['refresh_token']
session, _ = OAuthUserSession.objects.update_or_create(
pk=pk, defaults=defaults)
# credentials are valid, we should authenticate the user
user = OAuthBackend().authenticate(request, oauth_session=session)
if user:
logging.debug("Successfully authenticated %s via OAuth2", user)
user.backend = 'djangae.contrib.googleauth.backends.oauth2.%s' % OAuthBackend.__name__
# If we successfully authenticate, then we need to logout
# and back in again. This is because the user may have
# authenticated with another backend, but we need to re-auth
# with the OAuth backend
if request.user.is_authenticated:
# We refresh as authenticate may have changed the user
# and if logout ever does a save we might lose that
request.user.refresh_from_db()
auth.logout(request)
auth.login(request, user)
else:
failed = True
logging.warning(
"Failed Django authentication after getting oauth credentials"
)
else:
failed = True
logging.warning(
"Something failed during the OAuth authorization process for user: %s",
)
if failed and not has_next_url:
return HttpResponseBadRequest()
# We still redirect to the next_url, as this should trigger
# the oauth flow again, if we didn't authenticate
# successfully.
return HttpResponseRedirect(next_url)
def test_is_development_environment(self):
self.assertTrue(is_development_environment())
os.environ["GAE_ENV"] = 'standard'
self.assertFalse(is_development_environment())
del os.environ["GAE_ENV"]
from djangae import environment
def fix_c_whitelist():
from google.appengine.tools.devappserver2.python import sandbox
if '_sqlite3' not in sandbox._WHITE_LIST_C_MODULES:
sandbox._WHITE_LIST_C_MODULES.extend([
'_sqlite3',
'_ssl', # Workaround for App Engine bug #9246
'_socket'
])
# We do this globally for the local environment outside of dev_appserver
if environment.is_development_environment():
fix_c_whitelist()
def fix_sandbox():
"""
This is the nastiest thing in the world...
This WSGI middleware is the first chance we get to hook into anything. On the dev_appserver
at this point the Python sandbox will have already been initialized. The sandbox replaces stuff
like the subprocess module, and the select module. As well as disallows _sqlite3. These things
are really REALLY useful for development.
So here we dismantle parts of the sandbox. Firstly we add _sqlite3 to the allowed C modules.
This only happens on the dev_appserver, it would only die on live. Everything is checked so that
changes are only made if they haven't been made already.
from djangae import environment
def fix_c_whitelist():
from google.appengine.tools.devappserver2.python import sandbox
if '_sqlite3' not in sandbox._WHITE_LIST_C_MODULES:
sandbox._WHITE_LIST_C_MODULES.extend([
'_sqlite3',
'_ssl', # Workaround for App Engine bug #9246
'_socket'
])
# We do this globally for the local environment outside of dev_appserver
if environment.is_development_environment():
fix_c_whitelist()
def fix_sandbox():
"""
This is the nastiest thing in the world...
This WSGI middleware is the first chance we get to hook into anything. On the dev_appserver
at this point the Python sandbox will have already been initialized. The sandbox replaces stuff
like the subprocess module, and the select module. As well as disallows _sqlite3. These things
are really REALLY useful for development.
So here we dismantle parts of the sandbox. Firstly we add _sqlite3 to the allowed C modules.
This only happens on the dev_appserver, it would only die on live. Everything is checked so that
changes are only made if they haven't been made already.
