def add_artwork(gallery_id):
"""
Within a gallery entry an art work can be added.
Firstly check if user logged in to do this
"""
if session["user"]:
if request.method == "POST":
is_exhibit = True if request.form.get("exhibit") else False
is_sold = True if request.form.get("sold") else False
artwork = {
"artist": request.form.get("artist"),
"title": request.form.get("title"),
"media": request.form.get("media"),
"height": request.form.get("height"),
"width": request.form.get("width"),
"image": request.form.get("image"),
"price": request.form.get("price"),
"sold": is_sold,
"exhibit": is_exhibit,
"added_by": session["user"],
"added_on": datetime.datetime.now()
}
try:
artstub = mongo.db.artworks.insert_one(artwork)
except OperationFailure:
raise OperationFailure("Failure to add an artwork")
except Exception as e:
return e
"""
Once artwork added, need to record id within Gallery's
artworks array
"""
try:
mongo.db.gallery.update_one({"_id": ObjectId(gallery_id)}, {
"$addToSet": {
"artworks": {
"art_id": (ObjectId(artstub.inserted_id)),
"artist": (request.form.get("artist")),
"title": (request.form.get("title")),
"media": (request.form.get("media")),
"height": (request.form.get("height")),
"width": (request.form.get("width")),
"image": (request.form.get("image")),
"price": (request.form.get("price")),
"sold": (request.form.get("sold")),
"added_by": (session["user"]),
"added_on": (datetime.datetime.now())
}
}
})
except OperationFailure:
raise OperationFailure("Failure to add an artwork to gallery")
except Exception as e:
return e
flash("** Thanks {}, art work entry added **".format(
session["user"]),
category="message")
else:
flash("User not logged in to do this")
return redirect(url_for("login"))
return redirect(url_for("gallery"))
def update_one(self, *args, **kwargs): # skipcq: PYL-R0201
raise OperationFailure(Exception())
def _check_command_response(response,
msg=None,
allowable_errors=None,
parse_write_concern_error=False):
"""Check the response to a command for errors.
"""
if "ok" not in response:
# Server didn't recognize our message as a command.
raise OperationFailure(response.get("$err"), response.get("code"),
response)
# TODO: remove, this is moving to _check_gle_response
if response.get("wtimeout", False):
# MongoDB versions before 1.8.0 return the error message in an "errmsg"
# field. If "errmsg" exists "err" will also exist set to None, so we
# have to check for "errmsg" first.
raise WTimeoutError(response.get("errmsg", response.get("err")),
response.get("code"), response)
if parse_write_concern_error and 'writeConcernError' in response:
wce = response['writeConcernError']
raise WriteConcernError(wce['errmsg'], wce['code'], wce)
if not response["ok"]:
details = response
# Mongos returns the error details in a 'raw' object
# for some errors.
if "raw" in response:
for shard in itervalues(response["raw"]):
# Grab the first non-empty raw error from a shard.
if shard.get("errmsg") and not shard.get("ok"):
details = shard
break
errmsg = details["errmsg"]
if allowable_errors is None or errmsg not in allowable_errors:
# Server is "not master" or "recovering"
if (errmsg.startswith("not master")
or errmsg.startswith("node is recovering")):
raise NotMasterError(errmsg, response)
# Server assertion failures
if errmsg == "db assertion failure":
errmsg = ("db assertion failure, assertion: '%s'" %
details.get("assertion", ""))
raise OperationFailure(errmsg, details.get("assertionCode"),
response)
# Other errors
code = details.get("code")
# findAndModify with upsert can raise duplicate key error
if code in (11000, 11001, 12582):
raise DuplicateKeyError(errmsg, code, response)
elif code == 50:
raise ExecutionTimeout(errmsg, code, response)
elif code == 43:
raise CursorNotFound(errmsg, code, response)
msg = msg or "%s"
raise OperationFailure(msg % errmsg, code, response)
def configure(self, proto):
"""
Configures the protocol using the information gathered from the
remote Mongo instance. Such information may contain the max
BSON document size, replica set configuration, and the master
status of the instance.
"""
if not proto:
defer.returnValue(None)
reply = yield self.__send_ismaster(proto, timeout=self.initialDelay)
# Handle the reply from the "ismaster" query. The reply contains
# configuration information about the peer.
# Make sure we got a result document.
if len(reply.documents) != 1:
raise OperationFailure("TxMongo: invalid document length.")
# Get the configuration document from the reply.
config = reply.documents[0].decode()
# Make sure the command was successful.
if not config.get("ok"):
code = config.get("code")
msg = "TxMongo: " + config.get("err", "Unknown error")
raise OperationFailure(msg, code)
# Check that the replicaSet matches.
set_name = config.get("setName")
expected_set_name = self.uri["options"].get("replicaset")
if expected_set_name and (expected_set_name != set_name):
# Log the invalid replica set failure.
msg = "TxMongo: Mongo instance does not match requested replicaSet."
raise ConfigurationError(msg)
# Track max bson object size limit.
proto.max_bson_size = config.get("maxBsonObjectSize",
DEFAULT_MAX_BSON_SIZE)
proto.max_write_batch_size = config.get("maxWriteBatchSize",
DEFAULT_MAX_WRITE_BATCH_SIZE)
proto.set_wire_versions(config.get("minWireVersion", 0),
config.get("maxWireVersion", 0))
# Track the other hosts in the replica set.
hosts = config.get("hosts")
if isinstance(hosts, list) and hosts:
for host in hosts:
if ':' not in host:
host = (host, 27017)
else:
host = host.split(':', 1)
host[1] = int(host[1])
host = tuple(host)
if host not in self.__allnodes:
self.__allnodes.append(host)
# Check if this node is the master.
ismaster = config.get("ismaster")
if not ismaster:
msg = "TxMongo: MongoDB host `%s` is not master." % config.get(
'me')
raise AutoReconnect(msg)
def mock_next(*args, **kwargs):
change_stream._cursor.close()
raise OperationFailure('Mock server error', code=code)
def _authenticate_scram(credentials, sock_info, mechanism):
"""Authenticate using SCRAM."""
username = credentials.username
if mechanism == 'SCRAM-SHA-256':
digest = "sha256"
digestmod = hashlib.sha256
data = saslprep(credentials.password).encode("utf-8")
else:
digest = "sha1"
digestmod = hashlib.sha1
data = _password_digest(username, credentials.password).encode("utf-8")
source = credentials.source
cache = credentials.cache
# Make local
_hmac = hmac.HMAC
user = username.encode("utf-8").replace(b"=", b"=3D").replace(b",", b"=2C")
nonce = standard_b64encode(
(("%s" % (SystemRandom().random(), ))[2:]).encode("utf-8"))
first_bare = b"n=" + user + b",r=" + nonce
cmd = SON([('saslStart', 1), ('mechanism', mechanism),
('payload', Binary(b"n,," + first_bare)), ('autoAuthorize', 1)])
res = sock_info.command(source, cmd)
server_first = res['payload']
parsed = _parse_scram_response(server_first)
iterations = int(parsed[b'i'])
if iterations < 4096:
raise OperationFailure("Server returned an invalid iteration count.")
salt = parsed[b's']
rnonce = parsed[b'r']
if not rnonce.startswith(nonce):
raise OperationFailure("Server returned an invalid nonce.")
without_proof = b"c=biws,r=" + rnonce
if cache.data:
client_key, server_key, csalt, citerations = cache.data
else:
client_key, server_key, csalt, citerations = None, None, None, None
# Salt and / or iterations could change for a number of different
# reasons. Either changing invalidates the cache.
if not client_key or salt != csalt or iterations != citerations:
salted_pass = _hi(digest, data, standard_b64decode(salt), iterations)
client_key = _hmac(salted_pass, b"Client Key", digestmod).digest()
server_key = _hmac(salted_pass, b"Server Key", digestmod).digest()
cache.data = (client_key, server_key, salt, iterations)
stored_key = digestmod(client_key).digest()
auth_msg = b",".join((first_bare, server_first, without_proof))
client_sig = _hmac(stored_key, auth_msg, digestmod).digest()
client_proof = b"p=" + standard_b64encode(_xor(client_key, client_sig))
client_final = b",".join((without_proof, client_proof))
server_sig = standard_b64encode(
_hmac(server_key, auth_msg, digestmod).digest())
cmd = SON([('saslContinue', 1), ('conversationId', res['conversationId']),
('payload', Binary(client_final))])
res = sock_info.command(source, cmd)
parsed = _parse_scram_response(res['payload'])
if not compare_digest(parsed[b'v'], server_sig):
raise OperationFailure("Server returned an invalid signature.")
# Depending on how it's configured, Cyrus SASL (which the server uses)
# requires a third empty challenge.
if not res['done']:
cmd = SON([('saslContinue', 1),
('conversationId', res['conversationId']),
('payload', Binary(b''))])
res = sock_info.command(source, cmd)
if not res['done']:
raise OperationFailure('SASL conversation failed to complete.')
def _authenticate_gssapi(credentials, sock_info):
"""Authenticate using GSSAPI."""
if not HAVE_KERBEROS:
raise ConfigurationError(
'The "kerberos" module must be installed to use GSSAPI authentication.'
)
try:
username = credentials.username
password = credentials.password
props = credentials.mechanism_properties
# Starting here and continuing through the while loop below - establish
# the security context. See RFC 4752, Section 3.1, first paragraph.
host = sock_info.address[0]
if props.canonicalize_host_name:
host = _canonicalize_hostname(host)
service = props.service_name + "@" + host
if props.service_realm is not None:
service = service + "@" + props.service_realm
if password is not None:
if _USE_PRINCIPAL:
# Note that, though we use unquote_plus for unquoting URI
# options, we use quote here. Microsoft's UrlUnescape (used
# by WinKerberos) doesn't support +.
principal = ":".join((quote(username), quote(password)))
result, ctx = kerberos.authGSSClientInit(
service, principal, gssflags=kerberos.GSS_C_MUTUAL_FLAG)
else:
if "@" in username:
user, domain = username.split("@", 1)
else:
user, domain = username, None
result, ctx = kerberos.authGSSClientInit(
service,
gssflags=kerberos.GSS_C_MUTUAL_FLAG,
user=user,
domain=domain,
password=password,
)
else:
result, ctx = kerberos.authGSSClientInit(
service, gssflags=kerberos.GSS_C_MUTUAL_FLAG)
if result != kerberos.AUTH_GSS_COMPLETE:
raise OperationFailure("Kerberos context failed to initialize.")
try:
# pykerberos uses a weird mix of exceptions and return values
# to indicate errors.
# 0 == continue, 1 == complete, -1 == error
# Only authGSSClientStep can return 0.
if kerberos.authGSSClientStep(ctx, "") != 0:
raise OperationFailure(
"Unknown kerberos failure in step function.")
# Start a SASL conversation with mongod/s
# Note: pykerberos deals with base64 encoded byte strings.
# Since mongo accepts base64 strings as the payload we don't
# have to use bson.binary.Binary.
payload = kerberos.authGSSClientResponse(ctx)
cmd = SON([
("saslStart", 1),
("mechanism", "GSSAPI"),
("payload", payload),
("autoAuthorize", 1),
])
response = sock_info.command("$external", cmd)
# Limit how many times we loop to catch protocol / library issues
for _ in range(10):
result = kerberos.authGSSClientStep(ctx,
str(response["payload"]))
if result == -1:
raise OperationFailure(
"Unknown kerberos failure in step function.")
payload = kerberos.authGSSClientResponse(ctx) or ""
cmd = SON([
("saslContinue", 1),
("conversationId", response["conversationId"]),
("payload", payload),
])
response = sock_info.command("$external", cmd)
if result == kerberos.AUTH_GSS_COMPLETE:
break
else:
raise OperationFailure(
"Kerberos authentication failed to complete.")
# Once the security context is established actually authenticate.
# See RFC 4752, Section 3.1, last two paragraphs.
if kerberos.authGSSClientUnwrap(ctx, str(
response["payload"])) != 1:
raise OperationFailure(
"Unknown kerberos failure during GSS_Unwrap step.")
if kerberos.authGSSClientWrap(
ctx, kerberos.authGSSClientResponse(ctx), username) != 1:
raise OperationFailure(
"Unknown kerberos failure during GSS_Wrap step.")
payload = kerberos.authGSSClientResponse(ctx)
cmd = SON([
("saslContinue", 1),
("conversationId", response["conversationId"]),
("payload", payload),
])
sock_info.command("$external", cmd)
finally:
kerberos.authGSSClientClean(ctx)
except kerberos.KrbError as exc:
raise OperationFailure(str(exc))
def remove(collection_name: str, entity_id: ObjectId) -> DeleteResult:
status = get_db()[collection_name].delete_one({"_id": entity_id})
if not status.acknowledged:
raise OperationFailure("Failed deleting doc")
return status
class BaseConnection(object):
"""
The base methods used for all connection objects
"""
def __init__(self,
host='localhost',
port=27017,
db='test',
collection='test',
username=None,
password=None,
max_retries=2,
**options):
"""
Instantiate the Mongo class
This can take optional arguments, all of which help
connect to the right database
"""
#: The port that the MongoDB connection lives on
self.port = port
#: The host or IP address to connect to
self.host = host
#: The name of the database
self.db = db
#: The name of the collection
self.collection = collection
#: The database username
self.username = username
#: The database password
self.password = password
#: The number of retries to attempt to reconnect after a connection
#: is dropped.
self.max_retries = max_retries
# Additional options
self.options = options
def _connect_to_db(self, retries=0):
"""
Connect to the database, but do not initialize a connection.
Depending on which public method is used, it initiates either a standard
mongo connection or a gridfs connection
"""
retries = retries
try:
# Establish a Connection
connection = pymongo.Connection(self.host, self.port,
**self.options)
# Establish a database
database = connection[self.db]
# If user passed username and password args, give that to authenticate
if self.username and self.password:
database.authenticate(self.username, self.password)
# Handle the following exceptions, if retries is less than what is
# passed into this method, attempt to connect again. Otherwise,
# raise the proper exception.
except AutoReconnect, error_message:
time.sleep(2)
retries += 1
if retries <= self.max_retries:
self._connect_to_db(retries=retries)
else:
raise ConnectionFailure('Max number of retries (%s) reached. Error: %s'\
% (self.max_retries, error_message))
except OperationFailure, error_message:
time.sleep(2)
retries += 1
if retries <= self.max_retries:
self._connect_to_db(retries=retries)
else:
raise OperationFailure('Max number of retries (%s) reached. Error: %s'\
% (self.max_retries, error_message))
def test_checkOrCreateReplicaSet_initialize(self, mongo_client_mock):
mongo_client_mock.return_value.admin.command.side_effect = (
OperationFailure("no replset config has been received"),
self._getFixture("initiate-ok"))
self.service.checkOrCreateReplicaSet(self.cluster_object)
def create(collection_name: str, entity: Dict) -> bool:
status = get_db()[collection_name].insert_one(entity)
if not status.acknowledged:
raise OperationFailure("Failed inserting doc")
return status.acknowledged
"""
# TODO: Faceted Search
# Add the necessary stages to the pipeline variable in the correct order.
# pipeline.extend(...)
pipeline.append(skip_stage)
pipeline.append(limit_stage)
pipeline.append(facet_stage)
try:
movies = list(db.movies.aggregate(pipeline, allowDiskUse=True))[0]
count = list(db.movies.aggregate(counting, allowDiskUse=True))[
0].get("count")
return (movies, count)
except OperationFailure:
raise OperationFailure(
"Results too large to sort, be more restrictive in filter")
def build_query_sort_project(filters):
"""
Builds the `query` predicate, `sort` and `projection` attributes for a given
filters dictionary.
"""
query = {}
# The field "tomatoes.viewer.numReviews" only exists in the movies we want
# to display on the front page of MFlix, because they are famous or
# aesthetically pleasing. When we sort on it, the movies containing this
# field will be displayed at the top of the page.
sort = [("tomatoes.viewer.numReviews", DESCENDING)]
project = None
if filters:
def _authenticate_scram_sha1(credentials, sock_info):
"""Authenticate using SCRAM-SHA-1."""
username = credentials.username
password = credentials.password
source = credentials.source
# Make local
_hmac = hmac.HMAC
_sha1 = sha1
user = username.encode("utf-8").replace(b"=", b"=3D").replace(b",", b"=2C")
nonce = standard_b64encode(
(("%s" % (SystemRandom().random(),))[2:]).encode("utf-8"))
first_bare = b"n=" + user + b",r=" + nonce
cmd = SON([('saslStart', 1),
('mechanism', 'SCRAM-SHA-1'),
('payload', Binary(b"n,," + first_bare)),
('autoAuthorize', 1)])
res = sock_info.command(source, cmd)
server_first = res['payload']
parsed = _parse_scram_response(server_first)
iterations = int(parsed[b'i'])
salt = parsed[b's']
rnonce = parsed[b'r']
if not rnonce.startswith(nonce):
raise OperationFailure("Server returned an invalid nonce.")
without_proof = b"c=biws,r=" + rnonce
salted_pass = _hi(_password_digest(username, password).encode("utf-8"),
standard_b64decode(salt),
iterations)
client_key = _hmac(salted_pass, b"Client Key", _sha1).digest()
stored_key = _sha1(client_key).digest()
auth_msg = b",".join((first_bare, server_first, without_proof))
client_sig = _hmac(stored_key, auth_msg, _sha1).digest()
client_proof = b"p=" + standard_b64encode(_xor(client_key, client_sig))
client_final = b",".join((without_proof, client_proof))
server_key = _hmac(salted_pass, b"Server Key", _sha1).digest()
server_sig = standard_b64encode(
_hmac(server_key, auth_msg, _sha1).digest())
cmd = SON([('saslContinue', 1),
('conversationId', res['conversationId']),
('payload', Binary(client_final))])
res = sock_info.command(source, cmd)
parsed = _parse_scram_response(res['payload'])
if not compare_digest(parsed[b'v'], server_sig):
raise OperationFailure("Server returned an invalid signature.")
# Depending on how it's configured, Cyrus SASL (which the server uses)
# requires a third empty challenge.
if not res['done']:
cmd = SON([('saslContinue', 1),
('conversationId', res['conversationId']),
('payload', Binary(b''))])
res = sock_info.command(source, cmd)
if not res['done']:
raise OperationFailure('SASL conversation failed to complete.')
def test_before_request_unexpected_error(self, mock_rollback):
mock_rollback.side_effect = OperationFailure('daamn!')
with assert_raises(OperationFailure):
handlers.transaction_before_request()
def _authenticate_gssapi(credentials, sock_info):
"""Authenticate using GSSAPI.
"""
if not HAVE_KERBEROS:
raise ConfigurationError('The "kerberos" module must be '
'installed to use GSSAPI authentication.')
try:
username = credentials.username
password = credentials.password
props = credentials.mechanism_properties
# Starting here and continuing through the while loop below - establish
# the security context. See RFC 4752, Section 3.1, first paragraph.
host = sock_info.address[0]
if props.canonicalize_host_name:
host = socket.getfqdn(host)
service = props.service_name + '@' + host
if props.service_realm is not None:
service = service + '@' + props.service_realm
if password is not None:
if '@' in username:
user, domain = username.split('@', 1)
else:
user, domain = username, None
result, ctx = kerberos.authGSSClientInit(
service,
gssflags=kerberos.GSS_C_MUTUAL_FLAG,
user=user,
domain=domain,
password=password)
else:
result, ctx = kerberos.authGSSClientInit(
service, gssflags=kerberos.GSS_C_MUTUAL_FLAG)
if result != kerberos.AUTH_GSS_COMPLETE:
raise OperationFailure('Kerberos context failed to initialize.')
try:
# pykerberos uses a weird mix of exceptions and return values
# to indicate errors.
# 0 == continue, 1 == complete, -1 == error
# Only authGSSClientStep can return 0.
if kerberos.authGSSClientStep(ctx, '') != 0:
raise OperationFailure('Unknown kerberos '
'failure in step function.')
# Start a SASL conversation with mongod/s
# Note: pykerberos deals with base64 encoded byte strings.
# Since mongo accepts base64 strings as the payload we don't
# have to use bson.binary.Binary.
payload = kerberos.authGSSClientResponse(ctx)
cmd = SON([('saslStart', 1), ('mechanism', 'GSSAPI'),
('payload', payload), ('autoAuthorize', 1)])
response = sock_info.command('$external', cmd)
# Limit how many times we loop to catch protocol / library issues
for _ in range(10):
result = kerberos.authGSSClientStep(ctx,
str(response['payload']))
if result == -1:
raise OperationFailure('Unknown kerberos '
'failure in step function.')
payload = kerberos.authGSSClientResponse(ctx) or ''
cmd = SON([('saslContinue', 1),
('conversationId', response['conversationId']),
('payload', payload)])
response = sock_info.command('$external', cmd)
if result == kerberos.AUTH_GSS_COMPLETE:
break
else:
raise OperationFailure('Kerberos '
'authentication failed to complete.')
# Once the security context is established actually authenticate.
# See RFC 4752, Section 3.1, last two paragraphs.
if kerberos.authGSSClientUnwrap(ctx, str(
response['payload'])) != 1:
raise OperationFailure('Unknown kerberos '
'failure during GSS_Unwrap step.')
if kerberos.authGSSClientWrap(
ctx, kerberos.authGSSClientResponse(ctx), username) != 1:
raise OperationFailure('Unknown kerberos '
'failure during GSS_Wrap step.')
payload = kerberos.authGSSClientResponse(ctx)
cmd = SON([('saslContinue', 1),
('conversationId', response['conversationId']),
('payload', payload)])
sock_info.command('$external', cmd)
finally:
kerberos.authGSSClientClean(ctx)
except kerberos.KrbError as exc:
raise OperationFailure(str(exc))
def mock_next(self, *args, **kwargs):
self._CommandCursor__killed = True
raise OperationFailure('Mock server error')
def test_authenticate_fails():
db = create_autospec(Database)
error = "command SON([('saslStart', 1), ('mechanism', 'SCRAM-SHA-1'), ('payload', Binary('n,,n=foo,r=OTI3MzA3MTEzMTIx', 0)), ('autoAuthorize', 1)]) on namespace admin.$cmd failed: Authentication failed."
db.authenticate.side_effect = OperationFailure(error)
assert auth.authenticate(db, sentinel.user, sentinel.password) is False
def check_indexes(document_class):
"""Ensures indexes are correct.
If any indexes are missing they will be created.
If any of them are 'wrong' (fields have changed, etc.) all the indexes for
that collection will be dropped and rebuilt.
Args:
document_class (Document): The document class
Returns:
None
Raises:
mongoengine.OperationFailure: Unhandled mongo error
"""
from mongoengine.connection import get_db
from pymongo.errors import OperationFailure
from .models import Request
try:
# Building the indexes could take a while so it'd be nice to give some
# indication of what's happening. This would be perfect but can't use
# it! It's broken for text indexes!! MongoEngine is awesome!!
# diff = collection.compare_indexes(); if diff['missing'] is not None...
# Since we can't ACTUALLY compare the index spec with what already
# exists without ridiculous effort:
spec = document_class.list_indexes()
existing = document_class._get_collection().index_information()
if document_class == Request and "parent_instance_index" in existing:
raise OperationFailure("Old Request index found, rebuilding")
if len(spec) < len(existing):
raise OperationFailure("Extra index found, rebuilding")
if len(spec) > len(existing):
logger.warning(
"Found missing %s indexes, about to build them. This could "
"take a while :)",
document_class.__name__,
)
document_class.ensure_indexes()
except OperationFailure:
logger.warning(
"%s collection indexes verification failed, attempting to rebuild",
document_class.__name__,
)
# Unfortunately mongoengine sucks. The index that failed is only
# returned as part of the error message. I REALLY don't want to parse
# an error string to find the index to drop. Also, ME only verifies /
# creates the indexes in bulk - there's no way to iterate through the
# index definitions and try them one by one. Since our indexes should be
# small and built in the background anyway just redo all of them
try:
db = get_db()
db[document_class.__name__.lower()].drop_indexes()
logger.warning("Dropped indexes for %s collection",
document_class.__name__)
except OperationFailure:
logger.error(
"Dropping %s indexes failed, please check the database configuration",
document_class.__name__,
)
raise
if document_class == Request:
logger.warning(
"Request definition is potentially out of date. About to check and "
"update if necessary - this could take several minutes.")
# bg-utils 2.3.3 -> 2.3.4 create the `has_parent` field
_update_request_has_parent_model()
# bg-utils 2.4.6 -> 2.4.7 change parent to ReferenceField
_update_request_parent_field_type()
logger.warning("Request definition check/update complete.")
try:
document_class.ensure_indexes()
logger.warning("%s indexes rebuilt successfully",
document_class.__name__)
except OperationFailure:
logger.error(
"%s index rebuild failed, please check the database configuration",
document_class.__name__,
)
raise
def _authenticate_scram(credentials, sock_info, mechanism):
"""Authenticate using SCRAM."""
username = credentials.username
if mechanism == "SCRAM-SHA-256":
digest = "sha256"
digestmod = hashlib.sha256
data = saslprep(credentials.password).encode("utf-8")
else:
digest = "sha1"
digestmod = hashlib.sha1
data = _password_digest(username, credentials.password).encode("utf-8")
source = credentials.source
cache = credentials.cache
# Make local
_hmac = hmac.HMAC
ctx = sock_info.auth_ctx
if ctx and ctx.speculate_succeeded():
nonce, first_bare = ctx.scram_data
res = ctx.speculative_authenticate
else:
nonce, first_bare, cmd = _authenticate_scram_start(
credentials, mechanism)
res = sock_info.command(source, cmd)
server_first = res["payload"]
parsed = _parse_scram_response(server_first)
iterations = int(parsed[b"i"])
if iterations < 4096:
raise OperationFailure("Server returned an invalid iteration count.")
salt = parsed[b"s"]
rnonce = parsed[b"r"]
if not rnonce.startswith(nonce):
raise OperationFailure("Server returned an invalid nonce.")
without_proof = b"c=biws,r=" + rnonce
if cache.data:
client_key, server_key, csalt, citerations = cache.data
else:
client_key, server_key, csalt, citerations = None, None, None, None
# Salt and / or iterations could change for a number of different
# reasons. Either changing invalidates the cache.
if not client_key or salt != csalt or iterations != citerations:
salted_pass = hashlib.pbkdf2_hmac(digest, data,
standard_b64decode(salt), iterations)
client_key = _hmac(salted_pass, b"Client Key", digestmod).digest()
server_key = _hmac(salted_pass, b"Server Key", digestmod).digest()
cache.data = (client_key, server_key, salt, iterations)
stored_key = digestmod(client_key).digest()
auth_msg = b",".join((first_bare, server_first, without_proof))
client_sig = _hmac(stored_key, auth_msg, digestmod).digest()
client_proof = b"p=" + standard_b64encode(_xor(client_key, client_sig))
client_final = b",".join((without_proof, client_proof))
server_sig = standard_b64encode(
_hmac(server_key, auth_msg, digestmod).digest())
cmd = SON([
("saslContinue", 1),
("conversationId", res["conversationId"]),
("payload", Binary(client_final)),
])
res = sock_info.command(source, cmd)
parsed = _parse_scram_response(res["payload"])
if not hmac.compare_digest(parsed[b"v"], server_sig):
raise OperationFailure("Server returned an invalid signature.")
# A third empty challenge may be required if the server does not support
# skipEmptyExchange: SERVER-44857.
if not res["done"]:
cmd = SON([
("saslContinue", 1),
("conversationId", res["conversationId"]),
("payload", Binary(b"")),
])
res = sock_info.command(source, cmd)
if not res["done"]:
raise OperationFailure("SASL conversation failed to complete.")
def get_movies_faceted(filters, page, movies_per_page):
"""
Returns movies and runtime and ratings facets. Also returns the total
movies matched by the filter.
Uses the same sort_key as get_movies
"""
sort_key = "tomatoes.viewer.numReviews"
pipeline = []
if "cast" in filters:
pipeline.extend([{
"$match": {
"cast": {
"$in": filters.get("cast")
}
}
}, {
"$sort": {
sort_key: DESCENDING
}
}])
else:
raise AssertionError("No filters to pass to faceted search!")
counting = pipeline[:]
count_stage = {"$count": "count"}
counting.append(count_stage)
skip_stage = {"$skip": movies_per_page * page}
limit_stage = {"$limit": movies_per_page}
facet_stage = {
"$facet": {
"runtime": [{
"$bucket": {
"groupBy": "$runtime",
"boundaries": [0, 60, 90, 120, 180],
"default": "other",
"output": {
"count": {
"$sum": 1
}
}
}
}],
"rating": [{
"$bucket": {
"groupBy": "$metacritic",
"boundaries": [0, 50, 70, 90, 100],
"default": "other",
"output": {
"count": {
"$sum": 1
}
}
}
}],
"movies": [{
"$addFields": {
"title": "$title"
}
}]
}
}
pipeline.extend([skip_stage, limit_stage, facet_stage])
try:
movies = list(db.movies.aggregate(pipeline, allowDiskUse=True))[0]
count = list(db.movies.aggregate(counting,
allowDiskUse=True))[0].get("count")
return (movies, count)
except OperationFailure:
raise OperationFailure(
"Results too large to sort, be more restrictive in filter")
def execute_no_results(self, sock_info, generator):
"""Execute all operations, returning no results (w=0).
"""
if self.uses_collation:
raise ConfigurationError(
'Collation is unsupported for unacknowledged writes.')
if self.uses_array_filters:
raise ConfigurationError(
'arrayFilters is unsupported for unacknowledged writes.')
# Cannot have both unacknowledged writes and bypass document validation.
if self.bypass_doc_val and sock_info.max_wire_version >= 4:
raise OperationFailure("Cannot set bypass_document_validation with"
" unacknowledged write concern")
# OP_MSG
if sock_info.max_wire_version > 5:
if self.ordered:
return self.execute_command_no_results(sock_info, generator)
return self.execute_op_msg_no_results(sock_info, generator)
coll = self.collection
# If ordered is True we have to send GLE or use write
# commands so we can abort on the first error.
write_concern = WriteConcern(w=int(self.ordered))
op_id = _randint()
next_run = next(generator)
while next_run:
# An ordered bulk write needs to send acknowledged writes to short
# circuit the next run. However, the final message on the final
# run can be unacknowledged.
run = next_run
next_run = next(generator, None)
needs_ack = self.ordered and next_run is not None
try:
if run.op_type == _INSERT:
self.execute_insert_no_results(sock_info, run, op_id,
needs_ack)
elif run.op_type == _UPDATE:
for operation in run.ops:
doc = operation['u']
check_keys = True
if doc and next(iter(doc)).startswith('$'):
check_keys = False
coll._update(sock_info,
operation['q'],
doc,
operation['upsert'],
check_keys,
operation['multi'],
write_concern=write_concern,
op_id=op_id,
ordered=self.ordered,
bypass_doc_val=self.bypass_doc_val)
else:
for operation in run.ops:
coll._delete(sock_info, operation['q'],
not operation['limit'], write_concern,
op_id, self.ordered)
except OperationFailure:
if self.ordered:
break
def update_one(self, *args, **kwargs):
raise OperationFailure(Exception())
def _copy_database(
fromdb,
todb,
fromhost,
mechanism,
username,
password,
sock_info,
cmd_func):
"""Copy a database, perhaps from a remote host.
:Parameters:
- `fromdb`: Source database.
- `todb`: Target database.
- `fromhost`: Source host like 'foo.com', 'foo.com:27017', or None.
- `mechanism`: An authentication mechanism.
- `username`: A str or unicode, or None.
- `password`: A str or unicode, or None.
- `sock_info`: A SocketInfo instance.
- `cmd_func`: A callback taking args sock_info, database, command doc.
"""
if not isinstance(fromdb, basestring):
raise TypeError('from_name must be an instance '
'of %s' % (basestring.__name__,))
if not isinstance(todb, basestring):
raise TypeError('to_name must be an instance '
'of %s' % (basestring.__name__,))
_check_database_name(todb)
warnings.warn("copy_database is deprecated. Use the raw 'copydb' command"
" or db.copyDatabase() in the mongo shell. See"
" doc/examples/copydb.",
DeprecationWarning, stacklevel=2)
# It would be better if the user told us what mechanism to use, but for
# backwards compatibility with earlier PyMongos we don't require the
# mechanism. Hope 'fromhost' runs the same version as the target.
if mechanism == 'DEFAULT':
if sock_info.max_wire_version >= 3:
mechanism = 'SCRAM-SHA-1'
else:
mechanism = 'MONGODB-CR'
if username is not None:
if mechanism == 'SCRAM-SHA-1':
credentials = auth._build_credentials_tuple(mech=mechanism,
source='admin',
user=username,
passwd=password,
extra=None)
try:
auth._copydb_scram_sha1(credentials=credentials,
sock_info=sock_info,
cmd_func=cmd_func,
fromdb=fromdb,
todb=todb,
fromhost=fromhost)
except OperationFailure, exc:
errmsg = exc.details and exc.details.get('errmsg') or ''
if 'no such cmd: saslStart' in errmsg:
explanation = (
"%s doesn't support SCRAM-SHA-1, pass"
" mechanism='MONGODB-CR' to copy_database" % fromhost)
raise OperationFailure(explanation,
exc.code,
exc.details)
else:
raise
elif mechanism == 'MONGODB-CR':
get_nonce_cmd = SON([('copydbgetnonce', 1),
('fromhost', fromhost)])
get_nonce_response, _ = cmd_func(sock_info, 'admin', get_nonce_cmd)
nonce = get_nonce_response['nonce']
copydb_cmd = SON([('copydb', 1),
('fromdb', fromdb),
('todb', todb)])
copydb_cmd['username'] = username
copydb_cmd['nonce'] = nonce
copydb_cmd['key'] = auth._auth_key(nonce, username, password)
if fromhost is not None:
copydb_cmd['fromhost'] = fromhost
cmd_func(sock_info, 'admin', copydb_cmd)
else:
raise InvalidOperation('Authentication mechanism %r not supported'
' for copy_database' % mechanism)
def _check_command_response(response, max_wire_version,
allowable_errors=None,
parse_write_concern_error=False):
"""Check the response to a command for errors.
"""
if "ok" not in response:
# Server didn't recognize our message as a command.
raise OperationFailure(response.get("$err"),
response.get("code"),
response,
max_wire_version)
if parse_write_concern_error and 'writeConcernError' in response:
_error = response["writeConcernError"]
_labels = response.get("errorLabels")
if _labels:
_error.update({'errorLabels': _labels})
_raise_write_concern_error(_error)
if response["ok"]:
return
details = response
# Mongos returns the error details in a 'raw' object
# for some errors.
if "raw" in response:
for shard in response["raw"].values():
# Grab the first non-empty raw error from a shard.
if shard.get("errmsg") and not shard.get("ok"):
details = shard
break
errmsg = details["errmsg"]
code = details.get("code")
# For allowable errors, only check for error messages when the code is not
# included.
if allowable_errors:
if code is not None:
if code in allowable_errors:
return
elif errmsg in allowable_errors:
return
# Server is "not master" or "recovering"
if code is not None:
if code in _NOT_MASTER_CODES:
raise NotMasterError(errmsg, response)
elif "not master" in errmsg or "node is recovering" in errmsg:
raise NotMasterError(errmsg, response)
# Other errors
# findAndModify with upsert can raise duplicate key error
if code in (11000, 11001, 12582):
raise DuplicateKeyError(errmsg, code, response, max_wire_version)
elif code == 50:
raise ExecutionTimeout(errmsg, code, response, max_wire_version)
elif code == 43:
raise CursorNotFound(errmsg, code, response, max_wire_version)
raise OperationFailure(errmsg, code, response, max_wire_version)
def get_movies_faceted(filters, page, movies_per_page):
"""
Returns movies and runtime and ratings facets. Also returns the total
movies matched by the filter.
Uses the same sort_key as get_movies
"""
sort_key = "tomatoes.viewer.numReviews"
pipeline = []
if "cast" in filters:
pipeline.extend([{
"$match": {
"cast": {
"$in": filters.get("cast")
}
}
}, {
"$sort": {
sort_key: DESCENDING
}
}])
else:
raise AssertionError("No filters to pass to faceted search!")
counting = pipeline[:]
count_stage = {"$count": "count"}
counting.append(count_stage)
skip_stage = {"$skip": movies_per_page * page}
limit_stage = {"$limit": movies_per_page}
facet_stage = {
"$facet": {
"runtime": [{
"$bucket": {
"groupBy": "$runtime",
"boundaries": [0, 60, 90, 120, 180],
"default": "other",
"output": {
"count": {
"$sum": 1
}
}
}
}],
"rating": [{
"$bucket": {
"groupBy": "$metacritic",
"boundaries": [0, 50, 70, 90, 100],
"default": "other",
"output": {
"count": {
"$sum": 1
}
}
}
}],
"movies": [{
"$addFields": {
"title": "$title"
}
}]
}
}
"""
Ticket: Faceted Search
Please append the skip_stage, limit_stage, and facet_stage to the pipeline
(in that order).
The pipeline is a Python array, so you can use append() or extend() to
complete this task.
"""
# TODO: Faceted Search
# Add the necessary stages to the pipeline variable in the correct order.
#pipeline = []
pipeline.append(skip_stage)
pipeline.append(limit_stage)
pipeline.append(facet_stage)
try:
movies = list(db.movies.aggregate(pipeline, allowDiskUse=True))[0]
count = list(db.movies.aggregate(counting,
allowDiskUse=True))[0].get("count")
return (movies, count)
except OperationFailure:
raise OperationFailure(
"Results too large to sort, be more restrictive in filter")
def test_unicode_strs_operation_failure(self):
exc = OperationFailure('unicode \U0001f40d', 10,
{"errmsg": 'unicode \U0001f40d'})
self._test_unicode_strs(exc)
def _authenticate_gssapi(credentials, sock_info, cmd_func):
"""Authenticate using GSSAPI.
"""
try:
dummy, username, gsn = credentials
# Starting here and continuing through the while loop below - establish
# the security context. See RFC 4752, Section 3.1, first paragraph.
result, ctx = kerberos.authGSSClientInit(gsn + '@' + sock_info.host,
kerberos.GSS_C_MUTUAL_FLAG)
if result != kerberos.AUTH_GSS_COMPLETE:
raise OperationFailure('Kerberos context failed to initialize.')
try:
# pykerberos uses a weird mix of exceptions and return values
# to indicate errors.
# 0 == continue, 1 == complete, -1 == error
# Only authGSSClientStep can return 0.
if kerberos.authGSSClientStep(ctx, '') != 0:
raise OperationFailure('Unknown kerberos '
'failure in step function.')
# Start a SASL conversation with mongod/s
# Note: pykerberos deals with base64 encoded byte strings.
# Since mongo accepts base64 strings as the payload we don't
# have to use bson.binary.Binary.
payload = kerberos.authGSSClientResponse(ctx)
cmd = SON([('saslStart', 1), ('mechanism', 'GSSAPI'),
('payload', payload), ('autoAuthorize', 1)])
response, _ = cmd_func(sock_info, '$external', cmd)
# Limit how many times we loop to catch protocol / library issues
for _ in xrange(10):
result = kerberos.authGSSClientStep(ctx,
str(response['payload']))
if result == -1:
raise OperationFailure('Unknown kerberos '
'failure in step function.')
payload = kerberos.authGSSClientResponse(ctx) or ''
cmd = SON([('saslContinue', 1),
('conversationId', response['conversationId']),
('payload', payload)])
response, _ = cmd_func(sock_info, '$external', cmd)
if result == kerberos.AUTH_GSS_COMPLETE:
break
else:
raise OperationFailure('Kerberos '
'authentication failed to complete.')
# Once the security context is established actually authenticate.
# See RFC 4752, Section 3.1, last two paragraphs.
if kerberos.authGSSClientUnwrap(ctx, str(
response['payload'])) != 1:
raise OperationFailure('Unknown kerberos '
'failure during GSS_Unwrap step.')
if kerberos.authGSSClientWrap(
ctx, kerberos.authGSSClientResponse(ctx), username) != 1:
raise OperationFailure('Unknown kerberos '
'failure during GSS_Wrap step.')
payload = kerberos.authGSSClientResponse(ctx)
cmd = SON([('saslContinue', 1),
('conversationId', response['conversationId']),
('payload', payload)])
response, _ = cmd_func(sock_info, '$external', cmd)
finally:
kerberos.authGSSClientClean(ctx)
except kerberos.KrbError, exc:
raise OperationFailure(str(exc))
def test_pickle_OperationFailure(self):
exc = OperationFailure('error', code=5, details={}, max_wire_version=7)
self.assertOperationFailureEqual(exc, pickle.loads(pickle.dumps(exc)))
def _scram_sha1_conversation(credentials, sock_info, cmd_func, sasl_start,
sasl_continue):
"""Authenticate or copydb using SCRAM-SHA-1.
sasl_start and sasl_continue are SONs, the base command documents for
beginning and continuing the SASL conversation. They may be modified
by the callee.
:Parameters:
- `credentials`: A credentials tuple from _build_credentials_tuple.
- `sock_info`: A SocketInfo instance.
- `cmd_func`: A callback taking args sock_info, database, command doc.
- `sasl_start`: A SON.
- `sasl_continue`: A SON.
"""
source, username, password = credentials
# Make local
_hmac = hmac.HMAC
_sha1 = _SHA1
_sha1mod = _SHA1MOD
user = username.encode("utf-8").replace(_EQUAL, b("=3D")).replace(
_COMMA, b("=2C"))
nonce = standard_b64encode(
(("%s" % (SystemRandom().random(), ))[2:]).encode("utf-8"))
first_bare = b("n=") + user + b(",r=") + nonce
sasl_start['payload'] = Binary(b("n,,") + first_bare)
res, _ = cmd_func(sock_info, source, sasl_start)
server_first = res['payload']
parsed = _parse_scram_response(server_first)
iterations = int(parsed[b('i')])
salt = parsed[b('s')]
rnonce = parsed[b('r')]
if not rnonce.startswith(nonce):
raise OperationFailure("Server returned an invalid nonce.")
without_proof = b("c=biws,r=") + rnonce
salted_pass = _hi(
_password_digest(username, password).encode("utf-8"),
standard_b64decode(salt), iterations)
client_key = _hmac(salted_pass, b("Client Key"), _sha1mod).digest()
stored_key = _sha1(client_key).digest()
auth_msg = _COMMA.join((first_bare, server_first, without_proof))
client_sig = _hmac(stored_key, auth_msg, _sha1mod).digest()
client_proof = b("p=") + standard_b64encode(_xor(client_key, client_sig))
client_final = _COMMA.join((without_proof, client_proof))
server_key = _hmac(salted_pass, b("Server Key"), _sha1mod).digest()
server_sig = standard_b64encode(
_hmac(server_key, auth_msg, _SHA1MOD).digest())
cmd = sasl_continue.copy()
cmd['conversationId'] = res['conversationId']
cmd['payload'] = Binary(client_final)
res, _ = cmd_func(sock_info, source, cmd)
parsed = _parse_scram_response(res['payload'])
if not compare_digest(parsed[b('v')], server_sig):
raise OperationFailure("Server returned an invalid signature.")
# Depending on how it's configured, Cyrus SASL (which the server uses)
# requires a third empty challenge.
if not res['done']:
cmd = sasl_continue.copy()
cmd['conversationId'] = res['conversationId']
cmd['payload'] = Binary(_EMPTY)
res, _ = cmd_func(sock_info, source, cmd)
if not res['done']:
raise OperationFailure('SASL conversation failed to complete.')
def _aws_temp_credentials():
"""Construct temporary MONGODB-AWS credentials."""
access_key = os.environ.get('AWS_ACCESS_KEY_ID')
secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
if access_key and secret_key:
return _AWSCredential(access_key, secret_key,
os.environ.get('AWS_SESSION_TOKEN'))
# If the environment variable
# AWS_CONTAINER_CREDENTIALS_RELATIVE_URI is set then drivers MUST
# assume that it was set by an AWS ECS agent and use the URI
# http://169.254.170.2/$AWS_CONTAINER_CREDENTIALS_RELATIVE_URI to
# obtain temporary credentials.
relative_uri = os.environ.get('AWS_CONTAINER_CREDENTIALS_RELATIVE_URI')
if relative_uri is not None:
try:
res = requests.get(_AWS_REL_URI + relative_uri,
timeout=_AWS_HTTP_TIMEOUT)
res_json = res.json()
except (ValueError, requests.exceptions.RequestException):
raise OperationFailure(
'temporary MONGODB-AWS credentials could not be obtained')
else:
# If the environment variable AWS_CONTAINER_CREDENTIALS_RELATIVE_URI is
# not set drivers MUST assume we are on an EC2 instance and use the
# endpoint
# http://169.254.169.254/latest/meta-data/iam/security-credentials
# /<role-name>
# whereas role-name can be obtained from querying the URI
# http://169.254.169.254/latest/meta-data/iam/security-credentials/.
try:
# Get token
headers = {'X-aws-ec2-metadata-token-ttl-seconds': "30"}
res = requests.post(_AWS_EC2_URI + 'latest/api/token',
headers=headers,
timeout=_AWS_HTTP_TIMEOUT)
token = res.content
# Get role name
headers = {'X-aws-ec2-metadata-token': token}
res = requests.get(_AWS_EC2_URI + _AWS_EC2_PATH,
headers=headers,
timeout=_AWS_HTTP_TIMEOUT)
role = res.text
# Get temp creds
res = requests.get(_AWS_EC2_URI + _AWS_EC2_PATH + role,
headers=headers,
timeout=_AWS_HTTP_TIMEOUT)
res_json = res.json()
except (ValueError, requests.exceptions.RequestException):
raise OperationFailure(
'temporary MONGODB-AWS credentials could not be obtained')
try:
temp_user = res_json['AccessKeyId']
temp_password = res_json['SecretAccessKey']
token = res_json['Token']
except KeyError:
# If temporary credentials cannot be obtained then drivers MUST
# fail authentication and raise an error.
raise OperationFailure(
'temporary MONGODB-AWS credentials could not be obtained')
return _AWSCredential(temp_user, temp_password, token)
