def add_artwork(gallery_id): """ Within a gallery entry an art work can be added. Firstly check if user logged in to do this """ if session["user"]: if request.method == "POST": is_exhibit = True if request.form.get("exhibit") else False is_sold = True if request.form.get("sold") else False artwork = { "artist": request.form.get("artist"), "title": request.form.get("title"), "media": request.form.get("media"), "height": request.form.get("height"), "width": request.form.get("width"), "image": request.form.get("image"), "price": request.form.get("price"), "sold": is_sold, "exhibit": is_exhibit, "added_by": session["user"], "added_on": datetime.datetime.now() } try: artstub = mongo.db.artworks.insert_one(artwork) except OperationFailure: raise OperationFailure("Failure to add an artwork") except Exception as e: return e """ Once artwork added, need to record id within Gallery's artworks array """ try: mongo.db.gallery.update_one({"_id": ObjectId(gallery_id)}, { "$addToSet": { "artworks": { "art_id": (ObjectId(artstub.inserted_id)), "artist": (request.form.get("artist")), "title": (request.form.get("title")), "media": (request.form.get("media")), "height": (request.form.get("height")), "width": (request.form.get("width")), "image": (request.form.get("image")), "price": (request.form.get("price")), "sold": (request.form.get("sold")), "added_by": (session["user"]), "added_on": (datetime.datetime.now()) } } }) except OperationFailure: raise OperationFailure("Failure to add an artwork to gallery") except Exception as e: return e flash("** Thanks {}, art work entry added **".format( session["user"]), category="message") else: flash("User not logged in to do this") return redirect(url_for("login")) return redirect(url_for("gallery"))
def update_one(self, *args, **kwargs): # skipcq: PYL-R0201 raise OperationFailure(Exception())
def _check_command_response(response, msg=None, allowable_errors=None, parse_write_concern_error=False): """Check the response to a command for errors. """ if "ok" not in response: # Server didn't recognize our message as a command. raise OperationFailure(response.get("$err"), response.get("code"), response) # TODO: remove, this is moving to _check_gle_response if response.get("wtimeout", False): # MongoDB versions before 1.8.0 return the error message in an "errmsg" # field. If "errmsg" exists "err" will also exist set to None, so we # have to check for "errmsg" first. raise WTimeoutError(response.get("errmsg", response.get("err")), response.get("code"), response) if parse_write_concern_error and 'writeConcernError' in response: wce = response['writeConcernError'] raise WriteConcernError(wce['errmsg'], wce['code'], wce) if not response["ok"]: details = response # Mongos returns the error details in a 'raw' object # for some errors. if "raw" in response: for shard in itervalues(response["raw"]): # Grab the first non-empty raw error from a shard. if shard.get("errmsg") and not shard.get("ok"): details = shard break errmsg = details["errmsg"] if allowable_errors is None or errmsg not in allowable_errors: # Server is "not master" or "recovering" if (errmsg.startswith("not master") or errmsg.startswith("node is recovering")): raise NotMasterError(errmsg, response) # Server assertion failures if errmsg == "db assertion failure": errmsg = ("db assertion failure, assertion: '%s'" % details.get("assertion", "")) raise OperationFailure(errmsg, details.get("assertionCode"), response) # Other errors code = details.get("code") # findAndModify with upsert can raise duplicate key error if code in (11000, 11001, 12582): raise DuplicateKeyError(errmsg, code, response) elif code == 50: raise ExecutionTimeout(errmsg, code, response) elif code == 43: raise CursorNotFound(errmsg, code, response) msg = msg or "%s" raise OperationFailure(msg % errmsg, code, response)
def configure(self, proto): """ Configures the protocol using the information gathered from the remote Mongo instance. Such information may contain the max BSON document size, replica set configuration, and the master status of the instance. """ if not proto: defer.returnValue(None) reply = yield self.__send_ismaster(proto, timeout=self.initialDelay) # Handle the reply from the "ismaster" query. The reply contains # configuration information about the peer. # Make sure we got a result document. if len(reply.documents) != 1: raise OperationFailure("TxMongo: invalid document length.") # Get the configuration document from the reply. config = reply.documents[0].decode() # Make sure the command was successful. if not config.get("ok"): code = config.get("code") msg = "TxMongo: " + config.get("err", "Unknown error") raise OperationFailure(msg, code) # Check that the replicaSet matches. set_name = config.get("setName") expected_set_name = self.uri["options"].get("replicaset") if expected_set_name and (expected_set_name != set_name): # Log the invalid replica set failure. msg = "TxMongo: Mongo instance does not match requested replicaSet." raise ConfigurationError(msg) # Track max bson object size limit. proto.max_bson_size = config.get("maxBsonObjectSize", DEFAULT_MAX_BSON_SIZE) proto.max_write_batch_size = config.get("maxWriteBatchSize", DEFAULT_MAX_WRITE_BATCH_SIZE) proto.set_wire_versions(config.get("minWireVersion", 0), config.get("maxWireVersion", 0)) # Track the other hosts in the replica set. hosts = config.get("hosts") if isinstance(hosts, list) and hosts: for host in hosts: if ':' not in host: host = (host, 27017) else: host = host.split(':', 1) host[1] = int(host[1]) host = tuple(host) if host not in self.__allnodes: self.__allnodes.append(host) # Check if this node is the master. ismaster = config.get("ismaster") if not ismaster: msg = "TxMongo: MongoDB host `%s` is not master." % config.get( 'me') raise AutoReconnect(msg)
def mock_next(*args, **kwargs): change_stream._cursor.close() raise OperationFailure('Mock server error', code=code)
def _authenticate_scram(credentials, sock_info, mechanism): """Authenticate using SCRAM.""" username = credentials.username if mechanism == 'SCRAM-SHA-256': digest = "sha256" digestmod = hashlib.sha256 data = saslprep(credentials.password).encode("utf-8") else: digest = "sha1" digestmod = hashlib.sha1 data = _password_digest(username, credentials.password).encode("utf-8") source = credentials.source cache = credentials.cache # Make local _hmac = hmac.HMAC user = username.encode("utf-8").replace(b"=", b"=3D").replace(b",", b"=2C") nonce = standard_b64encode( (("%s" % (SystemRandom().random(), ))[2:]).encode("utf-8")) first_bare = b"n=" + user + b",r=" + nonce cmd = SON([('saslStart', 1), ('mechanism', mechanism), ('payload', Binary(b"n,," + first_bare)), ('autoAuthorize', 1)]) res = sock_info.command(source, cmd) server_first = res['payload'] parsed = _parse_scram_response(server_first) iterations = int(parsed[b'i']) if iterations < 4096: raise OperationFailure("Server returned an invalid iteration count.") salt = parsed[b's'] rnonce = parsed[b'r'] if not rnonce.startswith(nonce): raise OperationFailure("Server returned an invalid nonce.") without_proof = b"c=biws,r=" + rnonce if cache.data: client_key, server_key, csalt, citerations = cache.data else: client_key, server_key, csalt, citerations = None, None, None, None # Salt and / or iterations could change for a number of different # reasons. Either changing invalidates the cache. if not client_key or salt != csalt or iterations != citerations: salted_pass = _hi(digest, data, standard_b64decode(salt), iterations) client_key = _hmac(salted_pass, b"Client Key", digestmod).digest() server_key = _hmac(salted_pass, b"Server Key", digestmod).digest() cache.data = (client_key, server_key, salt, iterations) stored_key = digestmod(client_key).digest() auth_msg = b",".join((first_bare, server_first, without_proof)) client_sig = _hmac(stored_key, auth_msg, digestmod).digest() client_proof = b"p=" + standard_b64encode(_xor(client_key, client_sig)) client_final = b",".join((without_proof, client_proof)) server_sig = standard_b64encode( _hmac(server_key, auth_msg, digestmod).digest()) cmd = SON([('saslContinue', 1), ('conversationId', res['conversationId']), ('payload', Binary(client_final))]) res = sock_info.command(source, cmd) parsed = _parse_scram_response(res['payload']) if not compare_digest(parsed[b'v'], server_sig): raise OperationFailure("Server returned an invalid signature.") # Depending on how it's configured, Cyrus SASL (which the server uses) # requires a third empty challenge. if not res['done']: cmd = SON([('saslContinue', 1), ('conversationId', res['conversationId']), ('payload', Binary(b''))]) res = sock_info.command(source, cmd) if not res['done']: raise OperationFailure('SASL conversation failed to complete.')
def _authenticate_gssapi(credentials, sock_info): """Authenticate using GSSAPI.""" if not HAVE_KERBEROS: raise ConfigurationError( 'The "kerberos" module must be installed to use GSSAPI authentication.' ) try: username = credentials.username password = credentials.password props = credentials.mechanism_properties # Starting here and continuing through the while loop below - establish # the security context. See RFC 4752, Section 3.1, first paragraph. host = sock_info.address[0] if props.canonicalize_host_name: host = _canonicalize_hostname(host) service = props.service_name + "@" + host if props.service_realm is not None: service = service + "@" + props.service_realm if password is not None: if _USE_PRINCIPAL: # Note that, though we use unquote_plus for unquoting URI # options, we use quote here. Microsoft's UrlUnescape (used # by WinKerberos) doesn't support +. principal = ":".join((quote(username), quote(password))) result, ctx = kerberos.authGSSClientInit( service, principal, gssflags=kerberos.GSS_C_MUTUAL_FLAG) else: if "@" in username: user, domain = username.split("@", 1) else: user, domain = username, None result, ctx = kerberos.authGSSClientInit( service, gssflags=kerberos.GSS_C_MUTUAL_FLAG, user=user, domain=domain, password=password, ) else: result, ctx = kerberos.authGSSClientInit( service, gssflags=kerberos.GSS_C_MUTUAL_FLAG) if result != kerberos.AUTH_GSS_COMPLETE: raise OperationFailure("Kerberos context failed to initialize.") try: # pykerberos uses a weird mix of exceptions and return values # to indicate errors. # 0 == continue, 1 == complete, -1 == error # Only authGSSClientStep can return 0. if kerberos.authGSSClientStep(ctx, "") != 0: raise OperationFailure( "Unknown kerberos failure in step function.") # Start a SASL conversation with mongod/s # Note: pykerberos deals with base64 encoded byte strings. # Since mongo accepts base64 strings as the payload we don't # have to use bson.binary.Binary. payload = kerberos.authGSSClientResponse(ctx) cmd = SON([ ("saslStart", 1), ("mechanism", "GSSAPI"), ("payload", payload), ("autoAuthorize", 1), ]) response = sock_info.command("$external", cmd) # Limit how many times we loop to catch protocol / library issues for _ in range(10): result = kerberos.authGSSClientStep(ctx, str(response["payload"])) if result == -1: raise OperationFailure( "Unknown kerberos failure in step function.") payload = kerberos.authGSSClientResponse(ctx) or "" cmd = SON([ ("saslContinue", 1), ("conversationId", response["conversationId"]), ("payload", payload), ]) response = sock_info.command("$external", cmd) if result == kerberos.AUTH_GSS_COMPLETE: break else: raise OperationFailure( "Kerberos authentication failed to complete.") # Once the security context is established actually authenticate. # See RFC 4752, Section 3.1, last two paragraphs. if kerberos.authGSSClientUnwrap(ctx, str( response["payload"])) != 1: raise OperationFailure( "Unknown kerberos failure during GSS_Unwrap step.") if kerberos.authGSSClientWrap( ctx, kerberos.authGSSClientResponse(ctx), username) != 1: raise OperationFailure( "Unknown kerberos failure during GSS_Wrap step.") payload = kerberos.authGSSClientResponse(ctx) cmd = SON([ ("saslContinue", 1), ("conversationId", response["conversationId"]), ("payload", payload), ]) sock_info.command("$external", cmd) finally: kerberos.authGSSClientClean(ctx) except kerberos.KrbError as exc: raise OperationFailure(str(exc))
def remove(collection_name: str, entity_id: ObjectId) -> DeleteResult: status = get_db()[collection_name].delete_one({"_id": entity_id}) if not status.acknowledged: raise OperationFailure("Failed deleting doc") return status
class BaseConnection(object): """ The base methods used for all connection objects """ def __init__(self, host='localhost', port=27017, db='test', collection='test', username=None, password=None, max_retries=2, **options): """ Instantiate the Mongo class This can take optional arguments, all of which help connect to the right database """ #: The port that the MongoDB connection lives on self.port = port #: The host or IP address to connect to self.host = host #: The name of the database self.db = db #: The name of the collection self.collection = collection #: The database username self.username = username #: The database password self.password = password #: The number of retries to attempt to reconnect after a connection #: is dropped. self.max_retries = max_retries # Additional options self.options = options def _connect_to_db(self, retries=0): """ Connect to the database, but do not initialize a connection. Depending on which public method is used, it initiates either a standard mongo connection or a gridfs connection """ retries = retries try: # Establish a Connection connection = pymongo.Connection(self.host, self.port, **self.options) # Establish a database database = connection[self.db] # If user passed username and password args, give that to authenticate if self.username and self.password: database.authenticate(self.username, self.password) # Handle the following exceptions, if retries is less than what is # passed into this method, attempt to connect again. Otherwise, # raise the proper exception. except AutoReconnect, error_message: time.sleep(2) retries += 1 if retries <= self.max_retries: self._connect_to_db(retries=retries) else: raise ConnectionFailure('Max number of retries (%s) reached. Error: %s'\ % (self.max_retries, error_message)) except OperationFailure, error_message: time.sleep(2) retries += 1 if retries <= self.max_retries: self._connect_to_db(retries=retries) else: raise OperationFailure('Max number of retries (%s) reached. Error: %s'\ % (self.max_retries, error_message))
def test_checkOrCreateReplicaSet_initialize(self, mongo_client_mock): mongo_client_mock.return_value.admin.command.side_effect = ( OperationFailure("no replset config has been received"), self._getFixture("initiate-ok")) self.service.checkOrCreateReplicaSet(self.cluster_object)
def create(collection_name: str, entity: Dict) -> bool: status = get_db()[collection_name].insert_one(entity) if not status.acknowledged: raise OperationFailure("Failed inserting doc") return status.acknowledged
""" # TODO: Faceted Search # Add the necessary stages to the pipeline variable in the correct order. # pipeline.extend(...) pipeline.append(skip_stage) pipeline.append(limit_stage) pipeline.append(facet_stage) try: movies = list(db.movies.aggregate(pipeline, allowDiskUse=True))[0] count = list(db.movies.aggregate(counting, allowDiskUse=True))[ 0].get("count") return (movies, count) except OperationFailure: raise OperationFailure( "Results too large to sort, be more restrictive in filter") def build_query_sort_project(filters): """ Builds the `query` predicate, `sort` and `projection` attributes for a given filters dictionary. """ query = {} # The field "tomatoes.viewer.numReviews" only exists in the movies we want # to display on the front page of MFlix, because they are famous or # aesthetically pleasing. When we sort on it, the movies containing this # field will be displayed at the top of the page. sort = [("tomatoes.viewer.numReviews", DESCENDING)] project = None if filters:
def _authenticate_scram_sha1(credentials, sock_info): """Authenticate using SCRAM-SHA-1.""" username = credentials.username password = credentials.password source = credentials.source # Make local _hmac = hmac.HMAC _sha1 = sha1 user = username.encode("utf-8").replace(b"=", b"=3D").replace(b",", b"=2C") nonce = standard_b64encode( (("%s" % (SystemRandom().random(),))[2:]).encode("utf-8")) first_bare = b"n=" + user + b",r=" + nonce cmd = SON([('saslStart', 1), ('mechanism', 'SCRAM-SHA-1'), ('payload', Binary(b"n,," + first_bare)), ('autoAuthorize', 1)]) res = sock_info.command(source, cmd) server_first = res['payload'] parsed = _parse_scram_response(server_first) iterations = int(parsed[b'i']) salt = parsed[b's'] rnonce = parsed[b'r'] if not rnonce.startswith(nonce): raise OperationFailure("Server returned an invalid nonce.") without_proof = b"c=biws,r=" + rnonce salted_pass = _hi(_password_digest(username, password).encode("utf-8"), standard_b64decode(salt), iterations) client_key = _hmac(salted_pass, b"Client Key", _sha1).digest() stored_key = _sha1(client_key).digest() auth_msg = b",".join((first_bare, server_first, without_proof)) client_sig = _hmac(stored_key, auth_msg, _sha1).digest() client_proof = b"p=" + standard_b64encode(_xor(client_key, client_sig)) client_final = b",".join((without_proof, client_proof)) server_key = _hmac(salted_pass, b"Server Key", _sha1).digest() server_sig = standard_b64encode( _hmac(server_key, auth_msg, _sha1).digest()) cmd = SON([('saslContinue', 1), ('conversationId', res['conversationId']), ('payload', Binary(client_final))]) res = sock_info.command(source, cmd) parsed = _parse_scram_response(res['payload']) if not compare_digest(parsed[b'v'], server_sig): raise OperationFailure("Server returned an invalid signature.") # Depending on how it's configured, Cyrus SASL (which the server uses) # requires a third empty challenge. if not res['done']: cmd = SON([('saslContinue', 1), ('conversationId', res['conversationId']), ('payload', Binary(b''))]) res = sock_info.command(source, cmd) if not res['done']: raise OperationFailure('SASL conversation failed to complete.')
def test_before_request_unexpected_error(self, mock_rollback): mock_rollback.side_effect = OperationFailure('daamn!') with assert_raises(OperationFailure): handlers.transaction_before_request()
def _authenticate_gssapi(credentials, sock_info): """Authenticate using GSSAPI. """ if not HAVE_KERBEROS: raise ConfigurationError('The "kerberos" module must be ' 'installed to use GSSAPI authentication.') try: username = credentials.username password = credentials.password props = credentials.mechanism_properties # Starting here and continuing through the while loop below - establish # the security context. See RFC 4752, Section 3.1, first paragraph. host = sock_info.address[0] if props.canonicalize_host_name: host = socket.getfqdn(host) service = props.service_name + '@' + host if props.service_realm is not None: service = service + '@' + props.service_realm if password is not None: if '@' in username: user, domain = username.split('@', 1) else: user, domain = username, None result, ctx = kerberos.authGSSClientInit( service, gssflags=kerberos.GSS_C_MUTUAL_FLAG, user=user, domain=domain, password=password) else: result, ctx = kerberos.authGSSClientInit( service, gssflags=kerberos.GSS_C_MUTUAL_FLAG) if result != kerberos.AUTH_GSS_COMPLETE: raise OperationFailure('Kerberos context failed to initialize.') try: # pykerberos uses a weird mix of exceptions and return values # to indicate errors. # 0 == continue, 1 == complete, -1 == error # Only authGSSClientStep can return 0. if kerberos.authGSSClientStep(ctx, '') != 0: raise OperationFailure('Unknown kerberos ' 'failure in step function.') # Start a SASL conversation with mongod/s # Note: pykerberos deals with base64 encoded byte strings. # Since mongo accepts base64 strings as the payload we don't # have to use bson.binary.Binary. payload = kerberos.authGSSClientResponse(ctx) cmd = SON([('saslStart', 1), ('mechanism', 'GSSAPI'), ('payload', payload), ('autoAuthorize', 1)]) response = sock_info.command('$external', cmd) # Limit how many times we loop to catch protocol / library issues for _ in range(10): result = kerberos.authGSSClientStep(ctx, str(response['payload'])) if result == -1: raise OperationFailure('Unknown kerberos ' 'failure in step function.') payload = kerberos.authGSSClientResponse(ctx) or '' cmd = SON([('saslContinue', 1), ('conversationId', response['conversationId']), ('payload', payload)]) response = sock_info.command('$external', cmd) if result == kerberos.AUTH_GSS_COMPLETE: break else: raise OperationFailure('Kerberos ' 'authentication failed to complete.') # Once the security context is established actually authenticate. # See RFC 4752, Section 3.1, last two paragraphs. if kerberos.authGSSClientUnwrap(ctx, str( response['payload'])) != 1: raise OperationFailure('Unknown kerberos ' 'failure during GSS_Unwrap step.') if kerberos.authGSSClientWrap( ctx, kerberos.authGSSClientResponse(ctx), username) != 1: raise OperationFailure('Unknown kerberos ' 'failure during GSS_Wrap step.') payload = kerberos.authGSSClientResponse(ctx) cmd = SON([('saslContinue', 1), ('conversationId', response['conversationId']), ('payload', payload)]) sock_info.command('$external', cmd) finally: kerberos.authGSSClientClean(ctx) except kerberos.KrbError as exc: raise OperationFailure(str(exc))
def mock_next(self, *args, **kwargs): self._CommandCursor__killed = True raise OperationFailure('Mock server error')
def test_authenticate_fails(): db = create_autospec(Database) error = "command SON([('saslStart', 1), ('mechanism', 'SCRAM-SHA-1'), ('payload', Binary('n,,n=foo,r=OTI3MzA3MTEzMTIx', 0)), ('autoAuthorize', 1)]) on namespace admin.$cmd failed: Authentication failed." db.authenticate.side_effect = OperationFailure(error) assert auth.authenticate(db, sentinel.user, sentinel.password) is False
def check_indexes(document_class): """Ensures indexes are correct. If any indexes are missing they will be created. If any of them are 'wrong' (fields have changed, etc.) all the indexes for that collection will be dropped and rebuilt. Args: document_class (Document): The document class Returns: None Raises: mongoengine.OperationFailure: Unhandled mongo error """ from mongoengine.connection import get_db from pymongo.errors import OperationFailure from .models import Request try: # Building the indexes could take a while so it'd be nice to give some # indication of what's happening. This would be perfect but can't use # it! It's broken for text indexes!! MongoEngine is awesome!! # diff = collection.compare_indexes(); if diff['missing'] is not None... # Since we can't ACTUALLY compare the index spec with what already # exists without ridiculous effort: spec = document_class.list_indexes() existing = document_class._get_collection().index_information() if document_class == Request and "parent_instance_index" in existing: raise OperationFailure("Old Request index found, rebuilding") if len(spec) < len(existing): raise OperationFailure("Extra index found, rebuilding") if len(spec) > len(existing): logger.warning( "Found missing %s indexes, about to build them. This could " "take a while :)", document_class.__name__, ) document_class.ensure_indexes() except OperationFailure: logger.warning( "%s collection indexes verification failed, attempting to rebuild", document_class.__name__, ) # Unfortunately mongoengine sucks. The index that failed is only # returned as part of the error message. I REALLY don't want to parse # an error string to find the index to drop. Also, ME only verifies / # creates the indexes in bulk - there's no way to iterate through the # index definitions and try them one by one. Since our indexes should be # small and built in the background anyway just redo all of them try: db = get_db() db[document_class.__name__.lower()].drop_indexes() logger.warning("Dropped indexes for %s collection", document_class.__name__) except OperationFailure: logger.error( "Dropping %s indexes failed, please check the database configuration", document_class.__name__, ) raise if document_class == Request: logger.warning( "Request definition is potentially out of date. About to check and " "update if necessary - this could take several minutes.") # bg-utils 2.3.3 -> 2.3.4 create the `has_parent` field _update_request_has_parent_model() # bg-utils 2.4.6 -> 2.4.7 change parent to ReferenceField _update_request_parent_field_type() logger.warning("Request definition check/update complete.") try: document_class.ensure_indexes() logger.warning("%s indexes rebuilt successfully", document_class.__name__) except OperationFailure: logger.error( "%s index rebuild failed, please check the database configuration", document_class.__name__, ) raise
def _authenticate_scram(credentials, sock_info, mechanism): """Authenticate using SCRAM.""" username = credentials.username if mechanism == "SCRAM-SHA-256": digest = "sha256" digestmod = hashlib.sha256 data = saslprep(credentials.password).encode("utf-8") else: digest = "sha1" digestmod = hashlib.sha1 data = _password_digest(username, credentials.password).encode("utf-8") source = credentials.source cache = credentials.cache # Make local _hmac = hmac.HMAC ctx = sock_info.auth_ctx if ctx and ctx.speculate_succeeded(): nonce, first_bare = ctx.scram_data res = ctx.speculative_authenticate else: nonce, first_bare, cmd = _authenticate_scram_start( credentials, mechanism) res = sock_info.command(source, cmd) server_first = res["payload"] parsed = _parse_scram_response(server_first) iterations = int(parsed[b"i"]) if iterations < 4096: raise OperationFailure("Server returned an invalid iteration count.") salt = parsed[b"s"] rnonce = parsed[b"r"] if not rnonce.startswith(nonce): raise OperationFailure("Server returned an invalid nonce.") without_proof = b"c=biws,r=" + rnonce if cache.data: client_key, server_key, csalt, citerations = cache.data else: client_key, server_key, csalt, citerations = None, None, None, None # Salt and / or iterations could change for a number of different # reasons. Either changing invalidates the cache. if not client_key or salt != csalt or iterations != citerations: salted_pass = hashlib.pbkdf2_hmac(digest, data, standard_b64decode(salt), iterations) client_key = _hmac(salted_pass, b"Client Key", digestmod).digest() server_key = _hmac(salted_pass, b"Server Key", digestmod).digest() cache.data = (client_key, server_key, salt, iterations) stored_key = digestmod(client_key).digest() auth_msg = b",".join((first_bare, server_first, without_proof)) client_sig = _hmac(stored_key, auth_msg, digestmod).digest() client_proof = b"p=" + standard_b64encode(_xor(client_key, client_sig)) client_final = b",".join((without_proof, client_proof)) server_sig = standard_b64encode( _hmac(server_key, auth_msg, digestmod).digest()) cmd = SON([ ("saslContinue", 1), ("conversationId", res["conversationId"]), ("payload", Binary(client_final)), ]) res = sock_info.command(source, cmd) parsed = _parse_scram_response(res["payload"]) if not hmac.compare_digest(parsed[b"v"], server_sig): raise OperationFailure("Server returned an invalid signature.") # A third empty challenge may be required if the server does not support # skipEmptyExchange: SERVER-44857. if not res["done"]: cmd = SON([ ("saslContinue", 1), ("conversationId", res["conversationId"]), ("payload", Binary(b"")), ]) res = sock_info.command(source, cmd) if not res["done"]: raise OperationFailure("SASL conversation failed to complete.")
def get_movies_faceted(filters, page, movies_per_page): """ Returns movies and runtime and ratings facets. Also returns the total movies matched by the filter. Uses the same sort_key as get_movies """ sort_key = "tomatoes.viewer.numReviews" pipeline = [] if "cast" in filters: pipeline.extend([{ "$match": { "cast": { "$in": filters.get("cast") } } }, { "$sort": { sort_key: DESCENDING } }]) else: raise AssertionError("No filters to pass to faceted search!") counting = pipeline[:] count_stage = {"$count": "count"} counting.append(count_stage) skip_stage = {"$skip": movies_per_page * page} limit_stage = {"$limit": movies_per_page} facet_stage = { "$facet": { "runtime": [{ "$bucket": { "groupBy": "$runtime", "boundaries": [0, 60, 90, 120, 180], "default": "other", "output": { "count": { "$sum": 1 } } } }], "rating": [{ "$bucket": { "groupBy": "$metacritic", "boundaries": [0, 50, 70, 90, 100], "default": "other", "output": { "count": { "$sum": 1 } } } }], "movies": [{ "$addFields": { "title": "$title" } }] } } pipeline.extend([skip_stage, limit_stage, facet_stage]) try: movies = list(db.movies.aggregate(pipeline, allowDiskUse=True))[0] count = list(db.movies.aggregate(counting, allowDiskUse=True))[0].get("count") return (movies, count) except OperationFailure: raise OperationFailure( "Results too large to sort, be more restrictive in filter")
def execute_no_results(self, sock_info, generator): """Execute all operations, returning no results (w=0). """ if self.uses_collation: raise ConfigurationError( 'Collation is unsupported for unacknowledged writes.') if self.uses_array_filters: raise ConfigurationError( 'arrayFilters is unsupported for unacknowledged writes.') # Cannot have both unacknowledged writes and bypass document validation. if self.bypass_doc_val and sock_info.max_wire_version >= 4: raise OperationFailure("Cannot set bypass_document_validation with" " unacknowledged write concern") # OP_MSG if sock_info.max_wire_version > 5: if self.ordered: return self.execute_command_no_results(sock_info, generator) return self.execute_op_msg_no_results(sock_info, generator) coll = self.collection # If ordered is True we have to send GLE or use write # commands so we can abort on the first error. write_concern = WriteConcern(w=int(self.ordered)) op_id = _randint() next_run = next(generator) while next_run: # An ordered bulk write needs to send acknowledged writes to short # circuit the next run. However, the final message on the final # run can be unacknowledged. run = next_run next_run = next(generator, None) needs_ack = self.ordered and next_run is not None try: if run.op_type == _INSERT: self.execute_insert_no_results(sock_info, run, op_id, needs_ack) elif run.op_type == _UPDATE: for operation in run.ops: doc = operation['u'] check_keys = True if doc and next(iter(doc)).startswith('$'): check_keys = False coll._update(sock_info, operation['q'], doc, operation['upsert'], check_keys, operation['multi'], write_concern=write_concern, op_id=op_id, ordered=self.ordered, bypass_doc_val=self.bypass_doc_val) else: for operation in run.ops: coll._delete(sock_info, operation['q'], not operation['limit'], write_concern, op_id, self.ordered) except OperationFailure: if self.ordered: break
def update_one(self, *args, **kwargs): raise OperationFailure(Exception())
def _copy_database( fromdb, todb, fromhost, mechanism, username, password, sock_info, cmd_func): """Copy a database, perhaps from a remote host. :Parameters: - `fromdb`: Source database. - `todb`: Target database. - `fromhost`: Source host like 'foo.com', 'foo.com:27017', or None. - `mechanism`: An authentication mechanism. - `username`: A str or unicode, or None. - `password`: A str or unicode, or None. - `sock_info`: A SocketInfo instance. - `cmd_func`: A callback taking args sock_info, database, command doc. """ if not isinstance(fromdb, basestring): raise TypeError('from_name must be an instance ' 'of %s' % (basestring.__name__,)) if not isinstance(todb, basestring): raise TypeError('to_name must be an instance ' 'of %s' % (basestring.__name__,)) _check_database_name(todb) warnings.warn("copy_database is deprecated. Use the raw 'copydb' command" " or db.copyDatabase() in the mongo shell. See" " doc/examples/copydb.", DeprecationWarning, stacklevel=2) # It would be better if the user told us what mechanism to use, but for # backwards compatibility with earlier PyMongos we don't require the # mechanism. Hope 'fromhost' runs the same version as the target. if mechanism == 'DEFAULT': if sock_info.max_wire_version >= 3: mechanism = 'SCRAM-SHA-1' else: mechanism = 'MONGODB-CR' if username is not None: if mechanism == 'SCRAM-SHA-1': credentials = auth._build_credentials_tuple(mech=mechanism, source='admin', user=username, passwd=password, extra=None) try: auth._copydb_scram_sha1(credentials=credentials, sock_info=sock_info, cmd_func=cmd_func, fromdb=fromdb, todb=todb, fromhost=fromhost) except OperationFailure, exc: errmsg = exc.details and exc.details.get('errmsg') or '' if 'no such cmd: saslStart' in errmsg: explanation = ( "%s doesn't support SCRAM-SHA-1, pass" " mechanism='MONGODB-CR' to copy_database" % fromhost) raise OperationFailure(explanation, exc.code, exc.details) else: raise elif mechanism == 'MONGODB-CR': get_nonce_cmd = SON([('copydbgetnonce', 1), ('fromhost', fromhost)]) get_nonce_response, _ = cmd_func(sock_info, 'admin', get_nonce_cmd) nonce = get_nonce_response['nonce'] copydb_cmd = SON([('copydb', 1), ('fromdb', fromdb), ('todb', todb)]) copydb_cmd['username'] = username copydb_cmd['nonce'] = nonce copydb_cmd['key'] = auth._auth_key(nonce, username, password) if fromhost is not None: copydb_cmd['fromhost'] = fromhost cmd_func(sock_info, 'admin', copydb_cmd) else: raise InvalidOperation('Authentication mechanism %r not supported' ' for copy_database' % mechanism)
def _check_command_response(response, max_wire_version, allowable_errors=None, parse_write_concern_error=False): """Check the response to a command for errors. """ if "ok" not in response: # Server didn't recognize our message as a command. raise OperationFailure(response.get("$err"), response.get("code"), response, max_wire_version) if parse_write_concern_error and 'writeConcernError' in response: _error = response["writeConcernError"] _labels = response.get("errorLabels") if _labels: _error.update({'errorLabels': _labels}) _raise_write_concern_error(_error) if response["ok"]: return details = response # Mongos returns the error details in a 'raw' object # for some errors. if "raw" in response: for shard in response["raw"].values(): # Grab the first non-empty raw error from a shard. if shard.get("errmsg") and not shard.get("ok"): details = shard break errmsg = details["errmsg"] code = details.get("code") # For allowable errors, only check for error messages when the code is not # included. if allowable_errors: if code is not None: if code in allowable_errors: return elif errmsg in allowable_errors: return # Server is "not master" or "recovering" if code is not None: if code in _NOT_MASTER_CODES: raise NotMasterError(errmsg, response) elif "not master" in errmsg or "node is recovering" in errmsg: raise NotMasterError(errmsg, response) # Other errors # findAndModify with upsert can raise duplicate key error if code in (11000, 11001, 12582): raise DuplicateKeyError(errmsg, code, response, max_wire_version) elif code == 50: raise ExecutionTimeout(errmsg, code, response, max_wire_version) elif code == 43: raise CursorNotFound(errmsg, code, response, max_wire_version) raise OperationFailure(errmsg, code, response, max_wire_version)
def get_movies_faceted(filters, page, movies_per_page): """ Returns movies and runtime and ratings facets. Also returns the total movies matched by the filter. Uses the same sort_key as get_movies """ sort_key = "tomatoes.viewer.numReviews" pipeline = [] if "cast" in filters: pipeline.extend([{ "$match": { "cast": { "$in": filters.get("cast") } } }, { "$sort": { sort_key: DESCENDING } }]) else: raise AssertionError("No filters to pass to faceted search!") counting = pipeline[:] count_stage = {"$count": "count"} counting.append(count_stage) skip_stage = {"$skip": movies_per_page * page} limit_stage = {"$limit": movies_per_page} facet_stage = { "$facet": { "runtime": [{ "$bucket": { "groupBy": "$runtime", "boundaries": [0, 60, 90, 120, 180], "default": "other", "output": { "count": { "$sum": 1 } } } }], "rating": [{ "$bucket": { "groupBy": "$metacritic", "boundaries": [0, 50, 70, 90, 100], "default": "other", "output": { "count": { "$sum": 1 } } } }], "movies": [{ "$addFields": { "title": "$title" } }] } } """ Ticket: Faceted Search Please append the skip_stage, limit_stage, and facet_stage to the pipeline (in that order). The pipeline is a Python array, so you can use append() or extend() to complete this task. """ # TODO: Faceted Search # Add the necessary stages to the pipeline variable in the correct order. #pipeline = [] pipeline.append(skip_stage) pipeline.append(limit_stage) pipeline.append(facet_stage) try: movies = list(db.movies.aggregate(pipeline, allowDiskUse=True))[0] count = list(db.movies.aggregate(counting, allowDiskUse=True))[0].get("count") return (movies, count) except OperationFailure: raise OperationFailure( "Results too large to sort, be more restrictive in filter")
def test_unicode_strs_operation_failure(self): exc = OperationFailure('unicode \U0001f40d', 10, {"errmsg": 'unicode \U0001f40d'}) self._test_unicode_strs(exc)
def _authenticate_gssapi(credentials, sock_info, cmd_func): """Authenticate using GSSAPI. """ try: dummy, username, gsn = credentials # Starting here and continuing through the while loop below - establish # the security context. See RFC 4752, Section 3.1, first paragraph. result, ctx = kerberos.authGSSClientInit(gsn + '@' + sock_info.host, kerberos.GSS_C_MUTUAL_FLAG) if result != kerberos.AUTH_GSS_COMPLETE: raise OperationFailure('Kerberos context failed to initialize.') try: # pykerberos uses a weird mix of exceptions and return values # to indicate errors. # 0 == continue, 1 == complete, -1 == error # Only authGSSClientStep can return 0. if kerberos.authGSSClientStep(ctx, '') != 0: raise OperationFailure('Unknown kerberos ' 'failure in step function.') # Start a SASL conversation with mongod/s # Note: pykerberos deals with base64 encoded byte strings. # Since mongo accepts base64 strings as the payload we don't # have to use bson.binary.Binary. payload = kerberos.authGSSClientResponse(ctx) cmd = SON([('saslStart', 1), ('mechanism', 'GSSAPI'), ('payload', payload), ('autoAuthorize', 1)]) response, _ = cmd_func(sock_info, '$external', cmd) # Limit how many times we loop to catch protocol / library issues for _ in xrange(10): result = kerberos.authGSSClientStep(ctx, str(response['payload'])) if result == -1: raise OperationFailure('Unknown kerberos ' 'failure in step function.') payload = kerberos.authGSSClientResponse(ctx) or '' cmd = SON([('saslContinue', 1), ('conversationId', response['conversationId']), ('payload', payload)]) response, _ = cmd_func(sock_info, '$external', cmd) if result == kerberos.AUTH_GSS_COMPLETE: break else: raise OperationFailure('Kerberos ' 'authentication failed to complete.') # Once the security context is established actually authenticate. # See RFC 4752, Section 3.1, last two paragraphs. if kerberos.authGSSClientUnwrap(ctx, str( response['payload'])) != 1: raise OperationFailure('Unknown kerberos ' 'failure during GSS_Unwrap step.') if kerberos.authGSSClientWrap( ctx, kerberos.authGSSClientResponse(ctx), username) != 1: raise OperationFailure('Unknown kerberos ' 'failure during GSS_Wrap step.') payload = kerberos.authGSSClientResponse(ctx) cmd = SON([('saslContinue', 1), ('conversationId', response['conversationId']), ('payload', payload)]) response, _ = cmd_func(sock_info, '$external', cmd) finally: kerberos.authGSSClientClean(ctx) except kerberos.KrbError, exc: raise OperationFailure(str(exc))
def test_pickle_OperationFailure(self): exc = OperationFailure('error', code=5, details={}, max_wire_version=7) self.assertOperationFailureEqual(exc, pickle.loads(pickle.dumps(exc)))
def _scram_sha1_conversation(credentials, sock_info, cmd_func, sasl_start, sasl_continue): """Authenticate or copydb using SCRAM-SHA-1. sasl_start and sasl_continue are SONs, the base command documents for beginning and continuing the SASL conversation. They may be modified by the callee. :Parameters: - `credentials`: A credentials tuple from _build_credentials_tuple. - `sock_info`: A SocketInfo instance. - `cmd_func`: A callback taking args sock_info, database, command doc. - `sasl_start`: A SON. - `sasl_continue`: A SON. """ source, username, password = credentials # Make local _hmac = hmac.HMAC _sha1 = _SHA1 _sha1mod = _SHA1MOD user = username.encode("utf-8").replace(_EQUAL, b("=3D")).replace( _COMMA, b("=2C")) nonce = standard_b64encode( (("%s" % (SystemRandom().random(), ))[2:]).encode("utf-8")) first_bare = b("n=") + user + b(",r=") + nonce sasl_start['payload'] = Binary(b("n,,") + first_bare) res, _ = cmd_func(sock_info, source, sasl_start) server_first = res['payload'] parsed = _parse_scram_response(server_first) iterations = int(parsed[b('i')]) salt = parsed[b('s')] rnonce = parsed[b('r')] if not rnonce.startswith(nonce): raise OperationFailure("Server returned an invalid nonce.") without_proof = b("c=biws,r=") + rnonce salted_pass = _hi( _password_digest(username, password).encode("utf-8"), standard_b64decode(salt), iterations) client_key = _hmac(salted_pass, b("Client Key"), _sha1mod).digest() stored_key = _sha1(client_key).digest() auth_msg = _COMMA.join((first_bare, server_first, without_proof)) client_sig = _hmac(stored_key, auth_msg, _sha1mod).digest() client_proof = b("p=") + standard_b64encode(_xor(client_key, client_sig)) client_final = _COMMA.join((without_proof, client_proof)) server_key = _hmac(salted_pass, b("Server Key"), _sha1mod).digest() server_sig = standard_b64encode( _hmac(server_key, auth_msg, _SHA1MOD).digest()) cmd = sasl_continue.copy() cmd['conversationId'] = res['conversationId'] cmd['payload'] = Binary(client_final) res, _ = cmd_func(sock_info, source, cmd) parsed = _parse_scram_response(res['payload']) if not compare_digest(parsed[b('v')], server_sig): raise OperationFailure("Server returned an invalid signature.") # Depending on how it's configured, Cyrus SASL (which the server uses) # requires a third empty challenge. if not res['done']: cmd = sasl_continue.copy() cmd['conversationId'] = res['conversationId'] cmd['payload'] = Binary(_EMPTY) res, _ = cmd_func(sock_info, source, cmd) if not res['done']: raise OperationFailure('SASL conversation failed to complete.')
def _aws_temp_credentials(): """Construct temporary MONGODB-AWS credentials.""" access_key = os.environ.get('AWS_ACCESS_KEY_ID') secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY') if access_key and secret_key: return _AWSCredential(access_key, secret_key, os.environ.get('AWS_SESSION_TOKEN')) # If the environment variable # AWS_CONTAINER_CREDENTIALS_RELATIVE_URI is set then drivers MUST # assume that it was set by an AWS ECS agent and use the URI # http://169.254.170.2/$AWS_CONTAINER_CREDENTIALS_RELATIVE_URI to # obtain temporary credentials. relative_uri = os.environ.get('AWS_CONTAINER_CREDENTIALS_RELATIVE_URI') if relative_uri is not None: try: res = requests.get(_AWS_REL_URI + relative_uri, timeout=_AWS_HTTP_TIMEOUT) res_json = res.json() except (ValueError, requests.exceptions.RequestException): raise OperationFailure( 'temporary MONGODB-AWS credentials could not be obtained') else: # If the environment variable AWS_CONTAINER_CREDENTIALS_RELATIVE_URI is # not set drivers MUST assume we are on an EC2 instance and use the # endpoint # http://169.254.169.254/latest/meta-data/iam/security-credentials # /<role-name> # whereas role-name can be obtained from querying the URI # http://169.254.169.254/latest/meta-data/iam/security-credentials/. try: # Get token headers = {'X-aws-ec2-metadata-token-ttl-seconds': "30"} res = requests.post(_AWS_EC2_URI + 'latest/api/token', headers=headers, timeout=_AWS_HTTP_TIMEOUT) token = res.content # Get role name headers = {'X-aws-ec2-metadata-token': token} res = requests.get(_AWS_EC2_URI + _AWS_EC2_PATH, headers=headers, timeout=_AWS_HTTP_TIMEOUT) role = res.text # Get temp creds res = requests.get(_AWS_EC2_URI + _AWS_EC2_PATH + role, headers=headers, timeout=_AWS_HTTP_TIMEOUT) res_json = res.json() except (ValueError, requests.exceptions.RequestException): raise OperationFailure( 'temporary MONGODB-AWS credentials could not be obtained') try: temp_user = res_json['AccessKeyId'] temp_password = res_json['SecretAccessKey'] token = res_json['Token'] except KeyError: # If temporary credentials cannot be obtained then drivers MUST # fail authentication and raise an error. raise OperationFailure( 'temporary MONGODB-AWS credentials could not be obtained') return _AWSCredential(temp_user, temp_password, token)