def new_deployment(username): user = db.User.find_one({'username': username}) if user is None or (user is not None and not current_user.is_admin() and current_user != user): # No permission flash("Permission denied", 'danger') return redirect(url_for("index")) form = NewDeploymentForm() if form.validate_on_submit(): deployment_date = dateparse(form.deployment_date.data) delayed_mode = form.delayed_mode.data deployment_name = form.glider_name.data + '-' + \ deployment_date.strftime('%Y%m%dT%H%M') if delayed_mode: deployment_name += '-delayed' deployment = db.Deployment() # form.populate_obj(deployment) deployment.user_id = user._id deployment.username = username deployment.deployment_dir = os.path.join(username, deployment_name) deployment.updated = datetime.utcnow() deployment.deployment_date = deployment_date deployment.glider_name = form.glider_name.data deployment.name = deployment_name deployment.attribution = form.attribution.data deployment.delayed_mode = delayed_mode try: existing_deployment = db.Deployment.find_one( {'name': deployment_name}) if existing_deployment is not None: raise DuplicateKeyError("Duplicate Key Detected: name") existing_deployment = db.Deployment.find_one({ 'glider_name': form.glider_name.data, 'deployment_date': deployment_date }) if existing_deployment is not None: raise DuplicateKeyError( "Duplicate Key Detected: glider_name and deployment_date") deployment.save() flash("Deployment created", 'success') send_registration_email(username, deployment) except DuplicateKeyError: flash( "Deployment names must be unique across Glider DAC: %s already used" % deployment.name, 'danger') else: error_str = ", ".join( ["%s: %s" % (k, ", ".join(v)) for k, v in form.errors.iteritems()]) flash("Deployment could not be created: %s" % error_str, 'danger') return redirect(url_for('list_user_deployments', username=username))
async def test_update_by_id_pymongo_raises_duplicate_key_error_failure_on_retry( app, event): with patch.object(app["db"].event.collection, "find_one_and_replace", new=CoroutineMock())\ as mock_replace: mock_replace.side_effect = [ DuplicateKeyError("error after first run"), DuplicateKeyError("error after second run") ] with pytest.raises(DuplicateKeyError, match="error after second run"): await app["db"].event.update_by_id(event.id, event)
def add_user(name, email, hashedpw): """ Given a name, email and password, inserts a document with those credentials to the `users` collection. """ """ Ticket: Durable Writes Please increase the durability of this method by using a non-default write concern with ``insert_one``. """ try: # TODO: User Management # Insert a user with the "name", "email", and "password" fields. user = {"name": name, "email": email, "password": hashedpw} if get_user(email): raise DuplicateKeyError('error') # TODO: Durable Writes # Use a more durable Write Concern for this operation. w = WriteConcern(w="majority") while (not db.users.with_options(write_concern=w)): pass db.users.insert_one(user) return {"success": True} except DuplicateKeyError: return {"error": "A user with the given email already exists."}
def test_add_status_already_exists(status, statuses): ''' Test duplication detection for add_status ''' with patch('user_status.sql'): statuses.table.insert_one.side_effect = DuplicateKeyError("ERR_MSG") assert not statuses.add_status(*status)
def update_approval_status(data): if not deploymentUnitApprovalStatusDB.GetDeploymentUnitApprovalStatusById(data.get("_id").get("oid")): raise Exception("The status exists with the object id : "+ data.get("_id").get("oid")) try: return deploymentUnitApprovalStatusDB.UpdateDeploymentUnitApprovalStatus(data) except DuplicateKeyError as e: raise DuplicateKeyError("The status exists with same name")
def put(self, data, _id, content_type, filename, **kw): """upload file-only. Can not handle bookmarks.""" if _id in ['thumb', 'items', 'login']: raise DuplicateKeyError item_type, subtype = content_type.split('/', 1) if item_type in ['image', 'text', 'audio', 'video']: pass elif item_type == 'application' and \ filter(lambda k: subtype.find(k) > -1, ['compress', 'zip', 'tar']): item_type = 'archive' else: item_type = 'unknown' if self.mdb.find_one({'short_id': kw['short_id']}): raise DuplicateKeyError('short_id already exists') _id = self.gfs.put(data, _id=_id, content_type=content_type, filename=filename) kw.update({'_id': _id, 'item_type': item_type}) self.mdb.insert(kw) return _id
def test_add(self, mock_parser, mock_doc_init): bill = "bill" with self.subTest("when invalid schema"): mock_parser.side_effect = ValueError with self.assertRaises(ValueError): self.repository.add(bill) mock_parser.side_effect = None mock_parser.return_value = "parsed" with self.subTest("when has duplicate keys "): self.mock_collection.insert_many.side_effect = DuplicateKeyError( Exception()) with self.assertRaises(errors.UnprocessableDataError): self.repository.add(bill) self.mock_collection.insert_many.side_effect = None with self.subTest("when inserted with success"): actual = self.repository.add(bill) mock_parser.assert_called_with(bill, [schema.PHONE_BILL]) self.assertIsInstance(actual, Documents)
def test_add_user_failure(mock_collection, normal_user): mock_collection.insert_one.side_effect = DuplicateKeyError('repeated username') with pytest.raises(HTTPException) as excinfo: add_user(normal_user) assert excinfo.value.status_code == 400
def test_duplicate_exc(self, get_collection): """ in case of lock retry later after lock timeout """ get_collection.return_value.insert_one.side_effect = DuplicateKeyError( "E11000 duplicate key error collection") test_method_procedure = Mock() @concurrency_lock def test_method(*args, **kwargs): test_method_procedure(*args, **kwargs) dummy_task.retry = Mock(side_effect=Retry) with patch("celery_worker.locks.datetime") as datetime_mock: datetime_mock.utcnow.return_value = datetime.utcnow() with self.assertRaises(Retry): test_method(dummy_task) # run task_uid = args_to_uid( (test_method.__module__, test_method.__name__, tuple(), dict())) get_collection.return_value.insert_one.assert_called_once_with({ "_id": task_uid, "expireAt": datetime_mock.utcnow.return_value + timedelta(seconds=10) }) dummy_task.retry.assert_called_once_with(max_retries=20, countdown=70) test_method_procedure.assert_not_called()
async def test_update_by_id_pymongo_raises_duplicate_key_error_successful_on_retry( app, event): with patch.object(app["db"].event.collection, "find_one_and_replace", new=CoroutineMock())\ as mock_replace: mock_replace.side_effect = [DuplicateKeyError("error_msg"), 1] await app["db"].event.update_by_id(event.id, event)
def _check_command_response(response, max_wire_version, msg=None, allowable_errors=None, parse_write_concern_error=False): """Check the response to a command for errors. """ if "ok" not in response: # Server didn't recognize our message as a command. raise OperationFailure(response.get("$err"), response.get("code"), response, max_wire_version) if parse_write_concern_error and 'writeConcernError' in response: _raise_write_concern_error(response['writeConcernError']) if not response["ok"]: details = response # Mongos returns the error details in a 'raw' object # for some errors. if "raw" in response: for shard in itervalues(response["raw"]): # Grab the first non-empty raw error from a shard. if shard.get("errmsg") and not shard.get("ok"): details = shard break errmsg = details["errmsg"] if (allowable_errors is None or (errmsg not in allowable_errors and details.get("code") not in allowable_errors)): code = details.get("code") # Server is "not master" or "recovering" if code in _NOT_MASTER_CODES: raise NotMasterError(errmsg, response) elif ("not master" in errmsg or "node is recovering" in errmsg): raise NotMasterError(errmsg, response) # Server assertion failures if errmsg == "db assertion failure": errmsg = ("db assertion failure, assertion: '%s'" % details.get("assertion", "")) raise OperationFailure(errmsg, details.get("assertionCode"), response, max_wire_version) # Other errors # findAndModify with upsert can raise duplicate key error if code in (11000, 11001, 12582): raise DuplicateKeyError(errmsg, code, response, max_wire_version) elif code == 50: raise ExecutionTimeout(errmsg, code, response, max_wire_version) elif code == 43: raise CursorNotFound(errmsg, code, response, max_wire_version) msg = msg or "%s" raise OperationFailure(msg % errmsg, code, response, max_wire_version)
def _raise_last_write_error(write_errors): # If the last batch had multiple errors only report # the last error to emulate continue_on_error. error = write_errors[-1] if error.get("code") == 11000: raise DuplicateKeyError(error.get("errmsg"), 11000, error) raise WriteError(error.get("errmsg"), error.get("code"), error)
def test_append_insert_version_duplicatekey(): read_handler = Mock(append=Mock(__name__="")) previous_version = TPL_VERSION.copy() previous_version['version'] = 1 vs = create_autospec( VersionStore, instance=True, _collection=Mock(), _version_nums=Mock(find_one_and_update=Mock( return_value={'version': previous_version['version'] + 1})), _versions=Mock(insert_one=Mock(__name__="insert_one"), find_one=Mock(__name__="find_one", return_value=previous_version)), _arctic_lib=create_autospec(ArcticLibraryBinding, arctic=create_autospec( Arctic, mongo_host='some_host')), _publish_changes=False) vs._insert_version = lambda version: VersionStore._insert_version( vs, version) vs._versions.insert_one.side_effect = [ DuplicateKeyError("dup key error"), None ] vs._collection.database.connection.nodes = [] vs._read_handler.return_value = read_handler VersionStore.append(vs, 'sym', [1, 2, 3], prune_previous_version=False, upsert=False) assert vs._version_nums.find_one_and_update.call_count == 2 assert vs._versions.find_one.call_count == 2 assert read_handler.append.call_count == 2 assert vs._versions.insert_one.call_count == 2 assert vs._publish_change.call_count == 2
def _check_write_command_response(results): """Backward compatibility helper for write command error handling. """ errors = [ res for res in results if "writeErrors" in res[1] or "writeConcernError" in res[1] ] if errors: # If multiple batches had errors # raise from the last batch. offset, result = errors[-1] # Prefer write errors over write concern errors write_errors = result.get("writeErrors") if write_errors: # If the last batch had multiple errors only report # the last error to emulate continue_on_error. error = write_errors[-1] error["index"] += offset if error.get("code") == 11000: raise DuplicateKeyError(error.get("errmsg"), 11000, error) else: error = result["writeConcernError"] if "errInfo" in error and error["errInfo"].get('wtimeout'): # Make sure we raise WTimeoutError raise WTimeoutError(error.get("errmsg"), error.get("code"), error) raise OperationFailure(error.get("errmsg"), error.get("code"), error)
def test_add_user_already_exists(user, accounts): ''' Test that add_user responds to errors correctly ''' with patch('users.sql'): accounts.table.insert_one.side_effect = DuplicateKeyError("ERR_MSG") assert not accounts.add_user(*user)
def test_write_insert_version_duplicatekey(): write_handler = Mock(write=Mock(__name__="")) vs = create_autospec(VersionStore, instance=True, _collection=Mock(), _version_nums=Mock(find_one_and_update=Mock( return_value={'version': 1})), _versions=Mock(insert_one=Mock(__name__="insert_one"), find_one=Mock(__name__="find_one")), _arctic_lib=create_autospec( ArcticLibraryBinding, arctic=create_autospec(Arctic, mongo_host='some_host')), _publish_changes=False) vs._insert_version = lambda version: VersionStore._insert_version( vs, version) vs._versions.insert_one.side_effect = [ DuplicateKeyError("dup key error"), None ] vs._collection.database.connection.nodes = [] vs._write_handler.return_value = write_handler VersionStore.write(vs, 'sym', sentinel.data, prune_previous_version=False) assert vs._version_nums.find_one_and_update.call_count == 2 assert vs._versions.find_one.call_count == 2 assert write_handler.write.call_count == 2 assert vs._publish_change.call_count == 2 assert vs._versions.insert_one.call_count == 2
def _check_gle_response(result): """Return getlasterror response as a dict, or raise OperationFailure.""" # Did getlasterror itself fail? _check_command_response(result) if result.get("wtimeout", False): # MongoDB versions before 1.8.0 return the error message in an "errmsg" # field. If "errmsg" exists "err" will also exist set to None, so we # have to check for "errmsg" first. raise WTimeoutError(result.get("errmsg", result.get("err")), result.get("code"), result) error_msg = result.get("err", "") if error_msg is None: return result if error_msg.startswith("not master"): raise NotMasterError(error_msg, result) details = result # mongos returns the error code in an error object for some errors. if "errObjects" in result: for errobj in result["errObjects"]: if errobj.get("err") == error_msg: details = errobj break code = details.get("code") if code in (11000, 11001, 12582): raise DuplicateKeyError(details["err"], code, result) raise OperationFailure(details["err"], code, result)
def _check_command_response(response, max_wire_version, allowable_errors=None, parse_write_concern_error=False): """Check the response to a command for errors. """ if "ok" not in response: # Server didn't recognize our message as a command. raise OperationFailure(response.get("$err"), response.get("code"), response, max_wire_version) if parse_write_concern_error and 'writeConcernError' in response: _error = response["writeConcernError"] _labels = response.get("errorLabels") if _labels: _error.update({'errorLabels': _labels}) _raise_write_concern_error(_error) if response["ok"]: return details = response # Mongos returns the error details in a 'raw' object # for some errors. if "raw" in response: for shard in response["raw"].values(): # Grab the first non-empty raw error from a shard. if shard.get("errmsg") and not shard.get("ok"): details = shard break errmsg = details["errmsg"] code = details.get("code") # For allowable errors, only check for error messages when the code is not # included. if allowable_errors: if code is not None: if code in allowable_errors: return elif errmsg in allowable_errors: return # Server is "not primary" or "recovering" if code is not None: if code in _NOT_MASTER_CODES: raise NotPrimaryError(errmsg, response) elif "not master" in errmsg or "node is recovering" in errmsg: raise NotPrimaryError(errmsg, response) # Other errors # findAndModify with upsert can raise duplicate key error if code in (11000, 11001, 12582): raise DuplicateKeyError(errmsg, code, response, max_wire_version) elif code == 50: raise ExecutionTimeout(errmsg, code, response, max_wire_version) elif code == 43: raise CursorNotFound(errmsg, code, response, max_wire_version) raise OperationFailure(errmsg, code, response, max_wire_version)
def __check_response_to_last_error(self, response): """Check a response to a lastError message for errors. `response` is a byte string representing a response to the message. If it represents an error response we raise OperationFailure. Return the response as a document. """ response = helpers._unpack_response(response) assert response["number_returned"] == 1 error = response["data"][0] # TODO unify logic with database.error method if error.get("err", 0) is None: return error if error["err"] == "not master": self._reset() if "code" in error and error["code"] in [11000, 11001]: raise DuplicateKeyError(error["err"]) else: raise OperationFailure(error["err"]) return error
def test_save_unit_new_unit_race_condition(self, mock_associate, mock_add, mock_update, mock_get, mock_path): """ This simulates a case where the same unit gets added by another workflow before the save completes. In that case, the failover behavior is to update the unit instead of adding it. """ # Setup unit = self.mixin.init_unit('t', {'k': 'v'}, {'m': 'm1'}, '/bar') mock_add.side_effect = DuplicateKeyError('dups!') # raise an exception the first time around, then simulate the unit # having appeared since the last call. mock_get.side_effect = _create_mock_side_effect( [MissingResource, { '_id': 'existing' }]) # Test saved = self.mixin.save_unit(unit) # Verify self.assertEqual(2, mock_get.call_count) self.assertEqual(1, mock_update.call_count) self.assertEqual(1, mock_add.call_count) self.assertEqual(1, mock_associate.call_count) self.assertEqual(0, self.mixin._added_count) self.assertEqual(1, self.mixin._updated_count) self.assertEqual(saved.id, 'existing')
def __update(self, spec, updates, upsert=False, multi=False): bson_safe(spec) bson_safe(updates) result = dict( connectionId=None, updatedExisting=False, err=None, ok=1.0, n=0, nModified=0 ) for doc, mspec in self._find(spec): self._deindex(doc) mspec.update(updates) self._index(doc) result['n'] += 1 result['nModified'] += 1 if not multi: break if result['n']: result['updatedExisting'] = True return result if upsert: doc = dict(spec) MatchDoc(doc).update(updates, upserted=upsert) _id = doc.get('_id', ()) if _id == (): _id = doc['_id'] = bson.ObjectId() if _id in self._data: raise DuplicateKeyError('duplicate ID on upsert') self._index(doc) self._data[_id] = bcopy(doc) result['upserted'] = _id return result else: return result
def test_add_institution_failure(collection_mock, bank): collection_mock.insert_one.side_effect = DuplicateKeyError( 'Bank already exists') with pytest.raises(HTTPException) as excinfo: add_institution(bank) assert excinfo.value.status_code == 400
def __check_response_to_last_error(self, response): """Check a response to a lastError message for errors. `response` is a byte string representing a response to the message. If it represents an error response we raise OperationFailure. Return the response as a document. """ response = helpers._unpack_response(response) assert response["number_returned"] == 1 error = response["data"][0] helpers._check_command_response(error, self.disconnect) error_msg = error.get("err", "") if error_msg is None: return error if error_msg.startswith("not master"): self.disconnect() raise AutoReconnect(error_msg) if "code" in error: if error["code"] in [11000, 11001, 12582]: raise DuplicateKeyError(error["err"]) else: raise OperationFailure(error["err"], error["code"]) else: raise OperationFailure(error["err"])
def insert_one(doc): """ Pretend to add the document to the database. Throw an error if the document is already there, otherwise return a class within a "inserted_id". """ results = [] for datum in data: match = True counter = 0 for key in doc: if key not in datum or key == "date" or key == "data": continue counter += 1 if doc[key] != datum[key]: match = False if match and counter: results.append(doc) if results: raise DuplicateKeyError("bla") else: class Result: def __init__(self): self.inserted_id = ObjectId("58dbe045ef677d54224a01d2") return Result()
def test_add_currency_failure(collection_mock, institutions_mock, currency_in): collection_mock.insert_one.side_effect = DuplicateKeyError( 'currency already exists') with pytest.raises(HTTPException) as excinfo: add_instrument(currency_in) assert excinfo.value.status_code == 400 assert not institutions_mock.find_one.called
def _insert(self, data): if not '_id' in data: data['_id'] = ObjectId() object_id = data['_id'] if object_id in self._documents: raise DuplicateKeyError("Duplicate Key Error", 11000) self._documents[object_id] = copy.deepcopy(data) return object_id
def test_write_metadata_insert_version_dupkeyerror(): vs = _create_mock_versionstore() vs._versions.insert_one.__name__ = 'insert_one' vs._versions.insert_one.side_effect = [DuplicateKeyError('dup key error'), None] VersionStore.write_metadata(vs, symbol=TEST_SYMBOL, metadata=META_TO_WRITE) assert vs._version_nums.find_one_and_update.call_count == 2 assert vs._versions.insert_one.call_count == 2 assert vs._publish_change.call_count == 1
def test_add_account_cash_duplicate(mock_collection, mock_institutions, account_cash_in, normal_user): mock_collection.insert_one.side_effect = DuplicateKeyError('account already exists') with pytest.raises(HTTPException) as excinfo: add_account(account_cash_in, normal_user) assert excinfo.value.status_code == 400 assert not mock_institutions.find_one.called
def before_insert(self, instance, state, session): doc_window_hash = hashlib.sha1( str(instance.document_id) + str(instance.window_size)).hexdigest() if instance.__class__.query.find({ 'doc_window_hash': doc_window_hash }).count() > 0: raise DuplicateKeyError('Duplicate hash found ', doc_window_hash) instance.doc_window_hash = doc_window_hash
def _check_command_response(response, msg=None, allowable_errors=None): """Check the response to a command for errors. """ if "ok" not in response: # Server didn't recognize our message as a command. raise OperationFailure(response.get("$err"), response.get("code"), response) # TODO: remove, this is moving to _check_gle_response if response.get("wtimeout", False): # MongoDB versions before 1.8.0 return the error message in an "errmsg" # field. If "errmsg" exists "err" will also exist set to None, so we # have to check for "errmsg" first. raise WTimeoutError(response.get("errmsg", response.get("err")), response.get("code"), response) if not response["ok"]: details = response # Mongos returns the error details in a 'raw' object # for some errors. if "raw" in response: for shard in itervalues(response["raw"]): # Grab the first non-empty raw error from a shard. if shard.get("errmsg") and not shard.get("ok"): details = shard break errmsg = details["errmsg"] if allowable_errors is None or errmsg not in allowable_errors: # Server is "not master" or "recovering" if (errmsg.startswith("not master") or errmsg.startswith("node is recovering")): raise NotMasterError(errmsg, response) # Server assertion failures if errmsg == "db assertion failure": errmsg = ("db assertion failure, assertion: '%s'" % details.get("assertion", "")) raise OperationFailure(errmsg, details.get("assertionCode"), response) # Other errors code = details.get("code") # findAndModify with upsert can raise duplicate key error if code in (11000, 11001, 12582): raise DuplicateKeyError(errmsg, code, response) elif code == 50: raise ExecutionTimeout(errmsg, code, response) elif code == 43: raise CursorNotFound(errmsg, code, response) msg = msg or "%s" raise OperationFailure(msg % errmsg, code, response)