def new_deployment(username):
user = db.User.find_one({'username': username})
if user is None or (user is not None and not current_user.is_admin()
and current_user != user):
# No permission
flash("Permission denied", 'danger')
return redirect(url_for("index"))
form = NewDeploymentForm()
if form.validate_on_submit():
deployment_date = dateparse(form.deployment_date.data)
delayed_mode = form.delayed_mode.data
deployment_name = form.glider_name.data + '-' + \
deployment_date.strftime('%Y%m%dT%H%M')
if delayed_mode:
deployment_name += '-delayed'
deployment = db.Deployment()
# form.populate_obj(deployment)
deployment.user_id = user._id
deployment.username = username
deployment.deployment_dir = os.path.join(username, deployment_name)
deployment.updated = datetime.utcnow()
deployment.deployment_date = deployment_date
deployment.glider_name = form.glider_name.data
deployment.name = deployment_name
deployment.attribution = form.attribution.data
deployment.delayed_mode = delayed_mode
try:
existing_deployment = db.Deployment.find_one(
{'name': deployment_name})
if existing_deployment is not None:
raise DuplicateKeyError("Duplicate Key Detected: name")
existing_deployment = db.Deployment.find_one({
'glider_name':
form.glider_name.data,
'deployment_date':
deployment_date
})
if existing_deployment is not None:
raise DuplicateKeyError(
"Duplicate Key Detected: glider_name and deployment_date")
deployment.save()
flash("Deployment created", 'success')
send_registration_email(username, deployment)
except DuplicateKeyError:
flash(
"Deployment names must be unique across Glider DAC: %s already used"
% deployment.name, 'danger')
else:
error_str = ", ".join(
["%s: %s" % (k, ", ".join(v)) for k, v in form.errors.iteritems()])
flash("Deployment could not be created: %s" % error_str, 'danger')
return redirect(url_for('list_user_deployments', username=username))
async def test_update_by_id_pymongo_raises_duplicate_key_error_failure_on_retry(
app, event):
with patch.object(app["db"].event.collection, "find_one_and_replace", new=CoroutineMock())\
as mock_replace:
mock_replace.side_effect = [
DuplicateKeyError("error after first run"),
DuplicateKeyError("error after second run")
]
with pytest.raises(DuplicateKeyError, match="error after second run"):
await app["db"].event.update_by_id(event.id, event)
def add_user(name, email, hashedpw):
"""
Given a name, email and password, inserts a document with those credentials
to the `users` collection.
"""
"""
Ticket: Durable Writes
Please increase the durability of this method by using a non-default write
concern with ``insert_one``.
"""
try:
# TODO: User Management
# Insert a user with the "name", "email", and "password" fields.
user = {"name": name, "email": email, "password": hashedpw}
if get_user(email):
raise DuplicateKeyError('error')
# TODO: Durable Writes
# Use a more durable Write Concern for this operation.
w = WriteConcern(w="majority")
while (not db.users.with_options(write_concern=w)):
pass
db.users.insert_one(user)
return {"success": True}
except DuplicateKeyError:
return {"error": "A user with the given email already exists."}
def test_add_status_already_exists(status, statuses):
'''
Test duplication detection for add_status
'''
with patch('user_status.sql'):
statuses.table.insert_one.side_effect = DuplicateKeyError("ERR_MSG")
assert not statuses.add_status(*status)
def update_approval_status(data):
if not deploymentUnitApprovalStatusDB.GetDeploymentUnitApprovalStatusById(data.get("_id").get("oid")):
raise Exception("The status exists with the object id : "+ data.get("_id").get("oid"))
try:
return deploymentUnitApprovalStatusDB.UpdateDeploymentUnitApprovalStatus(data)
except DuplicateKeyError as e:
raise DuplicateKeyError("The status exists with same name")
def put(self, data, _id, content_type, filename, **kw):
"""upload file-only. Can not handle bookmarks."""
if _id in ['thumb', 'items', 'login']:
raise DuplicateKeyError
item_type, subtype = content_type.split('/', 1)
if item_type in ['image', 'text', 'audio', 'video']:
pass
elif item_type == 'application' and \
filter(lambda k: subtype.find(k) > -1, ['compress', 'zip', 'tar']):
item_type = 'archive'
else:
item_type = 'unknown'
if self.mdb.find_one({'short_id': kw['short_id']}):
raise DuplicateKeyError('short_id already exists')
_id = self.gfs.put(data,
_id=_id,
content_type=content_type,
filename=filename)
kw.update({'_id': _id, 'item_type': item_type})
self.mdb.insert(kw)
return _id
def test_add(self, mock_parser, mock_doc_init):
bill = "bill"
with self.subTest("when invalid schema"):
mock_parser.side_effect = ValueError
with self.assertRaises(ValueError):
self.repository.add(bill)
mock_parser.side_effect = None
mock_parser.return_value = "parsed"
with self.subTest("when has duplicate keys "):
self.mock_collection.insert_many.side_effect = DuplicateKeyError(
Exception())
with self.assertRaises(errors.UnprocessableDataError):
self.repository.add(bill)
self.mock_collection.insert_many.side_effect = None
with self.subTest("when inserted with success"):
actual = self.repository.add(bill)
mock_parser.assert_called_with(bill, [schema.PHONE_BILL])
self.assertIsInstance(actual, Documents)
def test_add_user_failure(mock_collection, normal_user):
mock_collection.insert_one.side_effect = DuplicateKeyError('repeated username')
with pytest.raises(HTTPException) as excinfo:
add_user(normal_user)
assert excinfo.value.status_code == 400
def test_duplicate_exc(self, get_collection):
"""
in case of lock retry later after lock timeout
"""
get_collection.return_value.insert_one.side_effect = DuplicateKeyError(
"E11000 duplicate key error collection")
test_method_procedure = Mock()
@concurrency_lock
def test_method(*args, **kwargs):
test_method_procedure(*args, **kwargs)
dummy_task.retry = Mock(side_effect=Retry)
with patch("celery_worker.locks.datetime") as datetime_mock:
datetime_mock.utcnow.return_value = datetime.utcnow()
with self.assertRaises(Retry):
test_method(dummy_task) # run
task_uid = args_to_uid(
(test_method.__module__, test_method.__name__, tuple(), dict()))
get_collection.return_value.insert_one.assert_called_once_with({
"_id":
task_uid,
"expireAt":
datetime_mock.utcnow.return_value + timedelta(seconds=10)
})
dummy_task.retry.assert_called_once_with(max_retries=20, countdown=70)
test_method_procedure.assert_not_called()
async def test_update_by_id_pymongo_raises_duplicate_key_error_successful_on_retry(
app, event):
with patch.object(app["db"].event.collection, "find_one_and_replace", new=CoroutineMock())\
as mock_replace:
mock_replace.side_effect = [DuplicateKeyError("error_msg"), 1]
await app["db"].event.update_by_id(event.id, event)
def _check_command_response(response,
max_wire_version,
msg=None,
allowable_errors=None,
parse_write_concern_error=False):
"""Check the response to a command for errors.
"""
if "ok" not in response:
# Server didn't recognize our message as a command.
raise OperationFailure(response.get("$err"), response.get("code"),
response, max_wire_version)
if parse_write_concern_error and 'writeConcernError' in response:
_raise_write_concern_error(response['writeConcernError'])
if not response["ok"]:
details = response
# Mongos returns the error details in a 'raw' object
# for some errors.
if "raw" in response:
for shard in itervalues(response["raw"]):
# Grab the first non-empty raw error from a shard.
if shard.get("errmsg") and not shard.get("ok"):
details = shard
break
errmsg = details["errmsg"]
if (allowable_errors is None
or (errmsg not in allowable_errors
and details.get("code") not in allowable_errors)):
code = details.get("code")
# Server is "not master" or "recovering"
if code in _NOT_MASTER_CODES:
raise NotMasterError(errmsg, response)
elif ("not master" in errmsg or "node is recovering" in errmsg):
raise NotMasterError(errmsg, response)
# Server assertion failures
if errmsg == "db assertion failure":
errmsg = ("db assertion failure, assertion: '%s'" %
details.get("assertion", ""))
raise OperationFailure(errmsg, details.get("assertionCode"),
response, max_wire_version)
# Other errors
# findAndModify with upsert can raise duplicate key error
if code in (11000, 11001, 12582):
raise DuplicateKeyError(errmsg, code, response,
max_wire_version)
elif code == 50:
raise ExecutionTimeout(errmsg, code, response,
max_wire_version)
elif code == 43:
raise CursorNotFound(errmsg, code, response, max_wire_version)
msg = msg or "%s"
raise OperationFailure(msg % errmsg, code, response,
max_wire_version)
def _raise_last_write_error(write_errors):
# If the last batch had multiple errors only report
# the last error to emulate continue_on_error.
error = write_errors[-1]
if error.get("code") == 11000:
raise DuplicateKeyError(error.get("errmsg"), 11000, error)
raise WriteError(error.get("errmsg"), error.get("code"), error)
def test_append_insert_version_duplicatekey():
read_handler = Mock(append=Mock(__name__=""))
previous_version = TPL_VERSION.copy()
previous_version['version'] = 1
vs = create_autospec(
VersionStore,
instance=True,
_collection=Mock(),
_version_nums=Mock(find_one_and_update=Mock(
return_value={'version': previous_version['version'] + 1})),
_versions=Mock(insert_one=Mock(__name__="insert_one"),
find_one=Mock(__name__="find_one",
return_value=previous_version)),
_arctic_lib=create_autospec(ArcticLibraryBinding,
arctic=create_autospec(
Arctic, mongo_host='some_host')),
_publish_changes=False)
vs._insert_version = lambda version: VersionStore._insert_version(
vs, version)
vs._versions.insert_one.side_effect = [
DuplicateKeyError("dup key error"), None
]
vs._collection.database.connection.nodes = []
vs._read_handler.return_value = read_handler
VersionStore.append(vs,
'sym', [1, 2, 3],
prune_previous_version=False,
upsert=False)
assert vs._version_nums.find_one_and_update.call_count == 2
assert vs._versions.find_one.call_count == 2
assert read_handler.append.call_count == 2
assert vs._versions.insert_one.call_count == 2
assert vs._publish_change.call_count == 2
def _check_write_command_response(results):
"""Backward compatibility helper for write command error handling.
"""
errors = [
res for res in results
if "writeErrors" in res[1] or "writeConcernError" in res[1]
]
if errors:
# If multiple batches had errors
# raise from the last batch.
offset, result = errors[-1]
# Prefer write errors over write concern errors
write_errors = result.get("writeErrors")
if write_errors:
# If the last batch had multiple errors only report
# the last error to emulate continue_on_error.
error = write_errors[-1]
error["index"] += offset
if error.get("code") == 11000:
raise DuplicateKeyError(error.get("errmsg"), 11000, error)
else:
error = result["writeConcernError"]
if "errInfo" in error and error["errInfo"].get('wtimeout'):
# Make sure we raise WTimeoutError
raise WTimeoutError(error.get("errmsg"), error.get("code"),
error)
raise OperationFailure(error.get("errmsg"), error.get("code"), error)
def test_add_user_already_exists(user, accounts):
'''
Test that add_user responds to errors correctly
'''
with patch('users.sql'):
accounts.table.insert_one.side_effect = DuplicateKeyError("ERR_MSG")
assert not accounts.add_user(*user)
def test_write_insert_version_duplicatekey():
write_handler = Mock(write=Mock(__name__=""))
vs = create_autospec(VersionStore,
instance=True,
_collection=Mock(),
_version_nums=Mock(find_one_and_update=Mock(
return_value={'version': 1})),
_versions=Mock(insert_one=Mock(__name__="insert_one"),
find_one=Mock(__name__="find_one")),
_arctic_lib=create_autospec(
ArcticLibraryBinding,
arctic=create_autospec(Arctic,
mongo_host='some_host')),
_publish_changes=False)
vs._insert_version = lambda version: VersionStore._insert_version(
vs, version)
vs._versions.insert_one.side_effect = [
DuplicateKeyError("dup key error"), None
]
vs._collection.database.connection.nodes = []
vs._write_handler.return_value = write_handler
VersionStore.write(vs, 'sym', sentinel.data, prune_previous_version=False)
assert vs._version_nums.find_one_and_update.call_count == 2
assert vs._versions.find_one.call_count == 2
assert write_handler.write.call_count == 2
assert vs._publish_change.call_count == 2
assert vs._versions.insert_one.call_count == 2
def _check_gle_response(result):
"""Return getlasterror response as a dict, or raise OperationFailure."""
# Did getlasterror itself fail?
_check_command_response(result)
if result.get("wtimeout", False):
# MongoDB versions before 1.8.0 return the error message in an "errmsg"
# field. If "errmsg" exists "err" will also exist set to None, so we
# have to check for "errmsg" first.
raise WTimeoutError(result.get("errmsg", result.get("err")),
result.get("code"),
result)
error_msg = result.get("err", "")
if error_msg is None:
return result
if error_msg.startswith("not master"):
raise NotMasterError(error_msg, result)
details = result
# mongos returns the error code in an error object for some errors.
if "errObjects" in result:
for errobj in result["errObjects"]:
if errobj.get("err") == error_msg:
details = errobj
break
code = details.get("code")
if code in (11000, 11001, 12582):
raise DuplicateKeyError(details["err"], code, result)
raise OperationFailure(details["err"], code, result)
def _check_command_response(response,
max_wire_version,
allowable_errors=None,
parse_write_concern_error=False):
"""Check the response to a command for errors.
"""
if "ok" not in response:
# Server didn't recognize our message as a command.
raise OperationFailure(response.get("$err"), response.get("code"),
response, max_wire_version)
if parse_write_concern_error and 'writeConcernError' in response:
_error = response["writeConcernError"]
_labels = response.get("errorLabels")
if _labels:
_error.update({'errorLabels': _labels})
_raise_write_concern_error(_error)
if response["ok"]:
return
details = response
# Mongos returns the error details in a 'raw' object
# for some errors.
if "raw" in response:
for shard in response["raw"].values():
# Grab the first non-empty raw error from a shard.
if shard.get("errmsg") and not shard.get("ok"):
details = shard
break
errmsg = details["errmsg"]
code = details.get("code")
# For allowable errors, only check for error messages when the code is not
# included.
if allowable_errors:
if code is not None:
if code in allowable_errors:
return
elif errmsg in allowable_errors:
return
# Server is "not primary" or "recovering"
if code is not None:
if code in _NOT_MASTER_CODES:
raise NotPrimaryError(errmsg, response)
elif "not master" in errmsg or "node is recovering" in errmsg:
raise NotPrimaryError(errmsg, response)
# Other errors
# findAndModify with upsert can raise duplicate key error
if code in (11000, 11001, 12582):
raise DuplicateKeyError(errmsg, code, response, max_wire_version)
elif code == 50:
raise ExecutionTimeout(errmsg, code, response, max_wire_version)
elif code == 43:
raise CursorNotFound(errmsg, code, response, max_wire_version)
raise OperationFailure(errmsg, code, response, max_wire_version)
def __check_response_to_last_error(self, response):
"""Check a response to a lastError message for errors.
`response` is a byte string representing a response to the message.
If it represents an error response we raise OperationFailure.
Return the response as a document.
"""
response = helpers._unpack_response(response)
assert response["number_returned"] == 1
error = response["data"][0]
# TODO unify logic with database.error method
if error.get("err", 0) is None:
return error
if error["err"] == "not master":
self._reset()
if "code" in error and error["code"] in [11000, 11001]:
raise DuplicateKeyError(error["err"])
else:
raise OperationFailure(error["err"])
return error
def test_save_unit_new_unit_race_condition(self, mock_associate, mock_add,
mock_update, mock_get,
mock_path):
"""
This simulates a case where the same unit gets added by another workflow
before the save completes. In that case, the failover behavior is to
update the unit instead of adding it.
"""
# Setup
unit = self.mixin.init_unit('t', {'k': 'v'}, {'m': 'm1'}, '/bar')
mock_add.side_effect = DuplicateKeyError('dups!')
# raise an exception the first time around, then simulate the unit
# having appeared since the last call.
mock_get.side_effect = _create_mock_side_effect(
[MissingResource, {
'_id': 'existing'
}])
# Test
saved = self.mixin.save_unit(unit)
# Verify
self.assertEqual(2, mock_get.call_count)
self.assertEqual(1, mock_update.call_count)
self.assertEqual(1, mock_add.call_count)
self.assertEqual(1, mock_associate.call_count)
self.assertEqual(0, self.mixin._added_count)
self.assertEqual(1, self.mixin._updated_count)
self.assertEqual(saved.id, 'existing')
def __update(self, spec, updates, upsert=False, multi=False):
bson_safe(spec)
bson_safe(updates)
result = dict(
connectionId=None,
updatedExisting=False,
err=None,
ok=1.0,
n=0,
nModified=0
)
for doc, mspec in self._find(spec):
self._deindex(doc)
mspec.update(updates)
self._index(doc)
result['n'] += 1
result['nModified'] += 1
if not multi: break
if result['n']:
result['updatedExisting'] = True
return result
if upsert:
doc = dict(spec)
MatchDoc(doc).update(updates, upserted=upsert)
_id = doc.get('_id', ())
if _id == ():
_id = doc['_id'] = bson.ObjectId()
if _id in self._data:
raise DuplicateKeyError('duplicate ID on upsert')
self._index(doc)
self._data[_id] = bcopy(doc)
result['upserted'] = _id
return result
else:
return result
def test_add_institution_failure(collection_mock, bank):
collection_mock.insert_one.side_effect = DuplicateKeyError(
'Bank already exists')
with pytest.raises(HTTPException) as excinfo:
add_institution(bank)
assert excinfo.value.status_code == 400
def __check_response_to_last_error(self, response):
"""Check a response to a lastError message for errors.
`response` is a byte string representing a response to the message.
If it represents an error response we raise OperationFailure.
Return the response as a document.
"""
response = helpers._unpack_response(response)
assert response["number_returned"] == 1
error = response["data"][0]
helpers._check_command_response(error, self.disconnect)
error_msg = error.get("err", "")
if error_msg is None:
return error
if error_msg.startswith("not master"):
self.disconnect()
raise AutoReconnect(error_msg)
if "code" in error:
if error["code"] in [11000, 11001, 12582]:
raise DuplicateKeyError(error["err"])
else:
raise OperationFailure(error["err"], error["code"])
else:
raise OperationFailure(error["err"])
def insert_one(doc):
"""
Pretend to add the document to the database.
Throw an error if the document is already there, otherwise
return a class within a "inserted_id".
"""
results = []
for datum in data:
match = True
counter = 0
for key in doc:
if key not in datum or key == "date" or key == "data":
continue
counter += 1
if doc[key] != datum[key]:
match = False
if match and counter:
results.append(doc)
if results:
raise DuplicateKeyError("bla")
else:
class Result:
def __init__(self):
self.inserted_id = ObjectId("58dbe045ef677d54224a01d2")
return Result()
def test_add_currency_failure(collection_mock, institutions_mock, currency_in):
collection_mock.insert_one.side_effect = DuplicateKeyError(
'currency already exists')
with pytest.raises(HTTPException) as excinfo:
add_instrument(currency_in)
assert excinfo.value.status_code == 400
assert not institutions_mock.find_one.called
def _insert(self, data):
if not '_id' in data:
data['_id'] = ObjectId()
object_id = data['_id']
if object_id in self._documents:
raise DuplicateKeyError("Duplicate Key Error", 11000)
self._documents[object_id] = copy.deepcopy(data)
return object_id
def test_write_metadata_insert_version_dupkeyerror():
vs = _create_mock_versionstore()
vs._versions.insert_one.__name__ = 'insert_one'
vs._versions.insert_one.side_effect = [DuplicateKeyError('dup key error'), None]
VersionStore.write_metadata(vs, symbol=TEST_SYMBOL, metadata=META_TO_WRITE)
assert vs._version_nums.find_one_and_update.call_count == 2
assert vs._versions.insert_one.call_count == 2
assert vs._publish_change.call_count == 1
def test_add_account_cash_duplicate(mock_collection, mock_institutions, account_cash_in, normal_user):
mock_collection.insert_one.side_effect = DuplicateKeyError('account already exists')
with pytest.raises(HTTPException) as excinfo:
add_account(account_cash_in, normal_user)
assert excinfo.value.status_code == 400
assert not mock_institutions.find_one.called
def before_insert(self, instance, state, session):
doc_window_hash = hashlib.sha1(
str(instance.document_id) + str(instance.window_size)).hexdigest()
if instance.__class__.query.find({
'doc_window_hash': doc_window_hash
}).count() > 0:
raise DuplicateKeyError('Duplicate hash found ', doc_window_hash)
instance.doc_window_hash = doc_window_hash
def _check_command_response(response, msg=None, allowable_errors=None):
"""Check the response to a command for errors.
"""
if "ok" not in response:
# Server didn't recognize our message as a command.
raise OperationFailure(response.get("$err"),
response.get("code"),
response)
# TODO: remove, this is moving to _check_gle_response
if response.get("wtimeout", False):
# MongoDB versions before 1.8.0 return the error message in an "errmsg"
# field. If "errmsg" exists "err" will also exist set to None, so we
# have to check for "errmsg" first.
raise WTimeoutError(response.get("errmsg", response.get("err")),
response.get("code"),
response)
if not response["ok"]:
details = response
# Mongos returns the error details in a 'raw' object
# for some errors.
if "raw" in response:
for shard in itervalues(response["raw"]):
# Grab the first non-empty raw error from a shard.
if shard.get("errmsg") and not shard.get("ok"):
details = shard
break
errmsg = details["errmsg"]
if allowable_errors is None or errmsg not in allowable_errors:
# Server is "not master" or "recovering"
if (errmsg.startswith("not master")
or errmsg.startswith("node is recovering")):
raise NotMasterError(errmsg, response)
# Server assertion failures
if errmsg == "db assertion failure":
errmsg = ("db assertion failure, assertion: '%s'" %
details.get("assertion", ""))
raise OperationFailure(errmsg,
details.get("assertionCode"),
response)
# Other errors
code = details.get("code")
# findAndModify with upsert can raise duplicate key error
if code in (11000, 11001, 12582):
raise DuplicateKeyError(errmsg, code, response)
elif code == 50:
raise ExecutionTimeout(errmsg, code, response)
elif code == 43:
raise CursorNotFound(errmsg, code, response)
msg = msg or "%s"
raise OperationFailure(msg % errmsg, code, response)