def main():
args = _get_args()
print("Get editor's user id")
editor_client = Client(server_url=args.server,
auth=tuple(args.editor_auth.split(':')))
editor_id = editor_client.server_info()['user']['id']
print("Get reviewer's user id")
reviewer_client = Client(server_url=args.server,
auth=tuple(args.reviewer_auth.split(':')))
reviewer_id = reviewer_client.server_info()['user']['id']
print("Create signoff workflow groups")
admin_client = Client(server_url=args.server,
auth=tuple(args.auth.split(':')),
bucket=args.bucket)
print("Create/update editors group")
editors_group = admin_client.create_group(id='editors',
data={'members': []}, if_not_exists=True)
editors_group['data']['members'] = editors_group['data']['members'] + [editor_id]
admin_client.update_group(id='editors', data=editors_group['data'], safe=True)
print("Create/update reviewers group")
reviewers_group = admin_client.create_group(id='reviewers',
data={'members': []}, if_not_exists=True)
reviewers_group['data']['members'] = reviewers_group['data']['members'] + [reviewer_id]
admin_client.update_group(id='reviewers', data=reviewers_group['data'], safe=True)
def main():
args = _get_args()
print("Get editor's user id")
editor_client = Client(server_url=args.server,
auth=tuple(args.editor_auth.split(':')))
editor_id = editor_client.server_info()['user']['id']
print("Get reviewer's user id")
reviewer_client = Client(server_url=args.server,
auth=tuple(args.reviewer_auth.split(':')))
reviewer_id = reviewer_client.server_info()['user']['id']
print("Create signoff workflow groups")
admin_client = Client(server_url=args.server,
auth=tuple(args.auth.split(':')),
bucket=args.bucket)
print("Create/update editors group")
editors_group = admin_client.create_group('editors', data={'members': []}, if_not_exists=True)
editors_group['data']['members'] = editors_group['data']['members'] + [editor_id]
admin_client.update_group('editors', editors_group['data'], safe=True)
print("Create/update reviewers group")
reviewers_group = admin_client.create_group('reviewers', data={'members': []}, if_not_exists=True)
reviewers_group['data']['members'] = reviewers_group['data']['members'] + [reviewer_id]
admin_client.update_group('reviewers', data=reviewers_group['data'], safe=True)
def fetch_signed_resources(server_url, auth):
# List signed collection using capabilities.
client = Client(server_url=server_url,
auth=auth,
bucket="monitor",
collection="changes")
info = client.server_info()
try:
resources = info["capabilities"]["signer"]["resources"]
except KeyError:
raise ValueError(
"No signer capabilities found. Run on *writer* server!")
# Build the list of signed collections, source -> preview -> destination
# For most cases, configuration of signed resources is specified by bucket and
# does not contain any collection information.
resources_by_bid = {}
resources_by_cid = {}
preview_buckets = set()
for resource in resources:
if resource["source"]["collection"] is not None:
resources_by_cid[(
resource["destination"]["bucket"],
resource["destination"]["collection"],
)] = resource
else:
resources_by_bid[resource["destination"]["bucket"]] = resource
if "preview" in resource:
preview_buckets.add(resource["preview"]["bucket"])
print("Read collection list from {}".format(
client.get_endpoint("collection")))
resources = []
monitored = client.get_records(_sort="bucket,collection")
for entry in monitored:
bid = entry["bucket"]
cid = entry["collection"]
# Skip preview collections entries
if bid in preview_buckets:
continue
if (bid, cid) in resources_by_cid:
r = resources_by_cid[(bid, cid)]
elif bid in resources_by_bid:
r = copy.deepcopy(resources_by_bid[bid])
r["source"]["collection"] = r["destination"]["collection"] = cid
if "preview" in r:
r["preview"]["collection"] = cid
else:
raise ValueError(f"Unknown signed collection {bid}/{cid}")
resources.append(r)
return resources
def test_delete_request_removes_data(conf, env, fxa_account, fxa_urls,
fxa_client):
if env == 'prod':
pytest.skip('kintowe GDPR tests are not run in production')
auth = FxABearerTokenAuth(
fxa_account.email,
fxa_account.password,
scopes=['sync:addon_storage'],
client_id=DEFAULT_CLIENT_ID,
account_server_url=fxa_urls['authentication'],
oauth_server_url=fxa_urls['oauth'],
)
# Add some data to chrome.storage (kintowe)
we_client = Client(server_url=conf.get(env, 'we_server_url'), auth=auth)
we_existing_records = we_client.get_records(collection=conf.get(
env, 'qa_collection'),
bucket='default')
assert len(we_existing_records) == 0
data = {"payload": {"encrypted": "SmluZ28gdGVzdA=="}}
we_record = we_client.create_record(
data=data,
collection=conf.get(env, 'qa_collection'),
bucket='default',
permissions={"read": ["system.Everyone"]})
we_record_id = we_record['data']['id']
we_updated_records = we_client.get_records(collection=conf.get(
env, 'qa_collection'),
bucket='default')
assert len(we_updated_records) == len(we_existing_records) + 1
# Get the aliases of the bucket we are putting data in and
# make sure that an unauthenticated user can see these records
# before we delete the account
we_bucket_id = we_client.server_info()["user"]["bucket"]
anon_we_client = Client(server_url=conf.get(env, 'we_server_url'))
resp = anon_we_client.get_record(id=we_record_id,
bucket=we_bucket_id,
collection=conf.get(env, 'qa_collection'))
assert resp['data']['id'] == we_record_id
# Delete FxA account
fxa_client.destroy_account(fxa_account.email, fxa_account.password)
# Wait 1 minute and then make sure the records do not exist because the
# Kinto client will throw an exception for non-existent records
time.sleep(60)
with pytest.raises(KintoException):
resp = anon_we_client.get_record(id=we_record_id,
bucket=we_bucket_id,
collection=conf.get(
env, 'qa_collection'))
def test_delete_request_removes_data(conf, env, fxa_account, fxa_urls,
fxa_client):
if env == 'prod':
pytest.skip('testpilot GDPR tests are not run in production')
auth = FxABearerTokenAuth(
fxa_account.email,
fxa_account.password,
client_id=DEFAULT_CLIENT_ID,
scopes=["https://identity.mozilla.com/apps/notes"],
account_server_url=fxa_urls['authentication'],
oauth_server_url=fxa_urls['oauth'],
)
# Add some data to the Notes collection
tp_client = Client(server_url=conf.get(env, 'tp_server_url'), auth=auth)
tp_existing_records = tp_client.get_records(collection='notes',
bucket='default')
assert len(tp_existing_records) == 0
data = {"subject": "QA Test", "value": "This stuff should get deleted"}
tp_record = tp_client.create_record(
data=data,
collection='notes',
permissions={"read": ["system.Everyone"]})
tp_record_id = tp_record['data']['id']
tp_updated_records = tp_client.get_records(collection='notes',
bucket='default')
assert len(tp_updated_records) == len(tp_existing_records) + 1
# Get the aliases of the bucket we are putting data in and
# make sure that an unauthenticated user can see these records
# before we delete the account
tp_bucket_id = tp_client.server_info()["user"]["bucket"]
anon_tp_client = Client(server_url=conf.get(env, 'tp_server_url'))
resp = anon_tp_client.get_record(id=tp_record_id,
bucket=tp_bucket_id,
collection='notes')
assert resp['data']['id'] == tp_record_id
# Delete FxA account
fxa_client.destroy_account(fxa_account.email, fxa_account.password)
# Wait 5 minutes and then make sure the records do not exist because the
# Kinto client will throw an exception for non-existent records
time.sleep(120)
with pytest.raises(KintoException):
resp = anon_tp_client.get_record(id=tp_record_id,
bucket=tp_bucket_id,
collection='notes')
class ClientTest(unittest.TestCase):
def setUp(self):
self.session = mock.MagicMock()
self.client = Client(session=self.session)
mock_response(self.session)
def test_server_info(self):
self.client.server_info()
self.session.request.assert_called_with('get', '/')
def test_context_manager_works_as_expected(self):
settings = {"batch_max_requests": 25}
self.session.request.side_effect = [({"settings": settings}, []),
({"responses": []}, [])]
with self.client.batch(bucket='mozilla', collection='test') as batch:
batch.create_record(id=1234, data={'foo': 'bar'})
batch.create_record(id=5678, data={'bar': 'baz'})
self.session.request.assert_called_with(
method='POST',
endpoint='/batch',
payload={'requests': [
{'body': {'data': {'foo': 'bar'}},
'path': '/buckets/mozilla/collections/test/records/1234',
'method': 'PUT',
'headers': {'If-None-Match': '*'}},
{'body': {'data': {'bar': 'baz'}},
'path': '/buckets/mozilla/collections/test/records/5678',
'method': 'PUT',
'headers': {'If-None-Match': '*'}}]})
def test_batch_raises_exception(self):
# Make the next call to sess.request raise a 403.
exception = KintoException()
exception.response = mock.MagicMock()
exception.response.status_code = 403
exception.request = mock.sentinel.request
self.session.request.side_effect = exception
with self.assertRaises(KintoException):
with self.client.batch(bucket='moz', collection='test') as batch:
batch.create_record(id=1234, data={'foo': 'bar'})
def test_batch_raises_exception_if_subrequest_failed(self):
error = {
"errno": 121,
"message": "This user cannot access this resource.",
"code": 403,
"error": "Forbidden"
}
self.session.request.side_effect = [
({"settings": {"batch_max_requests": 25}}, []),
({"responses": [
{"status": 200, "path": "/url1", "body": {}, "headers": {}},
{"status": 404, "path": "/url2", "body": error, "headers": {}}
]}, [])]
with self.assertRaises(KintoException):
with self.client.batch(bucket='moz', collection='test') as batch:
batch.create_record(id=1234, data={'foo': 'bar'})
batch.create_record(id=5678, data={'tutu': 'toto'})
def test_batch_options_are_transmitted(self):
settings = {"batch_max_requests": 25}
self.session.request.side_effect = [({"settings": settings}, [])]
with mock.patch('kinto_http.create_session') as create_session:
with self.client.batch(bucket='moz', collection='test', retry=12,
retry_after=20):
_, last_call_kwargs = create_session.call_args_list[-1]
self.assertEqual(last_call_kwargs['retry'], 12)
self.assertEqual(last_call_kwargs['retry_after'], 20)
def test_client_is_represented_properly(self):
client = Client(
server_url="https://kinto.notmyidea.org/v1",
bucket="homebrewing",
collection="recipes"
)
expected_repr = ("<KintoClient https://kinto.notmyidea.org/v1/"
"buckets/homebrewing/collections/recipes>")
assert str(client) == expected_repr
def test_client_uses_default_bucket_if_not_specified(self):
mock_response(self.session)
client = Client(session=self.session)
client.get_bucket()
self.session.request.assert_called_with('get', '/buckets/default')
def test_client_uses_passed_bucket_if_specified(self):
client = Client(
server_url="https://kinto.notmyidea.org/v1",
bucket="buck")
assert client._bucket_name == "buck"
def test_client_clone_with_auth(self):
client_clone = self.client.clone(auth=("reviewer", ""))
assert client_clone.session.auth == ("reviewer", "")
assert self.client.session != client_clone.session
assert self.client.session.server_url == client_clone.session.server_url
assert self.client.session.auth != client_clone.session.auth
assert self.client.session.nb_retry == client_clone.session.nb_retry
assert self.client.session.retry_after == client_clone.session.retry_after
assert self.client._bucket_name == client_clone._bucket_name
assert self.client._collection_name == client_clone._collection_name
def test_client_clone_with_server_url(self):
client_clone = self.client.clone(server_url="https://kinto.notmyidea.org/v1")
assert client_clone.session.server_url == "https://kinto.notmyidea.org/v1"
assert self.client.session != client_clone.session
assert self.client.session.server_url != client_clone.session.server_url
assert self.client.session.auth == client_clone.session.auth
assert self.client.session.nb_retry == client_clone.session.nb_retry
assert self.client.session.retry_after == client_clone.session.retry_after
assert self.client._bucket_name == client_clone._bucket_name
assert self.client._collection_name == client_clone._collection_name
def test_client_clone_with_new_session(self):
session = create_session(auth=("reviewer", ""),
server_url="https://kinto.notmyidea.org/v1")
client_clone = self.client.clone(session=session)
assert client_clone.session == session
assert self.client.session != client_clone.session
assert self.client.session.server_url != client_clone.session.server_url
assert self.client.session.auth != client_clone.session.auth
assert self.client._bucket_name == client_clone._bucket_name
assert self.client._collection_name == client_clone._collection_name
def test_client_clone_with_auth_and_server_url(self):
client_clone = self.client.clone(auth=("reviewer", ""),
server_url="https://kinto.notmyidea.org/v1")
assert client_clone.session.auth == ("reviewer", "")
assert client_clone.session.server_url == "https://kinto.notmyidea.org/v1"
assert self.client.session != client_clone.session
assert self.client.session.server_url != client_clone.session.server_url
assert self.client.session.auth != client_clone.session.auth
assert self.client.session.nb_retry == client_clone.session.nb_retry
assert self.client.session.retry_after == client_clone.session.retry_after
assert self.client._bucket_name == client_clone._bucket_name
assert self.client._collection_name == client_clone._collection_name
def test_client_clone_with_existing_session(self):
client_clone = self.client.clone(session=self.client.session)
assert self.client.session == client_clone.session
assert self.client.session.server_url == client_clone.session.server_url
assert self.client.session.auth == client_clone.session.auth
assert self.client._bucket_name == client_clone._bucket_name
assert self.client._collection_name == client_clone._collection_name
def test_client_clone_with_new_bucket_and_collection(self):
client_clone = self.client.clone(bucket="bucket_blah", collection="coll_blah")
assert self.client.session == client_clone.session
assert self.client.session.server_url == client_clone.session.server_url
assert self.client.session.auth == client_clone.session.auth
assert self.client.session.nb_retry == client_clone.session.nb_retry
assert self.client.session.retry_after == client_clone.session.retry_after
assert self.client._bucket_name != client_clone._bucket_name
assert self.client._collection_name != client_clone._collection_name
assert client_clone._bucket_name == "bucket_blah"
assert client_clone._collection_name == "coll_blah"
def test_client_clone_with_auth_and_server_url_bucket_and_collection(self):
client_clone = self.client.clone(auth=("reviewer", ""),
server_url="https://kinto.notmyidea.org/v1",
bucket="bucket_blah",
collection="coll_blah")
assert self.client.session != client_clone.session
assert self.client.session.server_url != client_clone.session.server_url
assert self.client.session.auth != client_clone.session.auth
assert self.client._bucket_name != client_clone._bucket_name
assert self.client._collection_name != client_clone._collection_name
assert client_clone.session.auth == ("reviewer", "")
assert client_clone.session.server_url == "https://kinto.notmyidea.org/v1"
assert client_clone._bucket_name == "bucket_blah"
assert client_clone._collection_name == "coll_blah"
class ClientTest(unittest.TestCase):
def setUp(self):
self.session = mock.MagicMock()
self.client = Client(session=self.session)
mock_response(self.session)
def test_server_info(self):
self.client.server_info()
self.session.request.assert_called_with('get', '/')
def test_context_manager_works_as_expected(self):
settings = {"batch_max_requests": 25}
self.session.request.side_effect = [({"settings": settings}, []),
({"responses": []}, [])]
with self.client.batch(bucket='mozilla', collection='test') as batch:
batch.create_record(id=1234, data={'foo': 'bar'})
batch.create_record(id=5678, data={'bar': 'baz'})
batch.patch_record(id=5678, data={'bar': 'biz'})
changes = JSONPatch([{'op': 'add', 'location': 'foo', 'value': 'bar'}])
batch.patch_record(id=5678, changes=changes)
self.session.request.assert_called_with(
method='POST',
endpoint='/batch',
payload={'requests': [
{'body': {'data': {'foo': 'bar'}},
'path': '/buckets/mozilla/collections/test/records/1234',
'method': 'PUT',
'headers': {'If-None-Match': '*'}},
{'body': {'data': {'bar': 'baz'}},
'path': '/buckets/mozilla/collections/test/records/5678',
'method': 'PUT',
'headers': {'If-None-Match': '*'}},
{'body': {'data': {'bar': 'biz'}},
'path': '/buckets/mozilla/collections/test/records/5678',
'method': 'PATCH',
'headers': {'Content-Type': 'application/json'}},
{'body': [{'op': 'add', 'location': 'foo', 'value': 'bar'}],
'path': '/buckets/mozilla/collections/test/records/5678',
'method': 'PATCH',
'headers': {'Content-Type': 'application/json-patch+json'}}]})
def test_batch_raises_exception(self):
# Make the next call to sess.request raise a 403.
exception = KintoException()
exception.response = mock.MagicMock()
exception.response.status_code = 403
exception.request = mock.sentinel.request
self.session.request.side_effect = exception
with self.assertRaises(KintoException):
with self.client.batch(bucket='moz', collection='test') as batch:
batch.create_record(id=1234, data={'foo': 'bar'})
def test_batch_raises_exception_if_subrequest_failed_with_code_5xx(self):
error = {
"errno": 121,
"message": "This user cannot access this resource.",
"code": 500,
"error": "Server Internal Error"
}
self.session.request.side_effect = [
({"settings": {"batch_max_requests": 25}}, []),
({"responses": [
{"status": 200, "path": "/url1", "body": {}, "headers": {}},
{"status": 500, "path": "/url2", "body": error, "headers": {}}
]}, [])]
with self.assertRaises(KintoException):
with self.client.batch(bucket='moz', collection='test') as batch:
batch.create_record(id=1234, data={'foo': 'bar'})
batch.create_record(id=5678, data={'tutu': 'toto'})
def test_batch_raises_exception_if_subrequest_failed_with_code_4xx(self):
error_403 = {
"errno": 121,
"message": "Forbidden",
"code": 403,
"error": "This user cannot access this resource."
}
error_400 = {
"code": 400,
"errno": 104,
"error": "Invalid parameters",
"message": "Bad Request",
}
self.session.request.side_effect = [
({"settings": {"batch_max_requests": 25}}, []),
({"responses": [
{"status": 200, "path": "/url1", "body": {}, "headers": {}},
{"status": 403, "path": "/url2", "body": error_403, "headers": {}},
{"status": 200, "path": "/url1", "body": {}, "headers": {}},
{"status": 400, "path": "/url2", "body": error_400, "headers": {}},
]}, [])]
with self.assertRaises(KintoBatchException) as cm:
with self.client.batch(bucket='moz', collection='test') as batch:
batch.create_record(id=1234, data={'foo': 'bar'})
batch.create_record(id=1987, data={'maz': 'miz'})
batch.create_record(id=1982, data={'plop': 'plip'})
batch.create_record(id=5678, data={'tutu': 'toto'})
raised = cm.exception
assert "403" in str(raised)
assert "400" in str(raised)
assert isinstance(raised.exceptions[0], KintoException)
assert raised.exceptions[0].response.status_code == 403
assert raised.exceptions[1].response.status_code == 400
resp, headers = raised.results[0]
assert len(resp["responses"]) == 4
assert resp["responses"][0]["status"] == 200
def test_batch_does_not_raise_exception_if_batch_4xx_errors_are_ignored(self):
error = {
"errno": 121,
"message": "Forbidden",
"code": 403,
"error": "This user cannot access this resource."
}
self.session.request.side_effect = [
({"settings": {"batch_max_requests": 25}}, []),
({"responses": [
{"status": 200, "path": "/url1", "body": {}, "headers": {}},
{"status": 403, "path": "/url2", "body": error, "headers": {}}
]}, [])]
client = Client(session=self.session, ignore_batch_4xx=True)
with client.batch(bucket='moz', collection='test') as batch: # Do not raise
batch.create_record(id=1234, data={'foo': 'bar'})
batch.create_record(id=5678, data={'tutu': 'toto'})
def test_batch_options_are_transmitted(self):
settings = {"batch_max_requests": 25}
self.session.request.side_effect = [({"settings": settings}, [])]
with mock.patch('kinto_http.create_session') as create_session:
with self.client.batch(bucket='moz', collection='test', retry=12,
retry_after=20):
_, last_call_kwargs = create_session.call_args_list[-1]
self.assertEqual(last_call_kwargs['retry'], 12)
self.assertEqual(last_call_kwargs['retry_after'], 20)
def test_client_is_represented_properly_with_bucket_and_collection(self):
client = Client(
server_url="https://kinto.notmyidea.org/v1",
bucket="homebrewing",
collection="recipes"
)
expected_repr = ("<KintoClient https://kinto.notmyidea.org/v1/"
"buckets/homebrewing/collections/recipes>")
assert str(client) == expected_repr
def test_client_is_represented_properly_with_bucket(self):
client = Client(
server_url="https://kinto.notmyidea.org/v1",
bucket="homebrewing",
)
expected_repr = ("<KintoClient https://kinto.notmyidea.org/v1/"
"buckets/homebrewing>")
assert str(client) == expected_repr
def test_client_is_represented_properly_without_bucket(self):
client = Client(
server_url="https://kinto.notmyidea.org/v1",
bucket=None
)
expected_repr = ("<KintoClient https://kinto.notmyidea.org/v1/>")
assert str(client) == expected_repr
def test_client_uses_default_bucket_if_not_specified(self):
mock_response(self.session)
client = Client(session=self.session)
client.get_bucket()
self.session.request.assert_called_with('get', '/buckets/default')
def test_client_uses_passed_bucket_if_specified(self):
client = Client(
server_url="https://kinto.notmyidea.org/v1",
bucket="buck")
assert client._bucket_name == "buck"
def test_client_clone_with_auth(self):
client_clone = self.client.clone(auth=("reviewer", ""))
assert client_clone.session.auth == ("reviewer", "")
assert self.client.session != client_clone.session
assert self.client.session.server_url == client_clone.session.server_url
assert self.client.session.auth != client_clone.session.auth
assert self.client.session.nb_retry == client_clone.session.nb_retry
assert self.client.session.retry_after == client_clone.session.retry_after
assert self.client._bucket_name == client_clone._bucket_name
assert self.client._collection_name == client_clone._collection_name
def test_client_clone_with_server_url(self):
client_clone = self.client.clone(server_url="https://kinto.notmyidea.org/v1")
assert client_clone.session.server_url == "https://kinto.notmyidea.org/v1"
assert self.client.session != client_clone.session
assert self.client.session.server_url != client_clone.session.server_url
assert self.client.session.auth == client_clone.session.auth
assert self.client.session.nb_retry == client_clone.session.nb_retry
assert self.client.session.retry_after == client_clone.session.retry_after
assert self.client._bucket_name == client_clone._bucket_name
assert self.client._collection_name == client_clone._collection_name
def test_client_clone_with_new_session(self):
session = create_session(auth=("reviewer", ""),
server_url="https://kinto.notmyidea.org/v1")
client_clone = self.client.clone(session=session)
assert client_clone.session == session
assert self.client.session != client_clone.session
assert self.client.session.server_url != client_clone.session.server_url
assert self.client.session.auth != client_clone.session.auth
assert self.client._bucket_name == client_clone._bucket_name
assert self.client._collection_name == client_clone._collection_name
def test_client_clone_with_auth_and_server_url(self):
client_clone = self.client.clone(auth=("reviewer", ""),
server_url="https://kinto.notmyidea.org/v1")
assert client_clone.session.auth == ("reviewer", "")
assert client_clone.session.server_url == "https://kinto.notmyidea.org/v1"
assert self.client.session != client_clone.session
assert self.client.session.server_url != client_clone.session.server_url
assert self.client.session.auth != client_clone.session.auth
assert self.client.session.nb_retry == client_clone.session.nb_retry
assert self.client.session.retry_after == client_clone.session.retry_after
assert self.client._bucket_name == client_clone._bucket_name
assert self.client._collection_name == client_clone._collection_name
def test_client_clone_with_existing_session(self):
client_clone = self.client.clone(session=self.client.session)
assert self.client.session == client_clone.session
assert self.client.session.server_url == client_clone.session.server_url
assert self.client.session.auth == client_clone.session.auth
assert self.client._bucket_name == client_clone._bucket_name
assert self.client._collection_name == client_clone._collection_name
def test_client_clone_with_new_bucket_and_collection(self):
client_clone = self.client.clone(bucket="bucket_blah", collection="coll_blah")
assert self.client.session == client_clone.session
assert self.client.session.server_url == client_clone.session.server_url
assert self.client.session.auth == client_clone.session.auth
assert self.client.session.nb_retry == client_clone.session.nb_retry
assert self.client.session.retry_after == client_clone.session.retry_after
assert self.client._bucket_name != client_clone._bucket_name
assert self.client._collection_name != client_clone._collection_name
assert client_clone._bucket_name == "bucket_blah"
assert client_clone._collection_name == "coll_blah"
def test_client_clone_with_auth_and_server_url_bucket_and_collection(self):
client_clone = self.client.clone(auth=("reviewer", ""),
server_url="https://kinto.notmyidea.org/v1",
bucket="bucket_blah",
collection="coll_blah")
assert self.client.session != client_clone.session
assert self.client.session.server_url != client_clone.session.server_url
assert self.client.session.auth != client_clone.session.auth
assert self.client._bucket_name != client_clone._bucket_name
assert self.client._collection_name != client_clone._collection_name
assert client_clone.session.auth == ("reviewer", "")
assert client_clone.session.server_url == "https://kinto.notmyidea.org/v1"
assert client_clone._bucket_name == "bucket_blah"
assert client_clone._collection_name == "coll_blah"
def backport_records(event, context, **kwargs):
"""Backport records creations, updates and deletions from one collection to another.
"""
server_url = event["server"]
source_auth = (
event.get("backport_records_source_auth")
or os.environ["BACKPORT_RECORDS_SOURCE_AUTH"]
)
source_bucket = (
event.get("backport_records_source_bucket")
or os.environ["BACKPORT_RECORDS_SOURCE_BUCKET"]
)
source_collection = (
event.get("backport_records_source_collection")
or os.environ["BACKPORT_RECORDS_SOURCE_COLLECTION"]
)
dest_auth = event.get(
"backport_records_dest_auth",
os.getenv("BACKPORT_RECORDS_DEST_AUTH", source_auth),
)
dest_bucket = event.get(
"backport_records_dest_bucket",
os.getenv("BACKPORT_RECORDS_DEST_BUCKET", source_bucket),
)
dest_collection = event.get(
"backport_records_dest_collection",
os.getenv("BACKPORT_RECORDS_DEST_COLLECTION", source_collection),
)
if source_bucket == dest_bucket and source_collection == dest_collection:
raise ValueError("Cannot copy records: destination is identical to source")
source_client = Client(
server_url=server_url,
bucket=source_bucket,
collection=source_collection,
auth=tuple(source_auth.split(":", 1))
if ":" in source_auth
else BearerTokenAuth(source_auth),
)
dest_client = Client(
server_url=server_url,
bucket=dest_bucket,
collection=dest_collection,
auth=tuple(dest_auth.split(":", 1))
if ":" in dest_auth
else BearerTokenAuth(dest_auth),
)
source_timestamp = source_client.get_records_timestamp()
dest_timestamp = dest_client.get_records_timestamp()
if source_timestamp <= dest_timestamp:
print("Records are in sync. Nothing to do.")
return
source_records = source_client.get_records()
dest_records_by_id = {r["id"]: r for r in dest_client.get_records()}
with dest_client.batch() as dest_batch:
# Create or update the destination records.
for r in source_records:
dest_record = dest_records_by_id.pop(r["id"], None)
if dest_record is None:
dest_batch.create_record(data=r)
elif r["last_modified"] > dest_record["last_modified"]:
dest_batch.update_record(data=r)
# Delete the records missing from source.
for r in dest_records_by_id.values():
dest_batch.delete_record(id=r["id"])
ops_count = len(dest_batch.results())
# If destination has signing, request review or auto-approve changes.
server_info = dest_client.server_info()
signer_config = server_info["capabilities"].get("signer", {})
signer_resources = signer_config.get("resources", [])
# Check destination collection config (sign-off required etc.)
signed_dest = [
r
for r in signer_resources
if r["source"]["bucket"] == dest_bucket
and r["source"]["collection"] == dest_collection
]
if len(signed_dest) == 0:
# Not explicitly configured. Check if configured at bucket level?
signed_dest = [
r
for r in signer_resources
if r["source"]["bucket"] == dest_bucket
and r["source"]["collection"] is None
]
# Destination has no signature enabled. Nothing to do.
if len(signed_dest) == 0:
print(f"Done. {ops_count} changes applied.")
return
has_autoapproval = not signed_dest[0].get(
"to_review_enabled", signer_config["to_review_enabled"]
) and not signed_dest[0].get(
"group_check_enabled", signer_config["group_check_enabled"]
)
if has_autoapproval:
# Approve the changes.
dest_client.patch_collection(data={"status": "to-sign"})
print(f"Done. {ops_count} changes applied and signed.")
else:
# Request review.
dest_client.patch_collection(data={"status": "to-review"})
print(f"Done. Requested review for {ops_count} changes.")
class ClientTest(unittest.TestCase):
def setUp(self):
self.session = mock.MagicMock()
self.client = Client(session=self.session)
mock_response(self.session)
def test_server_info(self):
self.client.server_info()
self.session.request.assert_called_with('get', '/')
def test_context_manager_works_as_expected(self):
settings = {"batch_max_requests": 25}
self.session.request.side_effect = [({
"settings": settings
}, []), ({
"responses": []
}, [])]
with self.client.batch(bucket='mozilla', collection='test') as batch:
batch.create_record(id=1234, data={'foo': 'bar'})
batch.create_record(id=5678, data={'bar': 'baz'})
self.session.request.assert_called_with(
method='POST',
endpoint='/batch',
payload={
'requests': [{
'body': {
'data': {
'foo': 'bar'
}
},
'path': '/buckets/mozilla/collections/test/records/1234',
'method': 'PUT',
'headers': {
'If-None-Match': '*',
'User-Agent': USER_AGENT
}
}, {
'body': {
'data': {
'bar': 'baz'
}
},
'path': '/buckets/mozilla/collections/test/records/5678',
'method': 'PUT',
'headers': {
'If-None-Match': '*',
'User-Agent': USER_AGENT
}
}]
})
def test_batch_raises_exception(self):
# Make the next call to sess.request raise a 403.
exception = KintoException()
exception.response = mock.MagicMock()
exception.response.status_code = 403
exception.request = mock.sentinel.request
self.session.request.side_effect = exception
with self.assertRaises(KintoException):
with self.client.batch(bucket='moz', collection='test') as batch:
batch.create_record(id=1234, data={'foo': 'bar'})
def test_batch_raises_exception_if_subrequest_failed_with_code_5xx(self):
error = {
"errno": 121,
"message": "This user cannot access this resource.",
"code": 500,
"error": "Server Internal Error"
}
self.session.request.side_effect = [({
"settings": {
"batch_max_requests": 25
}
}, []),
({
"responses": [{
"status": 200,
"path": "/url1",
"body": {},
"headers": {}
}, {
"status": 500,
"path": "/url2",
"body": error,
"headers": {}
}]
}, [])]
with self.assertRaises(KintoException):
with self.client.batch(bucket='moz', collection='test') as batch:
batch.create_record(id=1234, data={'foo': 'bar'})
batch.create_record(id=5678, data={'tutu': 'toto'})
def test_batch_dont_raise_exception_if_subrequest_failed_with_code_4xx(
self):
error = {
"errno": 121,
"message": "Forbidden",
"code": 403,
"error": "This user cannot access this resource."
}
self.session.request.side_effect = [({
"settings": {
"batch_max_requests": 25
}
}, []),
({
"responses": [{
"status": 200,
"path": "/url1",
"body": {},
"headers": {}
}, {
"status": 403,
"path": "/url2",
"body": error,
"headers": {}
}]
}, [])]
with self.client.batch(bucket='moz',
collection='test') as batch: # Do not raise
batch.create_record(id=1234, data={'foo': 'bar'})
batch.create_record(id=5678, data={'tutu': 'toto'})
def test_batch_options_are_transmitted(self):
settings = {"batch_max_requests": 25}
self.session.request.side_effect = [({"settings": settings}, [])]
with mock.patch('kinto_http.create_session') as create_session:
with self.client.batch(bucket='moz',
collection='test',
retry=12,
retry_after=20):
_, last_call_kwargs = create_session.call_args_list[-1]
self.assertEqual(last_call_kwargs['retry'], 12)
self.assertEqual(last_call_kwargs['retry_after'], 20)
def test_client_is_represented_properly_with_bucket_and_collection(self):
client = Client(server_url="https://kinto.notmyidea.org/v1",
bucket="homebrewing",
collection="recipes")
expected_repr = ("<KintoClient https://kinto.notmyidea.org/v1/"
"buckets/homebrewing/collections/recipes>")
assert str(client) == expected_repr
def test_client_is_represented_properly_with_bucket(self):
client = Client(
server_url="https://kinto.notmyidea.org/v1",
bucket="homebrewing",
)
expected_repr = ("<KintoClient https://kinto.notmyidea.org/v1/"
"buckets/homebrewing>")
assert str(client) == expected_repr
def test_client_is_represented_properly_without_bucket(self):
client = Client(server_url="https://kinto.notmyidea.org/v1",
bucket=None)
expected_repr = ("<KintoClient https://kinto.notmyidea.org/v1/>")
assert str(client) == expected_repr
def test_client_uses_default_bucket_if_not_specified(self):
mock_response(self.session)
client = Client(session=self.session)
client.get_bucket()
self.session.request.assert_called_with('get', '/buckets/default')
def test_client_uses_passed_bucket_if_specified(self):
client = Client(server_url="https://kinto.notmyidea.org/v1",
bucket="buck")
assert client._bucket_name == "buck"
def test_client_clone_with_auth(self):
client_clone = self.client.clone(auth=("reviewer", ""))
assert client_clone.session.auth == ("reviewer", "")
assert self.client.session != client_clone.session
assert self.client.session.server_url == client_clone.session.server_url
assert self.client.session.auth != client_clone.session.auth
assert self.client.session.nb_retry == client_clone.session.nb_retry
assert self.client.session.retry_after == client_clone.session.retry_after
assert self.client._bucket_name == client_clone._bucket_name
assert self.client._collection_name == client_clone._collection_name
def test_client_clone_with_server_url(self):
client_clone = self.client.clone(
server_url="https://kinto.notmyidea.org/v1")
assert client_clone.session.server_url == "https://kinto.notmyidea.org/v1"
assert self.client.session != client_clone.session
assert self.client.session.server_url != client_clone.session.server_url
assert self.client.session.auth == client_clone.session.auth
assert self.client.session.nb_retry == client_clone.session.nb_retry
assert self.client.session.retry_after == client_clone.session.retry_after
assert self.client._bucket_name == client_clone._bucket_name
assert self.client._collection_name == client_clone._collection_name
def test_client_clone_with_new_session(self):
session = create_session(auth=("reviewer", ""),
server_url="https://kinto.notmyidea.org/v1")
client_clone = self.client.clone(session=session)
assert client_clone.session == session
assert self.client.session != client_clone.session
assert self.client.session.server_url != client_clone.session.server_url
assert self.client.session.auth != client_clone.session.auth
assert self.client._bucket_name == client_clone._bucket_name
assert self.client._collection_name == client_clone._collection_name
def test_client_clone_with_auth_and_server_url(self):
client_clone = self.client.clone(
auth=("reviewer", ""), server_url="https://kinto.notmyidea.org/v1")
assert client_clone.session.auth == ("reviewer", "")
assert client_clone.session.server_url == "https://kinto.notmyidea.org/v1"
assert self.client.session != client_clone.session
assert self.client.session.server_url != client_clone.session.server_url
assert self.client.session.auth != client_clone.session.auth
assert self.client.session.nb_retry == client_clone.session.nb_retry
assert self.client.session.retry_after == client_clone.session.retry_after
assert self.client._bucket_name == client_clone._bucket_name
assert self.client._collection_name == client_clone._collection_name
def test_client_clone_with_existing_session(self):
client_clone = self.client.clone(session=self.client.session)
assert self.client.session == client_clone.session
assert self.client.session.server_url == client_clone.session.server_url
assert self.client.session.auth == client_clone.session.auth
assert self.client._bucket_name == client_clone._bucket_name
assert self.client._collection_name == client_clone._collection_name
def test_client_clone_with_new_bucket_and_collection(self):
client_clone = self.client.clone(bucket="bucket_blah",
collection="coll_blah")
assert self.client.session == client_clone.session
assert self.client.session.server_url == client_clone.session.server_url
assert self.client.session.auth == client_clone.session.auth
assert self.client.session.nb_retry == client_clone.session.nb_retry
assert self.client.session.retry_after == client_clone.session.retry_after
assert self.client._bucket_name != client_clone._bucket_name
assert self.client._collection_name != client_clone._collection_name
assert client_clone._bucket_name == "bucket_blah"
assert client_clone._collection_name == "coll_blah"
def test_client_clone_with_auth_and_server_url_bucket_and_collection(self):
client_clone = self.client.clone(
auth=("reviewer", ""),
server_url="https://kinto.notmyidea.org/v1",
bucket="bucket_blah",
collection="coll_blah")
assert self.client.session != client_clone.session
assert self.client.session.server_url != client_clone.session.server_url
assert self.client.session.auth != client_clone.session.auth
assert self.client._bucket_name != client_clone._bucket_name
assert self.client._collection_name != client_clone._collection_name
assert client_clone.session.auth == ("reviewer", "")
assert client_clone.session.server_url == "https://kinto.notmyidea.org/v1"
assert client_clone._bucket_name == "bucket_blah"
assert client_clone._collection_name == "coll_blah"
def main():
args = _get_args()
client = Client(server_url=args.server,
auth=tuple(args.auth.split(':')),
bucket=args.source_bucket,
collection=args.source_col)
if args.editor_auth is None:
args.editor_auth = args.auth
if args.reviewer_auth is None:
args.reviewer_auth = args.auth
editor_client = Client(server_url=args.server,
auth=tuple(args.editor_auth.split(':')),
bucket=args.source_bucket,
collection=args.source_col)
reviewer_client = Client(server_url=args.server,
auth=tuple(args.reviewer_auth.split(':')),
bucket=args.source_bucket,
collection=args.source_col)
# 0. initialize source bucket/collection (if necessary)
server_info = client.server_info()
editor_id = editor_client.server_info()['user']['id']
reviewer_id = reviewer_client.server_info()['user']['id']
print('Server: {0}'.format(args.server))
print('Author: {user[id]}'.format(**server_info))
print('Editor: {0}'.format(editor_id))
print('Reviewer: {0}'.format(reviewer_id))
# 0. check that this collection is well configured.
signer_capabilities = server_info['capabilities']['signer']
to_review_enabled = signer_capabilities.get('to_review_enabled', False)
group_check_enabled = signer_capabilities.get('group_check_enabled', False)
resources = [
r for r in signer_capabilities['resources']
if (args.source_bucket, args.source_col) == (r['source']['bucket'],
r['source']['collection'])
]
assert len(resources) > 0, 'Specified source not configured to be signed'
resource = resources[0]
if to_review_enabled and 'preview' in resource:
print(
'Signoff: {source[bucket]}/{source[collection]} => {preview[bucket]}/{preview[collection]} => {destination[bucket]}/{destination[collection]}'
.format(**resource))
else:
print(
'Signoff: {source[bucket]}/{source[collection]} => {destination[bucket]}/{destination[collection]}'
.format(**resource))
print('Group check: {0}'.format(group_check_enabled))
print('Review workflow: {0}'.format(to_review_enabled))
print('_' * 80)
bucket = client.create_bucket(if_not_exists=True)
client.patch_bucket(permissions={
'write': [editor_id, reviewer_id] + bucket['permissions']['write']
},
if_match=bucket['data']['last_modified'],
safe=True)
client.create_collection(if_not_exists=True)
if args.reset:
client.delete_records()
existing = 0
else:
existing_records = client.get_records()
existing = len(existing_records)
if group_check_enabled:
editors_group = signer_capabilities['editors_group']
client.create_group(editors_group,
data={'members': [editor_id]},
if_not_exists=True)
reviewers_group = signer_capabilities['reviewers_group']
client.create_group(reviewers_group,
data={'members': [reviewer_id]},
if_not_exists=True)
dest_client = Client(server_url=args.server,
bucket=resource['destination']['bucket'],
collection=resource['destination']['collection'])
preview_client = None
if to_review_enabled and 'preview' in resource:
preview_bucket = resource['preview']['bucket']
preview_collection = resource['preview']['collection']
preview_client = Client(server_url=args.server,
bucket=preview_bucket,
collection=preview_collection)
# 1. upload data
print('Author uploads 20 random records')
records = upload_records(client, 20)
# 2. ask for a signature
# 2.1 ask for review (noop on old versions)
print('Editor asks for review')
data = {"status": "to-review"}
editor_client.patch_collection(data=data)
# 2.2 check the preview collection (if enabled)
if preview_client:
print('Check preview collection')
preview_records = preview_client.get_records()
expected = existing + 20
assert len(preview_records) == expected, '%s != %s records' % (
len(preview_records), expected)
metadata = preview_client.get_collection()['data']
preview_signature = metadata.get('signature')
assert preview_signature, 'Preview collection not signed'
preview_timestamp = collection_timestamp(preview_client)
# 2.3 approve the review
print('Reviewer approves and triggers signature')
data = {"status": "to-sign"}
reviewer_client.patch_collection(data=data)
# 3. upload more data
print('Author creates 20 others records')
upload_records(client, 20)
print('Editor updates 5 random records')
for toupdate in random.sample(records, 5):
editor_client.patch_record(dict(newkey=_rand(10), **toupdate))
print('Author deletes 5 random records')
for todelete in random.sample(records, 5):
client.delete_record(todelete['id'])
expected = existing + 20 + 20 - 5
# 4. ask again for a signature
# 2.1 ask for review (noop on old versions)
print('Editor asks for review')
data = {"status": "to-review"}
editor_client.patch_collection(data=data)
# 2.2 check the preview collection (if enabled)
if preview_client:
print('Check preview collection')
preview_records = preview_client.get_records()
assert len(preview_records) == expected, '%s != %s records' % (
len(preview_records), expected)
# Diff size is 20 + 5 if updated records are also all deleted,
# or 30 if deletions and updates apply to different records.
diff_since_last = preview_client.get_records(_since=preview_timestamp)
assert 25 <= len(
diff_since_last
) <= 30, 'Changes since last signature are not consistent'
metadata = preview_client.get_collection()['data']
assert preview_signature != metadata[
'signature'], 'Preview collection not updated'
# 2.3 approve the review
print('Reviewer approves and triggers signature')
data = {"status": "to-sign"}
reviewer_client.patch_collection(data=data)
# 5. wait for the result
# 6. obtain the destination records and serialize canonically.
records = list(dest_client.get_records())
assert len(records) == expected, '%s != %s records' % (len(records),
expected)
timestamp = collection_timestamp(dest_client)
serialized = canonical_json(records, timestamp)
print('Hash is %r' % compute_hash(serialized))
# 7. get back the signed hash
dest_col = dest_client.get_collection()
signature = dest_col['data']['signature']
with open('pub', 'w') as f:
f.write(signature['public_key'])
# 8. verify the signature matches the hash
signer = ECDSASigner(public_key='pub')
try:
signer.verify(serialized, signature)
print('Signature OK')
except Exception:
print('Signature KO')
raise
def main():
args = _get_args()
client = Client(server_url=args.server, auth=tuple(args.auth.split(':')),
bucket=args.source_bucket,
collection=args.source_col)
if args.editor_auth is None:
args.editor_auth = args.auth
if args.reviewer_auth is None:
args.reviewer_auth = args.auth
editor_client = Client(server_url=args.server,
auth=tuple(args.editor_auth.split(':')),
bucket=args.source_bucket,
collection=args.source_col)
reviewer_client = Client(server_url=args.server,
auth=tuple(args.reviewer_auth.split(':')),
bucket=args.source_bucket,
collection=args.source_col)
# 0. initialize source bucket/collection (if necessary)
server_info = client.server_info()
editor_id = editor_client.server_info()['user']['id']
reviewer_id = reviewer_client.server_info()['user']['id']
print('Server: {0}'.format(args.server))
print('Author: {user[id]}'.format(**server_info))
print('Editor: {0}'.format(editor_id))
print('Reviewer: {0}'.format(reviewer_id))
# 0. check that this collection is well configured.
signer_capabilities = server_info['capabilities']['signer']
resources = [r for r in signer_capabilities['resources']
if (args.source_bucket, args.source_col) == (r['source']['bucket'], r['source']['collection']) or
(args.source_bucket, None) == (r['source']['bucket'], r['source']['collection'])]
assert len(resources) > 0, 'Specified source not configured to be signed'
resource = resources[0]
if 'preview' in resource:
print('Signoff: {source[bucket]}/{source[collection]} => {preview[bucket]}/{preview[collection]} => {destination[bucket]}/{destination[collection]}'.format(**resource))
else:
print('Signoff: {source[bucket]}/{source[collection]} => {destination[bucket]}/{destination[collection]}'.format(**resource))
print('_' * 80)
bucket = client.create_bucket(if_not_exists=True)
client.create_collection(permissions={'write': [editor_id, reviewer_id] + bucket['permissions']['write']}, if_not_exists=True)
editors_group = resource.get('editors_group') or signer_capabilities['editors_group']
editors_group = editors_group.format(collection_id=args.source_col)
client.patch_group(id=editors_group, data={'members': [editor_id]})
reviewers_group = resource.get('reviewers_group') or signer_capabilities['reviewers_group']
reviewers_group = reviewers_group.format(collection_id=args.source_col)
client.patch_group(id=reviewers_group, data={'members': [reviewer_id]})
if args.reset:
client.delete_records()
existing = 0
else:
existing_records = client.get_records()
existing = len(existing_records)
dest_col = resource['destination'].get('collection') or args.source_col
dest_client = Client(server_url=args.server,
bucket=resource['destination']['bucket'],
collection=dest_col)
preview_client = None
if 'preview' in resource:
preview_bucket = resource['preview']['bucket']
preview_collection = resource['preview'].get('collection') or args.source_col
preview_client = Client(server_url=args.server,
bucket=preview_bucket,
collection=preview_collection)
# 1. upload data
print('Author uploads 20 random records')
records = upload_records(client, 20)
# 2. ask for a signature
# 2.1 ask for review (noop on old versions)
print('Editor asks for review')
data = {"status": "to-review"}
editor_client.patch_collection(data=data)
# 2.2 check the preview collection (if enabled)
if preview_client:
print('Check preview collection')
preview_records = preview_client.get_records()
expected = existing + 20
assert len(preview_records) == expected, '%s != %s records' % (len(preview_records), expected)
metadata = preview_client.get_collection()['data']
preview_signature = metadata.get('signature')
assert preview_signature, 'Preview collection not signed'
preview_timestamp = preview_client.get_records_timestamp()
# 2.3 approve the review
print('Reviewer approves and triggers signature')
data = {"status": "to-sign"}
reviewer_client.patch_collection(data=data)
# 3. upload more data
print('Author creates 20 others records')
upload_records(client, 20)
print('Editor updates 5 random records')
for toupdate in random.sample(records, 5):
editor_client.patch_record(data=dict(newkey=_rand(10), **toupdate))
print('Author deletes 5 random records')
for todelete in random.sample(records, 5):
client.delete_record(id=todelete['id'])
expected = existing + 20 + 20 - 5
# 4. ask again for a signature
# 2.1 ask for review (noop on old versions)
print('Editor asks for review')
data = {"status": "to-review"}
editor_client.patch_collection(data=data)
# 2.2 check the preview collection (if enabled)
if preview_client:
print('Check preview collection')
preview_records = preview_client.get_records()
assert len(preview_records) == expected, '%s != %s records' % (len(preview_records), expected)
# Diff size is 20 + 5 if updated records are also all deleted,
# or 30 if deletions and updates apply to different records.
diff_since_last = preview_client.get_records(_since=preview_timestamp)
assert 25 <= len(diff_since_last) <= 30, 'Changes since last signature are not consistent'
metadata = preview_client.get_collection()['data']
assert preview_signature != metadata['signature'], 'Preview collection not updated'
# 2.3 approve the review
print('Reviewer approves and triggers signature')
data = {"status": "to-sign"}
reviewer_client.patch_collection(data=data)
# 5. wait for the result
# 6. obtain the destination records and serialize canonically.
records = list(dest_client.get_records())
assert len(records) == expected, '%s != %s records' % (len(records), expected)
timestamp = dest_client.get_records_timestamp()
serialized = canonical_json(records, timestamp)
print('Hash is %r' % compute_hash(serialized))
# 7. get back the signed hash
signature = dest_client.get_collection()['data']['signature']
with open('pub', 'w') as f:
f.write(signature['public_key'])
# 8. verify the signature matches the hash
signer = ECDSASigner(public_key='pub')
try:
signer.verify(serialized, signature)
print('Signature OK')
except Exception:
print('Signature KO')
raise
