def main(): args = _get_args() print("Get editor's user id") editor_client = Client(server_url=args.server, auth=tuple(args.editor_auth.split(':'))) editor_id = editor_client.server_info()['user']['id'] print("Get reviewer's user id") reviewer_client = Client(server_url=args.server, auth=tuple(args.reviewer_auth.split(':'))) reviewer_id = reviewer_client.server_info()['user']['id'] print("Create signoff workflow groups") admin_client = Client(server_url=args.server, auth=tuple(args.auth.split(':')), bucket=args.bucket) print("Create/update editors group") editors_group = admin_client.create_group(id='editors', data={'members': []}, if_not_exists=True) editors_group['data']['members'] = editors_group['data']['members'] + [editor_id] admin_client.update_group(id='editors', data=editors_group['data'], safe=True) print("Create/update reviewers group") reviewers_group = admin_client.create_group(id='reviewers', data={'members': []}, if_not_exists=True) reviewers_group['data']['members'] = reviewers_group['data']['members'] + [reviewer_id] admin_client.update_group(id='reviewers', data=reviewers_group['data'], safe=True)
def main(): args = _get_args() print("Get editor's user id") editor_client = Client(server_url=args.server, auth=tuple(args.editor_auth.split(':'))) editor_id = editor_client.server_info()['user']['id'] print("Get reviewer's user id") reviewer_client = Client(server_url=args.server, auth=tuple(args.reviewer_auth.split(':'))) reviewer_id = reviewer_client.server_info()['user']['id'] print("Create signoff workflow groups") admin_client = Client(server_url=args.server, auth=tuple(args.auth.split(':')), bucket=args.bucket) print("Create/update editors group") editors_group = admin_client.create_group('editors', data={'members': []}, if_not_exists=True) editors_group['data']['members'] = editors_group['data']['members'] + [editor_id] admin_client.update_group('editors', editors_group['data'], safe=True) print("Create/update reviewers group") reviewers_group = admin_client.create_group('reviewers', data={'members': []}, if_not_exists=True) reviewers_group['data']['members'] = reviewers_group['data']['members'] + [reviewer_id] admin_client.update_group('reviewers', data=reviewers_group['data'], safe=True)
def fetch_signed_resources(server_url, auth): # List signed collection using capabilities. client = Client(server_url=server_url, auth=auth, bucket="monitor", collection="changes") info = client.server_info() try: resources = info["capabilities"]["signer"]["resources"] except KeyError: raise ValueError( "No signer capabilities found. Run on *writer* server!") # Build the list of signed collections, source -> preview -> destination # For most cases, configuration of signed resources is specified by bucket and # does not contain any collection information. resources_by_bid = {} resources_by_cid = {} preview_buckets = set() for resource in resources: if resource["source"]["collection"] is not None: resources_by_cid[( resource["destination"]["bucket"], resource["destination"]["collection"], )] = resource else: resources_by_bid[resource["destination"]["bucket"]] = resource if "preview" in resource: preview_buckets.add(resource["preview"]["bucket"]) print("Read collection list from {}".format( client.get_endpoint("collection"))) resources = [] monitored = client.get_records(_sort="bucket,collection") for entry in monitored: bid = entry["bucket"] cid = entry["collection"] # Skip preview collections entries if bid in preview_buckets: continue if (bid, cid) in resources_by_cid: r = resources_by_cid[(bid, cid)] elif bid in resources_by_bid: r = copy.deepcopy(resources_by_bid[bid]) r["source"]["collection"] = r["destination"]["collection"] = cid if "preview" in r: r["preview"]["collection"] = cid else: raise ValueError(f"Unknown signed collection {bid}/{cid}") resources.append(r) return resources
def test_delete_request_removes_data(conf, env, fxa_account, fxa_urls, fxa_client): if env == 'prod': pytest.skip('kintowe GDPR tests are not run in production') auth = FxABearerTokenAuth( fxa_account.email, fxa_account.password, scopes=['sync:addon_storage'], client_id=DEFAULT_CLIENT_ID, account_server_url=fxa_urls['authentication'], oauth_server_url=fxa_urls['oauth'], ) # Add some data to chrome.storage (kintowe) we_client = Client(server_url=conf.get(env, 'we_server_url'), auth=auth) we_existing_records = we_client.get_records(collection=conf.get( env, 'qa_collection'), bucket='default') assert len(we_existing_records) == 0 data = {"payload": {"encrypted": "SmluZ28gdGVzdA=="}} we_record = we_client.create_record( data=data, collection=conf.get(env, 'qa_collection'), bucket='default', permissions={"read": ["system.Everyone"]}) we_record_id = we_record['data']['id'] we_updated_records = we_client.get_records(collection=conf.get( env, 'qa_collection'), bucket='default') assert len(we_updated_records) == len(we_existing_records) + 1 # Get the aliases of the bucket we are putting data in and # make sure that an unauthenticated user can see these records # before we delete the account we_bucket_id = we_client.server_info()["user"]["bucket"] anon_we_client = Client(server_url=conf.get(env, 'we_server_url')) resp = anon_we_client.get_record(id=we_record_id, bucket=we_bucket_id, collection=conf.get(env, 'qa_collection')) assert resp['data']['id'] == we_record_id # Delete FxA account fxa_client.destroy_account(fxa_account.email, fxa_account.password) # Wait 1 minute and then make sure the records do not exist because the # Kinto client will throw an exception for non-existent records time.sleep(60) with pytest.raises(KintoException): resp = anon_we_client.get_record(id=we_record_id, bucket=we_bucket_id, collection=conf.get( env, 'qa_collection'))
def test_delete_request_removes_data(conf, env, fxa_account, fxa_urls, fxa_client): if env == 'prod': pytest.skip('testpilot GDPR tests are not run in production') auth = FxABearerTokenAuth( fxa_account.email, fxa_account.password, client_id=DEFAULT_CLIENT_ID, scopes=["https://identity.mozilla.com/apps/notes"], account_server_url=fxa_urls['authentication'], oauth_server_url=fxa_urls['oauth'], ) # Add some data to the Notes collection tp_client = Client(server_url=conf.get(env, 'tp_server_url'), auth=auth) tp_existing_records = tp_client.get_records(collection='notes', bucket='default') assert len(tp_existing_records) == 0 data = {"subject": "QA Test", "value": "This stuff should get deleted"} tp_record = tp_client.create_record( data=data, collection='notes', permissions={"read": ["system.Everyone"]}) tp_record_id = tp_record['data']['id'] tp_updated_records = tp_client.get_records(collection='notes', bucket='default') assert len(tp_updated_records) == len(tp_existing_records) + 1 # Get the aliases of the bucket we are putting data in and # make sure that an unauthenticated user can see these records # before we delete the account tp_bucket_id = tp_client.server_info()["user"]["bucket"] anon_tp_client = Client(server_url=conf.get(env, 'tp_server_url')) resp = anon_tp_client.get_record(id=tp_record_id, bucket=tp_bucket_id, collection='notes') assert resp['data']['id'] == tp_record_id # Delete FxA account fxa_client.destroy_account(fxa_account.email, fxa_account.password) # Wait 5 minutes and then make sure the records do not exist because the # Kinto client will throw an exception for non-existent records time.sleep(120) with pytest.raises(KintoException): resp = anon_tp_client.get_record(id=tp_record_id, bucket=tp_bucket_id, collection='notes')
class ClientTest(unittest.TestCase): def setUp(self): self.session = mock.MagicMock() self.client = Client(session=self.session) mock_response(self.session) def test_server_info(self): self.client.server_info() self.session.request.assert_called_with('get', '/') def test_context_manager_works_as_expected(self): settings = {"batch_max_requests": 25} self.session.request.side_effect = [({"settings": settings}, []), ({"responses": []}, [])] with self.client.batch(bucket='mozilla', collection='test') as batch: batch.create_record(id=1234, data={'foo': 'bar'}) batch.create_record(id=5678, data={'bar': 'baz'}) self.session.request.assert_called_with( method='POST', endpoint='/batch', payload={'requests': [ {'body': {'data': {'foo': 'bar'}}, 'path': '/buckets/mozilla/collections/test/records/1234', 'method': 'PUT', 'headers': {'If-None-Match': '*'}}, {'body': {'data': {'bar': 'baz'}}, 'path': '/buckets/mozilla/collections/test/records/5678', 'method': 'PUT', 'headers': {'If-None-Match': '*'}}]}) def test_batch_raises_exception(self): # Make the next call to sess.request raise a 403. exception = KintoException() exception.response = mock.MagicMock() exception.response.status_code = 403 exception.request = mock.sentinel.request self.session.request.side_effect = exception with self.assertRaises(KintoException): with self.client.batch(bucket='moz', collection='test') as batch: batch.create_record(id=1234, data={'foo': 'bar'}) def test_batch_raises_exception_if_subrequest_failed(self): error = { "errno": 121, "message": "This user cannot access this resource.", "code": 403, "error": "Forbidden" } self.session.request.side_effect = [ ({"settings": {"batch_max_requests": 25}}, []), ({"responses": [ {"status": 200, "path": "/url1", "body": {}, "headers": {}}, {"status": 404, "path": "/url2", "body": error, "headers": {}} ]}, [])] with self.assertRaises(KintoException): with self.client.batch(bucket='moz', collection='test') as batch: batch.create_record(id=1234, data={'foo': 'bar'}) batch.create_record(id=5678, data={'tutu': 'toto'}) def test_batch_options_are_transmitted(self): settings = {"batch_max_requests": 25} self.session.request.side_effect = [({"settings": settings}, [])] with mock.patch('kinto_http.create_session') as create_session: with self.client.batch(bucket='moz', collection='test', retry=12, retry_after=20): _, last_call_kwargs = create_session.call_args_list[-1] self.assertEqual(last_call_kwargs['retry'], 12) self.assertEqual(last_call_kwargs['retry_after'], 20) def test_client_is_represented_properly(self): client = Client( server_url="https://kinto.notmyidea.org/v1", bucket="homebrewing", collection="recipes" ) expected_repr = ("<KintoClient https://kinto.notmyidea.org/v1/" "buckets/homebrewing/collections/recipes>") assert str(client) == expected_repr def test_client_uses_default_bucket_if_not_specified(self): mock_response(self.session) client = Client(session=self.session) client.get_bucket() self.session.request.assert_called_with('get', '/buckets/default') def test_client_uses_passed_bucket_if_specified(self): client = Client( server_url="https://kinto.notmyidea.org/v1", bucket="buck") assert client._bucket_name == "buck" def test_client_clone_with_auth(self): client_clone = self.client.clone(auth=("reviewer", "")) assert client_clone.session.auth == ("reviewer", "") assert self.client.session != client_clone.session assert self.client.session.server_url == client_clone.session.server_url assert self.client.session.auth != client_clone.session.auth assert self.client.session.nb_retry == client_clone.session.nb_retry assert self.client.session.retry_after == client_clone.session.retry_after assert self.client._bucket_name == client_clone._bucket_name assert self.client._collection_name == client_clone._collection_name def test_client_clone_with_server_url(self): client_clone = self.client.clone(server_url="https://kinto.notmyidea.org/v1") assert client_clone.session.server_url == "https://kinto.notmyidea.org/v1" assert self.client.session != client_clone.session assert self.client.session.server_url != client_clone.session.server_url assert self.client.session.auth == client_clone.session.auth assert self.client.session.nb_retry == client_clone.session.nb_retry assert self.client.session.retry_after == client_clone.session.retry_after assert self.client._bucket_name == client_clone._bucket_name assert self.client._collection_name == client_clone._collection_name def test_client_clone_with_new_session(self): session = create_session(auth=("reviewer", ""), server_url="https://kinto.notmyidea.org/v1") client_clone = self.client.clone(session=session) assert client_clone.session == session assert self.client.session != client_clone.session assert self.client.session.server_url != client_clone.session.server_url assert self.client.session.auth != client_clone.session.auth assert self.client._bucket_name == client_clone._bucket_name assert self.client._collection_name == client_clone._collection_name def test_client_clone_with_auth_and_server_url(self): client_clone = self.client.clone(auth=("reviewer", ""), server_url="https://kinto.notmyidea.org/v1") assert client_clone.session.auth == ("reviewer", "") assert client_clone.session.server_url == "https://kinto.notmyidea.org/v1" assert self.client.session != client_clone.session assert self.client.session.server_url != client_clone.session.server_url assert self.client.session.auth != client_clone.session.auth assert self.client.session.nb_retry == client_clone.session.nb_retry assert self.client.session.retry_after == client_clone.session.retry_after assert self.client._bucket_name == client_clone._bucket_name assert self.client._collection_name == client_clone._collection_name def test_client_clone_with_existing_session(self): client_clone = self.client.clone(session=self.client.session) assert self.client.session == client_clone.session assert self.client.session.server_url == client_clone.session.server_url assert self.client.session.auth == client_clone.session.auth assert self.client._bucket_name == client_clone._bucket_name assert self.client._collection_name == client_clone._collection_name def test_client_clone_with_new_bucket_and_collection(self): client_clone = self.client.clone(bucket="bucket_blah", collection="coll_blah") assert self.client.session == client_clone.session assert self.client.session.server_url == client_clone.session.server_url assert self.client.session.auth == client_clone.session.auth assert self.client.session.nb_retry == client_clone.session.nb_retry assert self.client.session.retry_after == client_clone.session.retry_after assert self.client._bucket_name != client_clone._bucket_name assert self.client._collection_name != client_clone._collection_name assert client_clone._bucket_name == "bucket_blah" assert client_clone._collection_name == "coll_blah" def test_client_clone_with_auth_and_server_url_bucket_and_collection(self): client_clone = self.client.clone(auth=("reviewer", ""), server_url="https://kinto.notmyidea.org/v1", bucket="bucket_blah", collection="coll_blah") assert self.client.session != client_clone.session assert self.client.session.server_url != client_clone.session.server_url assert self.client.session.auth != client_clone.session.auth assert self.client._bucket_name != client_clone._bucket_name assert self.client._collection_name != client_clone._collection_name assert client_clone.session.auth == ("reviewer", "") assert client_clone.session.server_url == "https://kinto.notmyidea.org/v1" assert client_clone._bucket_name == "bucket_blah" assert client_clone._collection_name == "coll_blah"
class ClientTest(unittest.TestCase): def setUp(self): self.session = mock.MagicMock() self.client = Client(session=self.session) mock_response(self.session) def test_server_info(self): self.client.server_info() self.session.request.assert_called_with('get', '/') def test_context_manager_works_as_expected(self): settings = {"batch_max_requests": 25} self.session.request.side_effect = [({"settings": settings}, []), ({"responses": []}, [])] with self.client.batch(bucket='mozilla', collection='test') as batch: batch.create_record(id=1234, data={'foo': 'bar'}) batch.create_record(id=5678, data={'bar': 'baz'}) batch.patch_record(id=5678, data={'bar': 'biz'}) changes = JSONPatch([{'op': 'add', 'location': 'foo', 'value': 'bar'}]) batch.patch_record(id=5678, changes=changes) self.session.request.assert_called_with( method='POST', endpoint='/batch', payload={'requests': [ {'body': {'data': {'foo': 'bar'}}, 'path': '/buckets/mozilla/collections/test/records/1234', 'method': 'PUT', 'headers': {'If-None-Match': '*'}}, {'body': {'data': {'bar': 'baz'}}, 'path': '/buckets/mozilla/collections/test/records/5678', 'method': 'PUT', 'headers': {'If-None-Match': '*'}}, {'body': {'data': {'bar': 'biz'}}, 'path': '/buckets/mozilla/collections/test/records/5678', 'method': 'PATCH', 'headers': {'Content-Type': 'application/json'}}, {'body': [{'op': 'add', 'location': 'foo', 'value': 'bar'}], 'path': '/buckets/mozilla/collections/test/records/5678', 'method': 'PATCH', 'headers': {'Content-Type': 'application/json-patch+json'}}]}) def test_batch_raises_exception(self): # Make the next call to sess.request raise a 403. exception = KintoException() exception.response = mock.MagicMock() exception.response.status_code = 403 exception.request = mock.sentinel.request self.session.request.side_effect = exception with self.assertRaises(KintoException): with self.client.batch(bucket='moz', collection='test') as batch: batch.create_record(id=1234, data={'foo': 'bar'}) def test_batch_raises_exception_if_subrequest_failed_with_code_5xx(self): error = { "errno": 121, "message": "This user cannot access this resource.", "code": 500, "error": "Server Internal Error" } self.session.request.side_effect = [ ({"settings": {"batch_max_requests": 25}}, []), ({"responses": [ {"status": 200, "path": "/url1", "body": {}, "headers": {}}, {"status": 500, "path": "/url2", "body": error, "headers": {}} ]}, [])] with self.assertRaises(KintoException): with self.client.batch(bucket='moz', collection='test') as batch: batch.create_record(id=1234, data={'foo': 'bar'}) batch.create_record(id=5678, data={'tutu': 'toto'}) def test_batch_raises_exception_if_subrequest_failed_with_code_4xx(self): error_403 = { "errno": 121, "message": "Forbidden", "code": 403, "error": "This user cannot access this resource." } error_400 = { "code": 400, "errno": 104, "error": "Invalid parameters", "message": "Bad Request", } self.session.request.side_effect = [ ({"settings": {"batch_max_requests": 25}}, []), ({"responses": [ {"status": 200, "path": "/url1", "body": {}, "headers": {}}, {"status": 403, "path": "/url2", "body": error_403, "headers": {}}, {"status": 200, "path": "/url1", "body": {}, "headers": {}}, {"status": 400, "path": "/url2", "body": error_400, "headers": {}}, ]}, [])] with self.assertRaises(KintoBatchException) as cm: with self.client.batch(bucket='moz', collection='test') as batch: batch.create_record(id=1234, data={'foo': 'bar'}) batch.create_record(id=1987, data={'maz': 'miz'}) batch.create_record(id=1982, data={'plop': 'plip'}) batch.create_record(id=5678, data={'tutu': 'toto'}) raised = cm.exception assert "403" in str(raised) assert "400" in str(raised) assert isinstance(raised.exceptions[0], KintoException) assert raised.exceptions[0].response.status_code == 403 assert raised.exceptions[1].response.status_code == 400 resp, headers = raised.results[0] assert len(resp["responses"]) == 4 assert resp["responses"][0]["status"] == 200 def test_batch_does_not_raise_exception_if_batch_4xx_errors_are_ignored(self): error = { "errno": 121, "message": "Forbidden", "code": 403, "error": "This user cannot access this resource." } self.session.request.side_effect = [ ({"settings": {"batch_max_requests": 25}}, []), ({"responses": [ {"status": 200, "path": "/url1", "body": {}, "headers": {}}, {"status": 403, "path": "/url2", "body": error, "headers": {}} ]}, [])] client = Client(session=self.session, ignore_batch_4xx=True) with client.batch(bucket='moz', collection='test') as batch: # Do not raise batch.create_record(id=1234, data={'foo': 'bar'}) batch.create_record(id=5678, data={'tutu': 'toto'}) def test_batch_options_are_transmitted(self): settings = {"batch_max_requests": 25} self.session.request.side_effect = [({"settings": settings}, [])] with mock.patch('kinto_http.create_session') as create_session: with self.client.batch(bucket='moz', collection='test', retry=12, retry_after=20): _, last_call_kwargs = create_session.call_args_list[-1] self.assertEqual(last_call_kwargs['retry'], 12) self.assertEqual(last_call_kwargs['retry_after'], 20) def test_client_is_represented_properly_with_bucket_and_collection(self): client = Client( server_url="https://kinto.notmyidea.org/v1", bucket="homebrewing", collection="recipes" ) expected_repr = ("<KintoClient https://kinto.notmyidea.org/v1/" "buckets/homebrewing/collections/recipes>") assert str(client) == expected_repr def test_client_is_represented_properly_with_bucket(self): client = Client( server_url="https://kinto.notmyidea.org/v1", bucket="homebrewing", ) expected_repr = ("<KintoClient https://kinto.notmyidea.org/v1/" "buckets/homebrewing>") assert str(client) == expected_repr def test_client_is_represented_properly_without_bucket(self): client = Client( server_url="https://kinto.notmyidea.org/v1", bucket=None ) expected_repr = ("<KintoClient https://kinto.notmyidea.org/v1/>") assert str(client) == expected_repr def test_client_uses_default_bucket_if_not_specified(self): mock_response(self.session) client = Client(session=self.session) client.get_bucket() self.session.request.assert_called_with('get', '/buckets/default') def test_client_uses_passed_bucket_if_specified(self): client = Client( server_url="https://kinto.notmyidea.org/v1", bucket="buck") assert client._bucket_name == "buck" def test_client_clone_with_auth(self): client_clone = self.client.clone(auth=("reviewer", "")) assert client_clone.session.auth == ("reviewer", "") assert self.client.session != client_clone.session assert self.client.session.server_url == client_clone.session.server_url assert self.client.session.auth != client_clone.session.auth assert self.client.session.nb_retry == client_clone.session.nb_retry assert self.client.session.retry_after == client_clone.session.retry_after assert self.client._bucket_name == client_clone._bucket_name assert self.client._collection_name == client_clone._collection_name def test_client_clone_with_server_url(self): client_clone = self.client.clone(server_url="https://kinto.notmyidea.org/v1") assert client_clone.session.server_url == "https://kinto.notmyidea.org/v1" assert self.client.session != client_clone.session assert self.client.session.server_url != client_clone.session.server_url assert self.client.session.auth == client_clone.session.auth assert self.client.session.nb_retry == client_clone.session.nb_retry assert self.client.session.retry_after == client_clone.session.retry_after assert self.client._bucket_name == client_clone._bucket_name assert self.client._collection_name == client_clone._collection_name def test_client_clone_with_new_session(self): session = create_session(auth=("reviewer", ""), server_url="https://kinto.notmyidea.org/v1") client_clone = self.client.clone(session=session) assert client_clone.session == session assert self.client.session != client_clone.session assert self.client.session.server_url != client_clone.session.server_url assert self.client.session.auth != client_clone.session.auth assert self.client._bucket_name == client_clone._bucket_name assert self.client._collection_name == client_clone._collection_name def test_client_clone_with_auth_and_server_url(self): client_clone = self.client.clone(auth=("reviewer", ""), server_url="https://kinto.notmyidea.org/v1") assert client_clone.session.auth == ("reviewer", "") assert client_clone.session.server_url == "https://kinto.notmyidea.org/v1" assert self.client.session != client_clone.session assert self.client.session.server_url != client_clone.session.server_url assert self.client.session.auth != client_clone.session.auth assert self.client.session.nb_retry == client_clone.session.nb_retry assert self.client.session.retry_after == client_clone.session.retry_after assert self.client._bucket_name == client_clone._bucket_name assert self.client._collection_name == client_clone._collection_name def test_client_clone_with_existing_session(self): client_clone = self.client.clone(session=self.client.session) assert self.client.session == client_clone.session assert self.client.session.server_url == client_clone.session.server_url assert self.client.session.auth == client_clone.session.auth assert self.client._bucket_name == client_clone._bucket_name assert self.client._collection_name == client_clone._collection_name def test_client_clone_with_new_bucket_and_collection(self): client_clone = self.client.clone(bucket="bucket_blah", collection="coll_blah") assert self.client.session == client_clone.session assert self.client.session.server_url == client_clone.session.server_url assert self.client.session.auth == client_clone.session.auth assert self.client.session.nb_retry == client_clone.session.nb_retry assert self.client.session.retry_after == client_clone.session.retry_after assert self.client._bucket_name != client_clone._bucket_name assert self.client._collection_name != client_clone._collection_name assert client_clone._bucket_name == "bucket_blah" assert client_clone._collection_name == "coll_blah" def test_client_clone_with_auth_and_server_url_bucket_and_collection(self): client_clone = self.client.clone(auth=("reviewer", ""), server_url="https://kinto.notmyidea.org/v1", bucket="bucket_blah", collection="coll_blah") assert self.client.session != client_clone.session assert self.client.session.server_url != client_clone.session.server_url assert self.client.session.auth != client_clone.session.auth assert self.client._bucket_name != client_clone._bucket_name assert self.client._collection_name != client_clone._collection_name assert client_clone.session.auth == ("reviewer", "") assert client_clone.session.server_url == "https://kinto.notmyidea.org/v1" assert client_clone._bucket_name == "bucket_blah" assert client_clone._collection_name == "coll_blah"
def backport_records(event, context, **kwargs): """Backport records creations, updates and deletions from one collection to another. """ server_url = event["server"] source_auth = ( event.get("backport_records_source_auth") or os.environ["BACKPORT_RECORDS_SOURCE_AUTH"] ) source_bucket = ( event.get("backport_records_source_bucket") or os.environ["BACKPORT_RECORDS_SOURCE_BUCKET"] ) source_collection = ( event.get("backport_records_source_collection") or os.environ["BACKPORT_RECORDS_SOURCE_COLLECTION"] ) dest_auth = event.get( "backport_records_dest_auth", os.getenv("BACKPORT_RECORDS_DEST_AUTH", source_auth), ) dest_bucket = event.get( "backport_records_dest_bucket", os.getenv("BACKPORT_RECORDS_DEST_BUCKET", source_bucket), ) dest_collection = event.get( "backport_records_dest_collection", os.getenv("BACKPORT_RECORDS_DEST_COLLECTION", source_collection), ) if source_bucket == dest_bucket and source_collection == dest_collection: raise ValueError("Cannot copy records: destination is identical to source") source_client = Client( server_url=server_url, bucket=source_bucket, collection=source_collection, auth=tuple(source_auth.split(":", 1)) if ":" in source_auth else BearerTokenAuth(source_auth), ) dest_client = Client( server_url=server_url, bucket=dest_bucket, collection=dest_collection, auth=tuple(dest_auth.split(":", 1)) if ":" in dest_auth else BearerTokenAuth(dest_auth), ) source_timestamp = source_client.get_records_timestamp() dest_timestamp = dest_client.get_records_timestamp() if source_timestamp <= dest_timestamp: print("Records are in sync. Nothing to do.") return source_records = source_client.get_records() dest_records_by_id = {r["id"]: r for r in dest_client.get_records()} with dest_client.batch() as dest_batch: # Create or update the destination records. for r in source_records: dest_record = dest_records_by_id.pop(r["id"], None) if dest_record is None: dest_batch.create_record(data=r) elif r["last_modified"] > dest_record["last_modified"]: dest_batch.update_record(data=r) # Delete the records missing from source. for r in dest_records_by_id.values(): dest_batch.delete_record(id=r["id"]) ops_count = len(dest_batch.results()) # If destination has signing, request review or auto-approve changes. server_info = dest_client.server_info() signer_config = server_info["capabilities"].get("signer", {}) signer_resources = signer_config.get("resources", []) # Check destination collection config (sign-off required etc.) signed_dest = [ r for r in signer_resources if r["source"]["bucket"] == dest_bucket and r["source"]["collection"] == dest_collection ] if len(signed_dest) == 0: # Not explicitly configured. Check if configured at bucket level? signed_dest = [ r for r in signer_resources if r["source"]["bucket"] == dest_bucket and r["source"]["collection"] is None ] # Destination has no signature enabled. Nothing to do. if len(signed_dest) == 0: print(f"Done. {ops_count} changes applied.") return has_autoapproval = not signed_dest[0].get( "to_review_enabled", signer_config["to_review_enabled"] ) and not signed_dest[0].get( "group_check_enabled", signer_config["group_check_enabled"] ) if has_autoapproval: # Approve the changes. dest_client.patch_collection(data={"status": "to-sign"}) print(f"Done. {ops_count} changes applied and signed.") else: # Request review. dest_client.patch_collection(data={"status": "to-review"}) print(f"Done. Requested review for {ops_count} changes.")
class ClientTest(unittest.TestCase): def setUp(self): self.session = mock.MagicMock() self.client = Client(session=self.session) mock_response(self.session) def test_server_info(self): self.client.server_info() self.session.request.assert_called_with('get', '/') def test_context_manager_works_as_expected(self): settings = {"batch_max_requests": 25} self.session.request.side_effect = [({ "settings": settings }, []), ({ "responses": [] }, [])] with self.client.batch(bucket='mozilla', collection='test') as batch: batch.create_record(id=1234, data={'foo': 'bar'}) batch.create_record(id=5678, data={'bar': 'baz'}) self.session.request.assert_called_with( method='POST', endpoint='/batch', payload={ 'requests': [{ 'body': { 'data': { 'foo': 'bar' } }, 'path': '/buckets/mozilla/collections/test/records/1234', 'method': 'PUT', 'headers': { 'If-None-Match': '*', 'User-Agent': USER_AGENT } }, { 'body': { 'data': { 'bar': 'baz' } }, 'path': '/buckets/mozilla/collections/test/records/5678', 'method': 'PUT', 'headers': { 'If-None-Match': '*', 'User-Agent': USER_AGENT } }] }) def test_batch_raises_exception(self): # Make the next call to sess.request raise a 403. exception = KintoException() exception.response = mock.MagicMock() exception.response.status_code = 403 exception.request = mock.sentinel.request self.session.request.side_effect = exception with self.assertRaises(KintoException): with self.client.batch(bucket='moz', collection='test') as batch: batch.create_record(id=1234, data={'foo': 'bar'}) def test_batch_raises_exception_if_subrequest_failed_with_code_5xx(self): error = { "errno": 121, "message": "This user cannot access this resource.", "code": 500, "error": "Server Internal Error" } self.session.request.side_effect = [({ "settings": { "batch_max_requests": 25 } }, []), ({ "responses": [{ "status": 200, "path": "/url1", "body": {}, "headers": {} }, { "status": 500, "path": "/url2", "body": error, "headers": {} }] }, [])] with self.assertRaises(KintoException): with self.client.batch(bucket='moz', collection='test') as batch: batch.create_record(id=1234, data={'foo': 'bar'}) batch.create_record(id=5678, data={'tutu': 'toto'}) def test_batch_dont_raise_exception_if_subrequest_failed_with_code_4xx( self): error = { "errno": 121, "message": "Forbidden", "code": 403, "error": "This user cannot access this resource." } self.session.request.side_effect = [({ "settings": { "batch_max_requests": 25 } }, []), ({ "responses": [{ "status": 200, "path": "/url1", "body": {}, "headers": {} }, { "status": 403, "path": "/url2", "body": error, "headers": {} }] }, [])] with self.client.batch(bucket='moz', collection='test') as batch: # Do not raise batch.create_record(id=1234, data={'foo': 'bar'}) batch.create_record(id=5678, data={'tutu': 'toto'}) def test_batch_options_are_transmitted(self): settings = {"batch_max_requests": 25} self.session.request.side_effect = [({"settings": settings}, [])] with mock.patch('kinto_http.create_session') as create_session: with self.client.batch(bucket='moz', collection='test', retry=12, retry_after=20): _, last_call_kwargs = create_session.call_args_list[-1] self.assertEqual(last_call_kwargs['retry'], 12) self.assertEqual(last_call_kwargs['retry_after'], 20) def test_client_is_represented_properly_with_bucket_and_collection(self): client = Client(server_url="https://kinto.notmyidea.org/v1", bucket="homebrewing", collection="recipes") expected_repr = ("<KintoClient https://kinto.notmyidea.org/v1/" "buckets/homebrewing/collections/recipes>") assert str(client) == expected_repr def test_client_is_represented_properly_with_bucket(self): client = Client( server_url="https://kinto.notmyidea.org/v1", bucket="homebrewing", ) expected_repr = ("<KintoClient https://kinto.notmyidea.org/v1/" "buckets/homebrewing>") assert str(client) == expected_repr def test_client_is_represented_properly_without_bucket(self): client = Client(server_url="https://kinto.notmyidea.org/v1", bucket=None) expected_repr = ("<KintoClient https://kinto.notmyidea.org/v1/>") assert str(client) == expected_repr def test_client_uses_default_bucket_if_not_specified(self): mock_response(self.session) client = Client(session=self.session) client.get_bucket() self.session.request.assert_called_with('get', '/buckets/default') def test_client_uses_passed_bucket_if_specified(self): client = Client(server_url="https://kinto.notmyidea.org/v1", bucket="buck") assert client._bucket_name == "buck" def test_client_clone_with_auth(self): client_clone = self.client.clone(auth=("reviewer", "")) assert client_clone.session.auth == ("reviewer", "") assert self.client.session != client_clone.session assert self.client.session.server_url == client_clone.session.server_url assert self.client.session.auth != client_clone.session.auth assert self.client.session.nb_retry == client_clone.session.nb_retry assert self.client.session.retry_after == client_clone.session.retry_after assert self.client._bucket_name == client_clone._bucket_name assert self.client._collection_name == client_clone._collection_name def test_client_clone_with_server_url(self): client_clone = self.client.clone( server_url="https://kinto.notmyidea.org/v1") assert client_clone.session.server_url == "https://kinto.notmyidea.org/v1" assert self.client.session != client_clone.session assert self.client.session.server_url != client_clone.session.server_url assert self.client.session.auth == client_clone.session.auth assert self.client.session.nb_retry == client_clone.session.nb_retry assert self.client.session.retry_after == client_clone.session.retry_after assert self.client._bucket_name == client_clone._bucket_name assert self.client._collection_name == client_clone._collection_name def test_client_clone_with_new_session(self): session = create_session(auth=("reviewer", ""), server_url="https://kinto.notmyidea.org/v1") client_clone = self.client.clone(session=session) assert client_clone.session == session assert self.client.session != client_clone.session assert self.client.session.server_url != client_clone.session.server_url assert self.client.session.auth != client_clone.session.auth assert self.client._bucket_name == client_clone._bucket_name assert self.client._collection_name == client_clone._collection_name def test_client_clone_with_auth_and_server_url(self): client_clone = self.client.clone( auth=("reviewer", ""), server_url="https://kinto.notmyidea.org/v1") assert client_clone.session.auth == ("reviewer", "") assert client_clone.session.server_url == "https://kinto.notmyidea.org/v1" assert self.client.session != client_clone.session assert self.client.session.server_url != client_clone.session.server_url assert self.client.session.auth != client_clone.session.auth assert self.client.session.nb_retry == client_clone.session.nb_retry assert self.client.session.retry_after == client_clone.session.retry_after assert self.client._bucket_name == client_clone._bucket_name assert self.client._collection_name == client_clone._collection_name def test_client_clone_with_existing_session(self): client_clone = self.client.clone(session=self.client.session) assert self.client.session == client_clone.session assert self.client.session.server_url == client_clone.session.server_url assert self.client.session.auth == client_clone.session.auth assert self.client._bucket_name == client_clone._bucket_name assert self.client._collection_name == client_clone._collection_name def test_client_clone_with_new_bucket_and_collection(self): client_clone = self.client.clone(bucket="bucket_blah", collection="coll_blah") assert self.client.session == client_clone.session assert self.client.session.server_url == client_clone.session.server_url assert self.client.session.auth == client_clone.session.auth assert self.client.session.nb_retry == client_clone.session.nb_retry assert self.client.session.retry_after == client_clone.session.retry_after assert self.client._bucket_name != client_clone._bucket_name assert self.client._collection_name != client_clone._collection_name assert client_clone._bucket_name == "bucket_blah" assert client_clone._collection_name == "coll_blah" def test_client_clone_with_auth_and_server_url_bucket_and_collection(self): client_clone = self.client.clone( auth=("reviewer", ""), server_url="https://kinto.notmyidea.org/v1", bucket="bucket_blah", collection="coll_blah") assert self.client.session != client_clone.session assert self.client.session.server_url != client_clone.session.server_url assert self.client.session.auth != client_clone.session.auth assert self.client._bucket_name != client_clone._bucket_name assert self.client._collection_name != client_clone._collection_name assert client_clone.session.auth == ("reviewer", "") assert client_clone.session.server_url == "https://kinto.notmyidea.org/v1" assert client_clone._bucket_name == "bucket_blah" assert client_clone._collection_name == "coll_blah"
def main(): args = _get_args() client = Client(server_url=args.server, auth=tuple(args.auth.split(':')), bucket=args.source_bucket, collection=args.source_col) if args.editor_auth is None: args.editor_auth = args.auth if args.reviewer_auth is None: args.reviewer_auth = args.auth editor_client = Client(server_url=args.server, auth=tuple(args.editor_auth.split(':')), bucket=args.source_bucket, collection=args.source_col) reviewer_client = Client(server_url=args.server, auth=tuple(args.reviewer_auth.split(':')), bucket=args.source_bucket, collection=args.source_col) # 0. initialize source bucket/collection (if necessary) server_info = client.server_info() editor_id = editor_client.server_info()['user']['id'] reviewer_id = reviewer_client.server_info()['user']['id'] print('Server: {0}'.format(args.server)) print('Author: {user[id]}'.format(**server_info)) print('Editor: {0}'.format(editor_id)) print('Reviewer: {0}'.format(reviewer_id)) # 0. check that this collection is well configured. signer_capabilities = server_info['capabilities']['signer'] to_review_enabled = signer_capabilities.get('to_review_enabled', False) group_check_enabled = signer_capabilities.get('group_check_enabled', False) resources = [ r for r in signer_capabilities['resources'] if (args.source_bucket, args.source_col) == (r['source']['bucket'], r['source']['collection']) ] assert len(resources) > 0, 'Specified source not configured to be signed' resource = resources[0] if to_review_enabled and 'preview' in resource: print( 'Signoff: {source[bucket]}/{source[collection]} => {preview[bucket]}/{preview[collection]} => {destination[bucket]}/{destination[collection]}' .format(**resource)) else: print( 'Signoff: {source[bucket]}/{source[collection]} => {destination[bucket]}/{destination[collection]}' .format(**resource)) print('Group check: {0}'.format(group_check_enabled)) print('Review workflow: {0}'.format(to_review_enabled)) print('_' * 80) bucket = client.create_bucket(if_not_exists=True) client.patch_bucket(permissions={ 'write': [editor_id, reviewer_id] + bucket['permissions']['write'] }, if_match=bucket['data']['last_modified'], safe=True) client.create_collection(if_not_exists=True) if args.reset: client.delete_records() existing = 0 else: existing_records = client.get_records() existing = len(existing_records) if group_check_enabled: editors_group = signer_capabilities['editors_group'] client.create_group(editors_group, data={'members': [editor_id]}, if_not_exists=True) reviewers_group = signer_capabilities['reviewers_group'] client.create_group(reviewers_group, data={'members': [reviewer_id]}, if_not_exists=True) dest_client = Client(server_url=args.server, bucket=resource['destination']['bucket'], collection=resource['destination']['collection']) preview_client = None if to_review_enabled and 'preview' in resource: preview_bucket = resource['preview']['bucket'] preview_collection = resource['preview']['collection'] preview_client = Client(server_url=args.server, bucket=preview_bucket, collection=preview_collection) # 1. upload data print('Author uploads 20 random records') records = upload_records(client, 20) # 2. ask for a signature # 2.1 ask for review (noop on old versions) print('Editor asks for review') data = {"status": "to-review"} editor_client.patch_collection(data=data) # 2.2 check the preview collection (if enabled) if preview_client: print('Check preview collection') preview_records = preview_client.get_records() expected = existing + 20 assert len(preview_records) == expected, '%s != %s records' % ( len(preview_records), expected) metadata = preview_client.get_collection()['data'] preview_signature = metadata.get('signature') assert preview_signature, 'Preview collection not signed' preview_timestamp = collection_timestamp(preview_client) # 2.3 approve the review print('Reviewer approves and triggers signature') data = {"status": "to-sign"} reviewer_client.patch_collection(data=data) # 3. upload more data print('Author creates 20 others records') upload_records(client, 20) print('Editor updates 5 random records') for toupdate in random.sample(records, 5): editor_client.patch_record(dict(newkey=_rand(10), **toupdate)) print('Author deletes 5 random records') for todelete in random.sample(records, 5): client.delete_record(todelete['id']) expected = existing + 20 + 20 - 5 # 4. ask again for a signature # 2.1 ask for review (noop on old versions) print('Editor asks for review') data = {"status": "to-review"} editor_client.patch_collection(data=data) # 2.2 check the preview collection (if enabled) if preview_client: print('Check preview collection') preview_records = preview_client.get_records() assert len(preview_records) == expected, '%s != %s records' % ( len(preview_records), expected) # Diff size is 20 + 5 if updated records are also all deleted, # or 30 if deletions and updates apply to different records. diff_since_last = preview_client.get_records(_since=preview_timestamp) assert 25 <= len( diff_since_last ) <= 30, 'Changes since last signature are not consistent' metadata = preview_client.get_collection()['data'] assert preview_signature != metadata[ 'signature'], 'Preview collection not updated' # 2.3 approve the review print('Reviewer approves and triggers signature') data = {"status": "to-sign"} reviewer_client.patch_collection(data=data) # 5. wait for the result # 6. obtain the destination records and serialize canonically. records = list(dest_client.get_records()) assert len(records) == expected, '%s != %s records' % (len(records), expected) timestamp = collection_timestamp(dest_client) serialized = canonical_json(records, timestamp) print('Hash is %r' % compute_hash(serialized)) # 7. get back the signed hash dest_col = dest_client.get_collection() signature = dest_col['data']['signature'] with open('pub', 'w') as f: f.write(signature['public_key']) # 8. verify the signature matches the hash signer = ECDSASigner(public_key='pub') try: signer.verify(serialized, signature) print('Signature OK') except Exception: print('Signature KO') raise
def main(): args = _get_args() client = Client(server_url=args.server, auth=tuple(args.auth.split(':')), bucket=args.source_bucket, collection=args.source_col) if args.editor_auth is None: args.editor_auth = args.auth if args.reviewer_auth is None: args.reviewer_auth = args.auth editor_client = Client(server_url=args.server, auth=tuple(args.editor_auth.split(':')), bucket=args.source_bucket, collection=args.source_col) reviewer_client = Client(server_url=args.server, auth=tuple(args.reviewer_auth.split(':')), bucket=args.source_bucket, collection=args.source_col) # 0. initialize source bucket/collection (if necessary) server_info = client.server_info() editor_id = editor_client.server_info()['user']['id'] reviewer_id = reviewer_client.server_info()['user']['id'] print('Server: {0}'.format(args.server)) print('Author: {user[id]}'.format(**server_info)) print('Editor: {0}'.format(editor_id)) print('Reviewer: {0}'.format(reviewer_id)) # 0. check that this collection is well configured. signer_capabilities = server_info['capabilities']['signer'] resources = [r for r in signer_capabilities['resources'] if (args.source_bucket, args.source_col) == (r['source']['bucket'], r['source']['collection']) or (args.source_bucket, None) == (r['source']['bucket'], r['source']['collection'])] assert len(resources) > 0, 'Specified source not configured to be signed' resource = resources[0] if 'preview' in resource: print('Signoff: {source[bucket]}/{source[collection]} => {preview[bucket]}/{preview[collection]} => {destination[bucket]}/{destination[collection]}'.format(**resource)) else: print('Signoff: {source[bucket]}/{source[collection]} => {destination[bucket]}/{destination[collection]}'.format(**resource)) print('_' * 80) bucket = client.create_bucket(if_not_exists=True) client.create_collection(permissions={'write': [editor_id, reviewer_id] + bucket['permissions']['write']}, if_not_exists=True) editors_group = resource.get('editors_group') or signer_capabilities['editors_group'] editors_group = editors_group.format(collection_id=args.source_col) client.patch_group(id=editors_group, data={'members': [editor_id]}) reviewers_group = resource.get('reviewers_group') or signer_capabilities['reviewers_group'] reviewers_group = reviewers_group.format(collection_id=args.source_col) client.patch_group(id=reviewers_group, data={'members': [reviewer_id]}) if args.reset: client.delete_records() existing = 0 else: existing_records = client.get_records() existing = len(existing_records) dest_col = resource['destination'].get('collection') or args.source_col dest_client = Client(server_url=args.server, bucket=resource['destination']['bucket'], collection=dest_col) preview_client = None if 'preview' in resource: preview_bucket = resource['preview']['bucket'] preview_collection = resource['preview'].get('collection') or args.source_col preview_client = Client(server_url=args.server, bucket=preview_bucket, collection=preview_collection) # 1. upload data print('Author uploads 20 random records') records = upload_records(client, 20) # 2. ask for a signature # 2.1 ask for review (noop on old versions) print('Editor asks for review') data = {"status": "to-review"} editor_client.patch_collection(data=data) # 2.2 check the preview collection (if enabled) if preview_client: print('Check preview collection') preview_records = preview_client.get_records() expected = existing + 20 assert len(preview_records) == expected, '%s != %s records' % (len(preview_records), expected) metadata = preview_client.get_collection()['data'] preview_signature = metadata.get('signature') assert preview_signature, 'Preview collection not signed' preview_timestamp = preview_client.get_records_timestamp() # 2.3 approve the review print('Reviewer approves and triggers signature') data = {"status": "to-sign"} reviewer_client.patch_collection(data=data) # 3. upload more data print('Author creates 20 others records') upload_records(client, 20) print('Editor updates 5 random records') for toupdate in random.sample(records, 5): editor_client.patch_record(data=dict(newkey=_rand(10), **toupdate)) print('Author deletes 5 random records') for todelete in random.sample(records, 5): client.delete_record(id=todelete['id']) expected = existing + 20 + 20 - 5 # 4. ask again for a signature # 2.1 ask for review (noop on old versions) print('Editor asks for review') data = {"status": "to-review"} editor_client.patch_collection(data=data) # 2.2 check the preview collection (if enabled) if preview_client: print('Check preview collection') preview_records = preview_client.get_records() assert len(preview_records) == expected, '%s != %s records' % (len(preview_records), expected) # Diff size is 20 + 5 if updated records are also all deleted, # or 30 if deletions and updates apply to different records. diff_since_last = preview_client.get_records(_since=preview_timestamp) assert 25 <= len(diff_since_last) <= 30, 'Changes since last signature are not consistent' metadata = preview_client.get_collection()['data'] assert preview_signature != metadata['signature'], 'Preview collection not updated' # 2.3 approve the review print('Reviewer approves and triggers signature') data = {"status": "to-sign"} reviewer_client.patch_collection(data=data) # 5. wait for the result # 6. obtain the destination records and serialize canonically. records = list(dest_client.get_records()) assert len(records) == expected, '%s != %s records' % (len(records), expected) timestamp = dest_client.get_records_timestamp() serialized = canonical_json(records, timestamp) print('Hash is %r' % compute_hash(serialized)) # 7. get back the signed hash signature = dest_client.get_collection()['data']['signature'] with open('pub', 'w') as f: f.write(signature['public_key']) # 8. verify the signature matches the hash signer = ECDSASigner(public_key='pub') try: signer.verify(serialized, signature) print('Signature OK') except Exception: print('Signature KO') raise