def validate_changes_collection(event, context, **kwargs): """Validate the entries of the monitor endpoint. """ # 1. Grab the changes collection server_url = event["server"] bucket = event.get("bucket", os.getenv("BUCKET", "monitor")) collection = event.get("collection", os.getenv("COLLECTION", "changes")) client = Client(server_url=server_url, bucket=bucket, collection=collection) print("Looking at %s: " % client.get_endpoint("collection")) collections = client.get_records() # 2. For each collection there, validate the ETag everything_ok = True for collection in collections: bid = collection["bucket"] cid = collection["collection"] last_modified = collection["last_modified"] etag = client.get_records_timestamp(bucket=bid, collection=cid) if str(etag) == str(last_modified): print("Etag OK for {}/{} : {}".format(bid, cid, etag)) else: everything_ok = False print("Etag NOT OK for {}/{} : {} != {}".format( bid, cid, last_modified, etag)) if not everything_ok: raise ValueError("One of the collection did not validate.")
def test_records_timestamp_retrieval(self): client = Client(server_url=self.server_url, auth=self.auth, bucket='mozilla', collection='payments') client.create_bucket() client.create_collection() record = client.create_record(data={'foo': 'bar'}, permissions={'read': ['account:alexis']}) etag = client.get_records_timestamp() assert str(etag) == str(record["data"]["last_modified"])
def test_records_timestamp_retrieval(self): client = Client(server_url=self.server_url, auth=self.auth, bucket='mozilla', collection='payments') client.create_bucket() client.create_collection() record = client.create_record(data={'foo': 'bar'}, permissions={'read': ['alexis']}) etag = client.get_records_timestamp() assert str(etag) == str(record["data"]["last_modified"])
def test_records_timestamp_retrieval(self): client = Client(server_url=self.server_url, auth=self.auth, bucket="mozilla", collection="payments") client.create_bucket() client.create_collection() record = client.create_record(data={"foo": "bar"}, permissions={"read": ["account:alexis"]}) etag = client.get_records_timestamp() assert str(etag) == str(record["data"]["last_modified"])
def download_collection_data(server_url, collection): client = Client( server_url=server_url, bucket=collection["bucket"], collection=collection["collection"], ) endpoint = client.get_endpoint("collection") # Collection metadata with cache busting metadata = client.get_collection( _expected=collection["last_modified"])["data"] # Download records with cache busting records = client.get_records(_sort="-last_modified", _expected=collection["last_modified"]) timestamp = client.get_records_timestamp() return (collection, endpoint, metadata, records, timestamp)
class RecordTest(unittest.TestCase): def setUp(self): self.session = mock.MagicMock() self.client = Client( session=self.session, bucket='mybucket', collection='mycollection') def test_record_id_is_given_after_creation(self): mock_response(self.session, data={'id': 5678}) record = self.client.create_record({'foo': 'bar'}) assert 'id' in record['data'].keys() def test_generated_record_id_is_an_uuid(self): mock_response(self.session) self.client.create_record({'foo': 'bar'}) id = self.session.request.mock_calls[0][1][1].split('/')[-1] uuid_regexp = r'[\w]{8}-[\w]{4}-[\w]{4}-[\w]{4}-[\w]{12}' self.assertRegexpMatches(id, uuid_regexp) def test_records_handles_permissions(self): mock_response(self.session) self.client.create_record( {'id': '1234', 'foo': 'bar'}, permissions=mock.sentinel.permissions) self.session.request.assert_called_with( 'put', '/buckets/mybucket/collections/mycollection/records/1234', data={'foo': 'bar', 'id': '1234'}, permissions=mock.sentinel.permissions, headers=DO_NOT_OVERWRITE) def test_collection_argument_takes_precedence(self): mock_response(self.session) # Specify a different collection name for the client and the operation. client = Client(session=self.session, bucket='mybucket', collection='wrong_collection') client.update_record(data={'id': '1234'}, collection='good_collection', permissions=mock.sentinel.permissions) self.session.request.assert_called_with( 'put', '/buckets/mybucket/collections/good_collection/records/1234', data={'id': '1234'}, headers=None, permissions=mock.sentinel.permissions) def test_record_id_is_derived_from_data_if_present(self): mock_response(self.session) self.client.create_record(data={'id': '1234', 'foo': 'bar'}, permissions=mock.sentinel.permissions) self.session.request.assert_called_with( 'put', '/buckets/mybucket/collections/mycollection/records/1234', data={'id': '1234', 'foo': 'bar'}, permissions=mock.sentinel.permissions, headers=DO_NOT_OVERWRITE) def test_data_and_permissions_are_added_on_create(self): mock_response(self.session) data = {'foo': 'bar'} permissions = {'read': ['mle']} self.client.create_record( id='1234', data=data, permissions=permissions) url = '/buckets/mybucket/collections/mycollection/records/1234' self.session.request.assert_called_with( 'put', url, data=data, permissions=permissions, headers=DO_NOT_OVERWRITE) def test_creation_sends_if_none_match_by_default(self): mock_response(self.session) data = {'foo': 'bar'} self.client.create_record( id='1234', data=data) url = '/buckets/mybucket/collections/mycollection/records/1234' self.session.request.assert_called_with( 'put', url, data=data, permissions=None, headers=DO_NOT_OVERWRITE) def test_creation_doesnt_add_if_none_match_when_overwrite(self): mock_response(self.session) data = {'foo': 'bar'} self.client.create_record(id='1234', data=data, safe=False) url = '/buckets/mybucket/collections/mycollection/records/1234' self.session.request.assert_called_with( 'put', url, data=data, permissions=None, headers=None) def test_records_issues_a_request_on_delete(self): mock_response(self.session) self.client.delete_record('1234') url = '/buckets/mybucket/collections/mycollection/records/1234' self.session.request.assert_called_with('delete', url, headers=None) def test_record_issues_a_request_on_retrieval(self): mock_response(self.session, data={'foo': 'bar'}) record = self.client.get_record('1234') self.assertEquals(record['data'], {'foo': 'bar'}) url = '/buckets/mybucket/collections/mycollection/records/1234' self.session.request.assert_called_with('get', url) def test_collection_can_retrieve_all_records(self): mock_response(self.session, data=[{'id': 'foo'}, {'id': 'bar'}]) records = self.client.get_records() assert list(records) == [{'id': 'foo'}, {'id': 'bar'}] def test_collection_can_retrieve_records_timestamp(self): mock_response(self.session, data=[{'id': 'foo'}, {'id': 'bar'}], headers={"ETag": '"12345"'}) timestamp = self.client.get_records_timestamp() assert timestamp == '12345' def test_records_timestamp_is_cached(self): mock_response(self.session, data=[{'id': 'foo'}, {'id': 'bar'}], headers={"ETag": '"12345"'}) self.client.get_records() timestamp = self.client.get_records_timestamp() assert timestamp == '12345' assert self.session.request.call_count == 1 def test_records_timestamp_is_cached_per_collection(self): mock_response(self.session, data=[{'id': 'foo'}, {'id': 'bar'}], headers={"ETag": '"12345"'}) self.client.get_records(collection="foo") mock_response(self.session, data=[{'id': 'foo'}, {'id': 'bar'}], headers={"ETag": '"67890"'}) self.client.get_records(collection="bar") timestamp = self.client.get_records_timestamp("foo") assert timestamp == '12345' timestamp = self.client.get_records_timestamp("bar") assert timestamp == '67890' def test_pagination_is_followed(self): # Mock the calls to request. link = ('http://example.org/buckets/buck/collections/coll/records/' '?token=1234') self.session.request.side_effect = [ # First one returns a list of items with a pagination token. build_response( [{'id': '1', 'value': 'item1'}, {'id': '2', 'value': 'item2'}, ], {'Next-Page': link}), # Second one returns a list of items without a pagination token. build_response( [{'id': '3', 'value': 'item3'}, {'id': '4', 'value': 'item4'}, ], ), ] records = self.client.get_records('bucket', 'collection') assert list(records) == [ {'id': '1', 'value': 'item1'}, {'id': '2', 'value': 'item2'}, {'id': '3', 'value': 'item3'}, {'id': '4', 'value': 'item4'}, ] def test_pagination_supports_if_none_match(self): link = ('http://example.org/buckets/buck/collections/coll/records/' '?token=1234') self.session.request.side_effect = [ # First one returns a list of items with a pagination token. build_response( [{'id': '1', 'value': 'item1'}, {'id': '2', 'value': 'item2'}, ], {'Next-Page': link}), # Second one returns a list of items without a pagination token. build_response( [{'id': '3', 'value': 'item3'}, {'id': '4', 'value': 'item4'}, ], ), ] self.client.get_records('bucket', 'collection', if_none_match="1234") # Check that the If-None-Match header is present in the requests. self.session.request.assert_any_call( 'get', '/buckets/collection/collections/bucket/records', headers={'If-None-Match': '"1234"'}, params={}) self.session.request.assert_any_call( 'get', link, headers={'If-None-Match': '"1234"'}, params={}) def test_collection_can_delete_a_record(self): mock_response(self.session, data={'id': 1234}) resp = self.client.delete_record(id=1234) assert resp == {'id': 1234} url = '/buckets/mybucket/collections/mycollection/records/1234' self.session.request.assert_called_with('delete', url, headers=None) def test_record_delete_if_match(self): data = {} mock_response(self.session, data=data) deleted = self.client.delete_record( collection='mycollection', bucket='mybucket', id='1', if_match=1234) assert deleted == data url = '/buckets/mybucket/collections/mycollection/records/1' self.session.request.assert_called_with( 'delete', url, headers={'If-Match': '"1234"'}) def test_record_delete_if_match_not_included_if_not_safe(self): data = {} mock_response(self.session, data=data) deleted = self.client.delete_record( collection='mycollection', bucket='mybucket', id='1', if_match=1234, safe=False) assert deleted == data url = '/buckets/mybucket/collections/mycollection/records/1' self.session.request.assert_called_with( 'delete', url, headers=None) def test_update_record_gets_the_id_from_data_if_exists(self): mock_response(self.session) self.client.update_record( bucket='mybucket', collection='mycollection', data={'id': 1, 'foo': 'bar'}) self.session.request.assert_called_with( 'put', '/buckets/mybucket/collections/mycollection/records/1', data={'id': 1, 'foo': 'bar'}, headers=None, permissions=None) def test_update_record_handles_if_match(self): mock_response(self.session) self.client.update_record( bucket='mybucket', collection='mycollection', data={'id': 1, 'foo': 'bar'}, if_match=1234) headers = {'If-Match': '"1234"'} self.session.request.assert_called_with( 'put', '/buckets/mybucket/collections/mycollection/records/1', data={'id': 1, 'foo': 'bar'}, headers=headers, permissions=None) def test_patch_record_uses_the_patch_method(self): mock_response(self.session) self.client.patch_record( bucket='mybucket', collection='mycollection', data={'id': 1, 'foo': 'bar'}) self.session.request.assert_called_with( 'patch', '/buckets/mybucket/collections/mycollection/records/1', data={'id': 1, 'foo': 'bar'}, headers=None, permissions=None) def test_update_record_raises_if_no_id_is_given(self): with self.assertRaises(KeyError) as cm: self.client.update_record( data={'foo': 'bar'}, # Omit the id on purpose here. bucket='mybucket', collection='mycollection' ) assert text_type(cm.exception) == ( "'Unable to update a record, need an id.'") def test_get_or_create_doesnt_raise_in_case_of_conflict(self): data = { 'permissions': mock.sentinel.permissions, 'data': {'foo': 'bar'} } self.session.request.side_effect = [ get_http_error(status=412), (data, None) ] returned_data = self.client.create_record( bucket="buck", collection="coll", data={'id': 1234, 'foo': 'bar'}, if_not_exists=True) # Should not raise. assert returned_data == data def test_get_or_create_raise_in_other_cases(self): self.session.request.side_effect = get_http_error(status=500) with self.assertRaises(KintoException): self.client.create_record( bucket="buck", collection="coll", data={'foo': 'bar'}, if_not_exists=True) def test_create_record_raises_a_special_error_on_403(self): self.session.request.side_effect = get_http_error(status=403) with self.assertRaises(KintoException) as e: self.client.create_record( bucket="buck", collection="coll", data={'foo': 'bar'}) expected_msg = ("Unauthorized. Please check that the collection exists" " and that you have the permission to create or write " "on this collection record.") assert e.exception.message == expected_msg
class RecordTest(unittest.TestCase): def setUp(self): self.session = mock.MagicMock() self.session.request.return_value = (mock.sentinel.response, mock.sentinel.count) self.client = Client( session=self.session, bucket='mybucket', collection='mycollection') def test_record_id_is_given_after_creation(self): mock_response(self.session, data={'id': 5678}) record = self.client.create_record(data={'foo': 'bar'}) assert 'id' in record['data'].keys() def test_generated_record_id_is_an_uuid(self): mock_response(self.session) self.client.create_record(data={'foo': 'bar'}) id = self.session.request.mock_calls[0][1][1].split('/')[-1] uuid_regexp = r'[\w]{8}-[\w]{4}-[\w]{4}-[\w]{4}-[\w]{12}' self.assertRegex(id, uuid_regexp) def test_records_handles_permissions(self): mock_response(self.session) self.client.create_record(data={'id': '1234', 'foo': 'bar'}, permissions=mock.sentinel.permissions) self.session.request.assert_called_with( 'put', '/buckets/mybucket/collections/mycollection/records/1234', data={'foo': 'bar', 'id': '1234'}, permissions=mock.sentinel.permissions, headers=DO_NOT_OVERWRITE) def test_collection_argument_takes_precedence(self): mock_response(self.session) # Specify a different collection name for the client and the operation. client = Client(session=self.session, bucket='mybucket', collection='wrong_collection') client.update_record(data={'id': '1234'}, collection='good_collection', permissions=mock.sentinel.permissions) self.session.request.assert_called_with( 'put', '/buckets/mybucket/collections/good_collection/records/1234', data={'id': '1234'}, headers=None, permissions=mock.sentinel.permissions) def test_record_id_is_derived_from_data_if_present(self): mock_response(self.session) self.client.create_record(data={'id': '1234', 'foo': 'bar'}, permissions=mock.sentinel.permissions) self.session.request.assert_called_with( 'put', '/buckets/mybucket/collections/mycollection/records/1234', data={'id': '1234', 'foo': 'bar'}, permissions=mock.sentinel.permissions, headers=DO_NOT_OVERWRITE) def test_data_and_permissions_are_added_on_create(self): mock_response(self.session) data = {'foo': 'bar'} permissions = {'read': ['mle']} self.client.create_record(id='1234', data=data, permissions=permissions) url = '/buckets/mybucket/collections/mycollection/records/1234' self.session.request.assert_called_with( 'put', url, data=data, permissions=permissions, headers=DO_NOT_OVERWRITE) def test_creation_sends_if_none_match_by_default(self): mock_response(self.session) data = {'foo': 'bar'} self.client.create_record(id='1234', data=data) url = '/buckets/mybucket/collections/mycollection/records/1234' self.session.request.assert_called_with( 'put', url, data=data, permissions=None, headers=DO_NOT_OVERWRITE) def test_creation_doesnt_add_if_none_match_when_overwrite(self): mock_response(self.session) data = {'foo': 'bar'} self.client.create_record(id='1234', data=data, safe=False) url = '/buckets/mybucket/collections/mycollection/records/1234' self.session.request.assert_called_with( 'put', url, data=data, permissions=None, headers=None) def test_records_issues_a_request_on_delete(self): mock_response(self.session) self.client.delete_record(id='1234') url = '/buckets/mybucket/collections/mycollection/records/1234' self.session.request.assert_called_with('delete', url, headers=None) def test_record_issues_a_request_on_retrieval(self): mock_response(self.session, data={'foo': 'bar'}) record = self.client.get_record(id='1234') self.assertEqual(record['data'], {'foo': 'bar'}) url = '/buckets/mybucket/collections/mycollection/records/1234' self.session.request.assert_called_with('get', url) def test_collection_can_retrieve_all_records(self): mock_response(self.session, data=[{'id': 'foo'}, {'id': 'bar'}]) records = self.client.get_records() assert list(records) == [{'id': 'foo'}, {'id': 'bar'}] def test_collection_can_retrieve_records_timestamp(self): mock_response(self.session, headers={"ETag": '"12345"'}) timestamp = self.client.get_records_timestamp() assert timestamp == '12345' def test_records_timestamp_is_cached(self): mock_response(self.session, data=[{'id': 'foo'}, {'id': 'bar'}], headers={"ETag": '"12345"'}) self.client.get_records() timestamp = self.client.get_records_timestamp() assert timestamp == '12345' assert self.session.request.call_count == 1 def test_records_timestamp_is_cached_per_collection(self): mock_response(self.session, data=[{'id': 'foo'}, {'id': 'bar'}], headers={"ETag": '"12345"'}) self.client.get_records(collection="foo") mock_response(self.session, data=[{'id': 'foo'}, {'id': 'bar'}], headers={"ETag": '"67890"'}) self.client.get_records(collection="bar") timestamp = self.client.get_records_timestamp(collection="foo") assert timestamp == '12345' timestamp = self.client.get_records_timestamp(collection="bar") assert timestamp == '67890' def test_pagination_is_followed(self): # Mock the calls to request. link = ('http://example.org/buckets/buck/collections/coll/records/' '?token=1234') self.session.request.side_effect = [ # First one returns a list of items with a pagination token. build_response( [{'id': '1', 'value': 'item1'}, {'id': '2', 'value': 'item2'}, ], {'Next-Page': link}), build_response( [{'id': '3', 'value': 'item3'}, {'id': '4', 'value': 'item4'}, ], {'Next-Page': link}), # Second one returns a list of items without a pagination token. build_response( [{'id': '5', 'value': 'item5'}, {'id': '6', 'value': 'item6'}, ], ), ] records = self.client.get_records(bucket='bucket', collection='collection') assert list(records) == [ {'id': '1', 'value': 'item1'}, {'id': '2', 'value': 'item2'}, {'id': '3', 'value': 'item3'}, {'id': '4', 'value': 'item4'}, {'id': '5', 'value': 'item5'}, {'id': '6', 'value': 'item6'}, ] def test_pagination_is_followed_for_number_of_pages(self): # Mock the calls to request. link = ('http://example.org/buckets/buck/collections/coll/records/' '?token=1234') self.session.request.side_effect = [ # First one returns a list of items with a pagination token. build_response( [{'id': '1', 'value': 'item1'}, {'id': '2', 'value': 'item2'}, ], {'Next-Page': link}), build_response( [{'id': '3', 'value': 'item3'}, {'id': '4', 'value': 'item4'}, ], {'Next-Page': link}), # Second one returns a list of items without a pagination token. build_response( [{'id': '5', 'value': 'item5'}, {'id': '6', 'value': 'item6'}, ], ), ] records = self.client.get_records(bucket='bucket', collection='collection', pages=2) assert list(records) == [ {'id': '1', 'value': 'item1'}, {'id': '2', 'value': 'item2'}, {'id': '3', 'value': 'item3'}, {'id': '4', 'value': 'item4'}, ] def test_pagination_is_not_followed_if_limit_is_specified(self): # Mock the calls to request. link = ('http://example.org/buckets/buck/collections/coll/records/' '?token=1234') self.session.request.side_effect = [ build_response( [{'id': '1', 'value': 'item1'}, {'id': '2', 'value': 'item2'}, ], {'Next-Page': link}), build_response( [{'id': '3', 'value': 'item3'}, {'id': '4', 'value': 'item4'}, ], ), ] records = self.client.get_records(bucket='bucket', collection='collection', _limit=2) assert list(records) == [ {'id': '1', 'value': 'item1'}, {'id': '2', 'value': 'item2'} ] def test_pagination_supports_if_none_match(self): link = ('http://example.org/buckets/buck/collections/coll/records/' '?token=1234') self.session.request.side_effect = [ # First one returns a list of items with a pagination token. build_response( [{'id': '1', 'value': 'item1'}, {'id': '2', 'value': 'item2'}, ], {'Next-Page': link}), # Second one returns a list of items without a pagination token. build_response( [{'id': '3', 'value': 'item3'}, {'id': '4', 'value': 'item4'}, ], ), ] self.client.get_records(bucket='bucket', collection='collection', if_none_match="1234") # Check that the If-None-Match header is present in the requests. self.session.request.assert_any_call( 'get', '/buckets/bucket/collections/collection/records', headers={'If-None-Match': '"1234"'}, params={}) self.session.request.assert_any_call( 'get', link, headers={'If-None-Match': '"1234"'}, params={}) def test_collection_can_delete_a_record(self): mock_response(self.session, data={'id': 1234}) resp = self.client.delete_record(id=1234) assert resp == {'id': 1234} url = '/buckets/mybucket/collections/mycollection/records/1234' self.session.request.assert_called_with('delete', url, headers=None) def test_record_delete_if_match(self): data = {} mock_response(self.session, data=data) deleted = self.client.delete_record(collection='mycollection', bucket='mybucket', id='1', if_match=1234) assert deleted == data url = '/buckets/mybucket/collections/mycollection/records/1' self.session.request.assert_called_with( 'delete', url, headers={'If-Match': '"1234"'}) def test_record_delete_if_match_not_included_if_not_safe(self): data = {} mock_response(self.session, data=data) deleted = self.client.delete_record(collection='mycollection', bucket='mybucket', id='1', if_match=1234, safe=False) assert deleted == data url = '/buckets/mybucket/collections/mycollection/records/1' self.session.request.assert_called_with( 'delete', url, headers=None) def test_update_record_gets_the_id_from_data_if_exists(self): mock_response(self.session) self.client.update_record(bucket='mybucket', collection='mycollection', data={'id': 1, 'foo': 'bar'}) self.session.request.assert_called_with( 'put', '/buckets/mybucket/collections/mycollection/records/1', data={'id': 1, 'foo': 'bar'}, headers=None, permissions=None) def test_update_record_handles_if_match(self): mock_response(self.session) self.client.update_record(bucket='mybucket', collection='mycollection', data={'id': 1, 'foo': 'bar'}, if_match=1234) headers = {'If-Match': '"1234"'} self.session.request.assert_called_with( 'put', '/buckets/mybucket/collections/mycollection/records/1', data={'id': 1, 'foo': 'bar'}, headers=headers, permissions=None) def test_patch_record_uses_the_patch_method(self): mock_response(self.session) self.client.patch_record(bucket='mybucket', collection='mycollection', data={'id': 1, 'foo': 'bar'}) self.session.request.assert_called_with( 'patch', '/buckets/mybucket/collections/mycollection/records/1', payload={'data': {'id': 1, 'foo': 'bar'}}, headers={"Content-Type": "application/json"}) def test_patch_record_recognizes_patchtype(self): mock_response(self.session) self.client.patch_record(bucket='mybucket', collection='mycollection', changes=MergePatch({'foo': 'bar'}, {'read': ['alice']}), id=1) self.session.request.assert_called_with( 'patch', '/buckets/mybucket/collections/mycollection/records/1', payload={'data': {'foo': 'bar'}, 'permissions': {'read': ['alice']}}, headers={"Content-Type": "application/merge-patch+json"}, ) def test_patch_record_understands_jsonpatch(self): mock_response(self.session) self.client.patch_record( bucket='mybucket', collection='mycollection', changes=JSONPatch([{'op': 'add', 'patch': '/baz', 'value': 'qux'}]), id=1) self.session.request.assert_called_with( 'patch', '/buckets/mybucket/collections/mycollection/records/1', payload=[{'op': 'add', 'patch': '/baz', 'value': 'qux'}], headers={"Content-Type": "application/json-patch+json"}, ) def test_patch_record_requires_data_to_be_patch_type(self): with pytest.raises(TypeError, match="couldn't understand patch body 5"): self.client.patch_record(id=1, collection='testcoll', bucket='testbucket', changes=5) def test_patch_record_requires_id(self): with pytest.raises(KeyError, match="Unable to patch record, need an id."): self.client.patch_record(collection='testcoll', bucket='testbucket', data={}) def test_update_record_raises_if_no_id_is_given(self): with self.assertRaises(KeyError) as cm: self.client.update_record(data={'foo': 'bar'}, # Omit the id on purpose here. bucket='mybucket', collection='mycollection') assert str(cm.exception) == "'Unable to update a record, need an id.'" def test_get_or_create_doesnt_raise_in_case_of_conflict(self): data = { 'permissions': mock.sentinel.permissions, 'data': {'foo': 'bar'} } self.session.request.side_effect = [ get_http_error(status=412), (data, None) ] returned_data = self.client.create_record(bucket="buck", collection="coll", data={'id': 1234, 'foo': 'bar'}, if_not_exists=True) # Should not raise. assert returned_data == data def test_get_or_create_raise_in_other_cases(self): self.session.request.side_effect = get_http_error(status=500) with self.assertRaises(KintoException): self.client.create_record(bucket="buck", collection="coll", data={'foo': 'bar'}, id='record', if_not_exists=True) def test_create_record_raises_a_special_error_on_403(self): self.session.request.side_effect = get_http_error(status=403) with self.assertRaises(KintoException) as e: self.client.create_record(bucket="buck", collection="coll", data={'foo': 'bar'}) expected_msg = ("Unauthorized. Please check that the collection exists" " and that you have the permission to create or write " "on this collection record.") assert e.exception.message == expected_msg def test_create_record_can_deduce_id_from_data(self): self.client.create_record(data={'id': 'record'}, bucket='buck', collection='coll') self.session.request.assert_called_with( 'put', '/buckets/buck/collections/coll/records/record', data={'id': 'record'}, permissions=None, headers=DO_NOT_OVERWRITE) def test_update_record_can_deduce_id_from_data(self): self.client.update_record(data={'id': 'record'}, bucket='buck', collection='coll') self.session.request.assert_called_with( 'put', '/buckets/buck/collections/coll/records/record', data={'id': 'record'}, permissions=None, headers=None)
def backport_records(event, context, **kwargs): """Backport records creations, updates and deletions from one collection to another. """ server_url = event["server"] source_auth = ( event.get("backport_records_source_auth") or os.environ["BACKPORT_RECORDS_SOURCE_AUTH"] ) source_bucket = ( event.get("backport_records_source_bucket") or os.environ["BACKPORT_RECORDS_SOURCE_BUCKET"] ) source_collection = ( event.get("backport_records_source_collection") or os.environ["BACKPORT_RECORDS_SOURCE_COLLECTION"] ) dest_auth = event.get( "backport_records_dest_auth", os.getenv("BACKPORT_RECORDS_DEST_AUTH", source_auth), ) dest_bucket = event.get( "backport_records_dest_bucket", os.getenv("BACKPORT_RECORDS_DEST_BUCKET", source_bucket), ) dest_collection = event.get( "backport_records_dest_collection", os.getenv("BACKPORT_RECORDS_DEST_COLLECTION", source_collection), ) if source_bucket == dest_bucket and source_collection == dest_collection: raise ValueError("Cannot copy records: destination is identical to source") source_client = Client( server_url=server_url, bucket=source_bucket, collection=source_collection, auth=tuple(source_auth.split(":", 1)) if ":" in source_auth else BearerTokenAuth(source_auth), ) dest_client = Client( server_url=server_url, bucket=dest_bucket, collection=dest_collection, auth=tuple(dest_auth.split(":", 1)) if ":" in dest_auth else BearerTokenAuth(dest_auth), ) source_timestamp = source_client.get_records_timestamp() dest_timestamp = dest_client.get_records_timestamp() if source_timestamp <= dest_timestamp: print("Records are in sync. Nothing to do.") return source_records = source_client.get_records() dest_records_by_id = {r["id"]: r for r in dest_client.get_records()} with dest_client.batch() as dest_batch: # Create or update the destination records. for r in source_records: dest_record = dest_records_by_id.pop(r["id"], None) if dest_record is None: dest_batch.create_record(data=r) elif r["last_modified"] > dest_record["last_modified"]: dest_batch.update_record(data=r) # Delete the records missing from source. for r in dest_records_by_id.values(): dest_batch.delete_record(id=r["id"]) ops_count = len(dest_batch.results()) # If destination has signing, request review or auto-approve changes. server_info = dest_client.server_info() signer_config = server_info["capabilities"].get("signer", {}) signer_resources = signer_config.get("resources", []) # Check destination collection config (sign-off required etc.) signed_dest = [ r for r in signer_resources if r["source"]["bucket"] == dest_bucket and r["source"]["collection"] == dest_collection ] if len(signed_dest) == 0: # Not explicitly configured. Check if configured at bucket level? signed_dest = [ r for r in signer_resources if r["source"]["bucket"] == dest_bucket and r["source"]["collection"] is None ] # Destination has no signature enabled. Nothing to do. if len(signed_dest) == 0: print(f"Done. {ops_count} changes applied.") return has_autoapproval = not signed_dest[0].get( "to_review_enabled", signer_config["to_review_enabled"] ) and not signed_dest[0].get( "group_check_enabled", signer_config["group_check_enabled"] ) if has_autoapproval: # Approve the changes. dest_client.patch_collection(data={"status": "to-sign"}) print(f"Done. {ops_count} changes applied and signed.") else: # Request review. dest_client.patch_collection(data={"status": "to-review"}) print(f"Done. Requested review for {ops_count} changes.")
def main(): args = _get_args() client = Client(server_url=args.server, auth=tuple(args.auth.split(':')), bucket=args.source_bucket, collection=args.source_col) if args.editor_auth is None: args.editor_auth = args.auth if args.reviewer_auth is None: args.reviewer_auth = args.auth editor_client = Client(server_url=args.server, auth=tuple(args.editor_auth.split(':')), bucket=args.source_bucket, collection=args.source_col) reviewer_client = Client(server_url=args.server, auth=tuple(args.reviewer_auth.split(':')), bucket=args.source_bucket, collection=args.source_col) # 0. initialize source bucket/collection (if necessary) server_info = client.server_info() editor_id = editor_client.server_info()['user']['id'] reviewer_id = reviewer_client.server_info()['user']['id'] print('Server: {0}'.format(args.server)) print('Author: {user[id]}'.format(**server_info)) print('Editor: {0}'.format(editor_id)) print('Reviewer: {0}'.format(reviewer_id)) # 0. check that this collection is well configured. signer_capabilities = server_info['capabilities']['signer'] resources = [r for r in signer_capabilities['resources'] if (args.source_bucket, args.source_col) == (r['source']['bucket'], r['source']['collection']) or (args.source_bucket, None) == (r['source']['bucket'], r['source']['collection'])] assert len(resources) > 0, 'Specified source not configured to be signed' resource = resources[0] if 'preview' in resource: print('Signoff: {source[bucket]}/{source[collection]} => {preview[bucket]}/{preview[collection]} => {destination[bucket]}/{destination[collection]}'.format(**resource)) else: print('Signoff: {source[bucket]}/{source[collection]} => {destination[bucket]}/{destination[collection]}'.format(**resource)) print('_' * 80) bucket = client.create_bucket(if_not_exists=True) client.create_collection(permissions={'write': [editor_id, reviewer_id] + bucket['permissions']['write']}, if_not_exists=True) editors_group = resource.get('editors_group') or signer_capabilities['editors_group'] editors_group = editors_group.format(collection_id=args.source_col) client.patch_group(id=editors_group, data={'members': [editor_id]}) reviewers_group = resource.get('reviewers_group') or signer_capabilities['reviewers_group'] reviewers_group = reviewers_group.format(collection_id=args.source_col) client.patch_group(id=reviewers_group, data={'members': [reviewer_id]}) if args.reset: client.delete_records() existing = 0 else: existing_records = client.get_records() existing = len(existing_records) dest_col = resource['destination'].get('collection') or args.source_col dest_client = Client(server_url=args.server, bucket=resource['destination']['bucket'], collection=dest_col) preview_client = None if 'preview' in resource: preview_bucket = resource['preview']['bucket'] preview_collection = resource['preview'].get('collection') or args.source_col preview_client = Client(server_url=args.server, bucket=preview_bucket, collection=preview_collection) # 1. upload data print('Author uploads 20 random records') records = upload_records(client, 20) # 2. ask for a signature # 2.1 ask for review (noop on old versions) print('Editor asks for review') data = {"status": "to-review"} editor_client.patch_collection(data=data) # 2.2 check the preview collection (if enabled) if preview_client: print('Check preview collection') preview_records = preview_client.get_records() expected = existing + 20 assert len(preview_records) == expected, '%s != %s records' % (len(preview_records), expected) metadata = preview_client.get_collection()['data'] preview_signature = metadata.get('signature') assert preview_signature, 'Preview collection not signed' preview_timestamp = preview_client.get_records_timestamp() # 2.3 approve the review print('Reviewer approves and triggers signature') data = {"status": "to-sign"} reviewer_client.patch_collection(data=data) # 3. upload more data print('Author creates 20 others records') upload_records(client, 20) print('Editor updates 5 random records') for toupdate in random.sample(records, 5): editor_client.patch_record(data=dict(newkey=_rand(10), **toupdate)) print('Author deletes 5 random records') for todelete in random.sample(records, 5): client.delete_record(id=todelete['id']) expected = existing + 20 + 20 - 5 # 4. ask again for a signature # 2.1 ask for review (noop on old versions) print('Editor asks for review') data = {"status": "to-review"} editor_client.patch_collection(data=data) # 2.2 check the preview collection (if enabled) if preview_client: print('Check preview collection') preview_records = preview_client.get_records() assert len(preview_records) == expected, '%s != %s records' % (len(preview_records), expected) # Diff size is 20 + 5 if updated records are also all deleted, # or 30 if deletions and updates apply to different records. diff_since_last = preview_client.get_records(_since=preview_timestamp) assert 25 <= len(diff_since_last) <= 30, 'Changes since last signature are not consistent' metadata = preview_client.get_collection()['data'] assert preview_signature != metadata['signature'], 'Preview collection not updated' # 2.3 approve the review print('Reviewer approves and triggers signature') data = {"status": "to-sign"} reviewer_client.patch_collection(data=data) # 5. wait for the result # 6. obtain the destination records and serialize canonically. records = list(dest_client.get_records()) assert len(records) == expected, '%s != %s records' % (len(records), expected) timestamp = dest_client.get_records_timestamp() serialized = canonical_json(records, timestamp) print('Hash is %r' % compute_hash(serialized)) # 7. get back the signed hash signature = dest_client.get_collection()['data']['signature'] with open('pub', 'w') as f: f.write(signature['public_key']) # 8. verify the signature matches the hash signer = ECDSASigner(public_key='pub') try: signer.verify(serialized, signature) print('Signature OK') except Exception: print('Signature KO') raise