def get_all_subject_schemas(subject_name, host=TASR_HOST, port=TASR_PORT, timeout=TIMEOUT): ''' GET /tasr/subject/<subject name>/all_schemas Retrieves all the (canonical) schema versions registered for a subject, in version order, one per line in the response body. The multi-type IDs are included in the headers for confirmation. ''' url = ('http://%s:%s/tasr/subject/%s/all_schemas' % (host, port, subject_name)) resp = requests.get(url, timeout=timeout) if resp == None: raise TASRError('Timeout for get all subject schemas request.') if resp.status_code != 200: raise TASRError('Failed to get all subject schemas (status code: %s)' % resp.status_code) meta = SubjectHeaderBot.extract_metadata(resp)[subject_name] buff = StringIO.StringIO(resp.content) schemas = [] version = 1 for schema_str in buff: ras = RegisteredAvroSchema() ras.schema_str = schema_str.strip() ras.gv_dict[subject_name] = version if ras.sha256_id != meta.sha256_id_list[version - 1]: raise TASRError('Generated SHA256 ID did not match passed ID.') schemas.append(ras) version += 1 buff.close() return schemas
def test_compatible_with_self(self): '''A schema should always be back-compatible with itself.''' ras = RegisteredAvroSchema() ras.schema_str = self.schema_str self.assertTrue(MasterAvroSchema([ras, ]).is_compatible(ras), 'expected schema to be back-compatible with self') # and confirm that it works with the convenience method self.assertTrue(ras.back_compatible_with(ras), 'expected schema to be back-compatible with self')
def test_compatible_with_nullable_field_added(self): '''Adding a nullable field (with default null) should be fine.''' ras = RegisteredAvroSchema() ras.schema_str = self.schema_str # create schema with extra field added new_ras = RegisteredAvroSchema() new_ras.schema_str = self.get_schema_permutation(self.schema_str) self.assertTrue(MasterAvroSchema([ras, ]).is_compatible(new_ras), 'expected new schema to be back-compatible') # and confirm that it works with the convenience method self.assertTrue(new_ras.back_compatible_with(ras), 'expected new schema to be back-compatible')
def test_compatible_with_nullable_field_removed(self): '''Removing a nullable field (with default null) should be fine.''' ras = RegisteredAvroSchema() ras.schema_str = self.schema_str # create schema with extra field added new_ras = RegisteredAvroSchema() new_ras.schema_str = self.get_schema_permutation(self.schema_str) # we reverse the order, using "new" as the pre-existing schema self.assertTrue(MasterAvroSchema([new_ras, ]).is_compatible(ras), 'expected schema to be back-compatible') # and confirm that it works with the convenience method self.assertTrue(ras.back_compatible_with(new_ras), 'expected schema to be back-compatible')
def test_compatible_with_required_field_made_null_then_removed(self): '''Converting a required field to a nullable one is fine, and so is removing a nullable field. We test the whole sequence here.''' ras = RegisteredAvroSchema() ras.schema_str = self.schema_str # create first schema with extra, non-nullable field added jd = json.loads(self.schema_str) non_nullable_field_dict = {"name": "gold__extra", "type": "string"} jd['fields'].append(non_nullable_field_dict) first_ras = RegisteredAvroSchema() first_ras.schema_str = json.dumps(jd) # now create second schema, with the extra field made nullable second_ras = RegisteredAvroSchema() second_ras.schema_str = self.get_schema_permutation(self.schema_str, "gold__extra", "string") # the base ras is the third (newest) schema in the sequence mas = MasterAvroSchema([first_ras, second_ras]) self.assertTrue(mas.is_compatible(ras), 'expected new schema to be back-compatible') # and confirm that it works with the convenience method self.assertTrue(ras.back_compatible_with([first_ras, second_ras]), 'expected new schema to be back-compatible') # make sure that reversing the order of the first and second fails try: MasterAvroSchema([second_ras, first_ras]) self.fail('should have raised a ValueError as this order is bad') except ValueError: pass # ensure that using the convenience method avoidf the raise self.assertFalse(ras.back_compatible_with([second_ras, first_ras]), 'expected new schema to NOT be back-compatible')
def test_required_map_field_is_self_compatible(self): '''Check that schemas with a required map field type are compatible.''' jd = json.loads(self.schema_str) req_map_field_dict = {"name": "extra", "type": "map", "values": "string"} jd['fields'].append(req_map_field_dict) ras = RegisteredAvroSchema() ras.schema_str = json.dumps(jd) self.assertTrue(MasterAvroSchema([ras, ]).is_compatible(ras), 'expected schema to be back-compatible with self') # and confirm that it works with the convenience method self.assertTrue(ras.back_compatible_with(ras), 'expected schema to be back-compatible with self')
def test_not_compatible_with_non_nullable_field_added(self): '''Adding a non-nullable field is not allowed.''' ras = RegisteredAvroSchema() ras.schema_str = self.schema_str # create schema with extra field added jd = json.loads(self.schema_str) non_nullable_field_dict = {"name": "gold__extra", "type": "string"} jd['fields'].append(non_nullable_field_dict) new_ras = RegisteredAvroSchema() new_ras.schema_str = json.dumps(jd) self.assertFalse(MasterAvroSchema([ras, ]).is_compatible(new_ras), 'expected new schema to NOT be back-compatible') # and confirm that it works with the convenience method self.assertFalse(new_ras.back_compatible_with(ras), 'expected new schema to be back-compatible')
def reg_schema_from_url(url, method='GET', data=None, headers=None, timeout=TIMEOUT, err_404='No such object.'): '''A generic method to call a URL and transform the reply into a RegisteredSchema object. Most of the API calls can use this skeleton. ''' schema_str = None resp = None if headers == None: headers = {'Accept': 'application/json', } elif isinstance(headers, dict): headers['Accept'] = 'application/json' try: if method.upper() == 'GET': resp = requests.get(url, timeout=timeout) schema_str = resp.content elif method.upper() == 'POST': resp = requests.post(url, data=data, headers=headers, timeout=timeout) schema_str = resp.content elif method.upper() == 'PUT': resp = requests.put(url, data=data, headers=headers, timeout=timeout) schema_str = resp.content # check for error cases if resp == None: raise TASRError('Timeout for request to %s' % url) if 404 == resp.status_code: raise TASRError(err_404) if 409 == resp.status_code: raise TASRError(resp.content) if not resp.status_code in [200, 201]: raise TASRError('Failed request to %s (status code: %s)' % (url, resp.status_code)) # OK - so construct the RS and return it ras = RegisteredAvroSchema() ras.schema_str = schema_str ras.created = True if resp.status_code == 201 else False schema_meta = SchemaHeaderBot.extract_metadata(resp) if schema_str and not schema_meta.sha256_id == ras.sha256_id: raise TASRError('Schema was modified in transit.') ras.update_from_schema_metadata(schema_meta) return ras except Exception as exc: raise TASRError(exc)
def test_not_compatible_with_field_type_change(self): '''Changing the type of a field is not allowed.''' # create schema with a string field added jd = json.loads(self.schema_str) string_field_dict = {"name": "gold__extra", "type": "string"} jd['fields'].append(string_field_dict) str_ras = RegisteredAvroSchema() str_ras.schema_str = json.dumps(jd) # create a new schema where the field is an int type jd2 = json.loads(self.schema_str) int_field_dict = {"name": "gold__extra", "type": "int"} jd2['fields'].append(int_field_dict) int_ras = RegisteredAvroSchema() int_ras.schema_str = json.dumps(jd2) self.assertFalse(MasterAvroSchema([str_ras, ]).is_compatible(int_ras), 'expected schema to NOT be back-compatible') self.assertFalse(int_ras.back_compatible_with(str_ras), 'expected schema to NOT be back-compatible') # and test the reverse case as well self.assertFalse(MasterAvroSchema([int_ras, ]).is_compatible(str_ras), 'expected schema to NOT be back-compatible') self.assertFalse(str_ras.back_compatible_with(int_ras), 'expected schema to NOT be back-compatible')
def schema_for_schema_str(schema_str, object_on_miss=False, host=TASR_HOST, port=TASR_PORT, timeout=TIMEOUT): ''' POST /tasr/schema In essence this is very similar to the schema_for_id_str, but with the calculation of the ID string being moved to the server. That is, the client POSTs the schema JSON itself, the server canonicalizes it, then calculates the SHA256-based ID string for what was sent, then looks for a matching schema based on that ID string. This allows clients that do not know how to canonicalize or hash the schemas to find the metadata (is it registered, what version does it have for a topic) with what they have. A RegisteredSchema object is returned if the schema string POSTed has been registered for one or more topics. If the schema string POSTed has yet to be registered for a topic and the object_on_miss flag is True, a RegisteredSchema calculated for the POSTed schema string is returned (it will have no topic-versions as there are none). This provides an easy way for a client to get the ID strings to use for subsequent requests. If the object_on_miss flag is False (the default), then a request for a previously unregistered schema will raise a TASRError. ''' url = 'http://%s:%s/tasr/schema' % (host, port) headers = {'content-type': 'application/json; charset=utf8', } resp = requests.post(url, data=schema_str, headers=headers, timeout=timeout) if resp == None: raise TASRError('Timeout for request to %s' % url) if 200 == resp.status_code: # success -- return a normal reg schema ras = RegisteredAvroSchema() ras.schema_str = resp.context schema_meta = SchemaHeaderBot.extract_metadata(resp) ras.update_from_schema_metadata(schema_meta) return ras elif 404 == resp.status_code and object_on_miss: ras = RegisteredAvroSchema() ras.schema_str = schema_str schema_meta = SchemaHeaderBot.extract_metadata(resp) ras.update_from_schema_metadata(schema_meta) return ras raise TASRError('Schema not registered to any topics.')
def test_not_compatible_with_nullable_field_type_change(self): '''Changing the type of a field is not allowed.''' # create schema with a string field added str_ras = RegisteredAvroSchema() str_ras.schema_str = self.get_schema_permutation(self.schema_str, "gold__extra", "string") # create a new schema where the field is a nullable int type int_ras = RegisteredAvroSchema() int_ras.schema_str = self.get_schema_permutation(self.schema_str, "gold__extra", "int") self.assertFalse(MasterAvroSchema([str_ras, ]).is_compatible(int_ras), 'expected schema to NOT be back-compatible') self.assertFalse(int_ras.back_compatible_with(str_ras), 'expected schema to NOT be back-compatible') # and test the reverse case as well self.assertFalse(MasterAvroSchema([int_ras, ]).is_compatible(str_ras), 'expected schema to NOT be back-compatible') self.assertFalse(str_ras.back_compatible_with(int_ras), 'expected schema to NOT be back-compatible')
def test_compatible_with_non_nullable_field_removed(self): '''Removing a non-nullable field is OK -- treated as converting it to a nullable field with a default null, then removing that field.''' ras = RegisteredAvroSchema() ras.schema_str = self.schema_str # create schema with extra field added jd = json.loads(self.schema_str) non_nullable_field_dict = {"name": "gold__extra", "type": "string"} jd['fields'].append(non_nullable_field_dict) new_ras = RegisteredAvroSchema() new_ras.schema_str = json.dumps(jd) self.assertTrue(MasterAvroSchema([new_ras, ]).is_compatible(ras), 'expected schema to be back-compatible') self.assertTrue(ras.back_compatible_with(new_ras), 'expected schema to be back-compatible') # make sure the reverse order fails self.assertFalse(MasterAvroSchema([ras, ]).is_compatible(new_ras), 'expected schema to NOT be back-compatible') self.assertFalse(new_ras.back_compatible_with(ras), 'expected schema to NOT be back-compatible')
def test_set_schema_str(self): ras = RegisteredAvroSchema() ras.schema_str = self.schema_str self.assertEqual(self.expect_sha256_id, ras.sha256_id, 'unexpected ID')