def get_all_subject_schemas(subject_name,
host=TASR_HOST, port=TASR_PORT, timeout=TIMEOUT):
''' GET /tasr/subject/<subject name>/all_schemas
Retrieves all the (canonical) schema versions registered for a subject,
in version order, one per line in the response body. The multi-type IDs
are included in the headers for confirmation.
'''
url = ('http://%s:%s/tasr/subject/%s/all_schemas' %
(host, port, subject_name))
resp = requests.get(url, timeout=timeout)
if resp == None:
raise TASRError('Timeout for get all subject schemas request.')
if resp.status_code != 200:
raise TASRError('Failed to get all subject schemas (status code: %s)' %
resp.status_code)
meta = SubjectHeaderBot.extract_metadata(resp)[subject_name]
buff = StringIO.StringIO(resp.content)
schemas = []
version = 1
for schema_str in buff:
ras = RegisteredAvroSchema()
ras.schema_str = schema_str.strip()
ras.gv_dict[subject_name] = version
if ras.sha256_id != meta.sha256_id_list[version - 1]:
raise TASRError('Generated SHA256 ID did not match passed ID.')
schemas.append(ras)
version += 1
buff.close()
return schemas
def test_compatible_with_self(self):
'''A schema should always be back-compatible with itself.'''
ras = RegisteredAvroSchema()
ras.schema_str = self.schema_str
self.assertTrue(MasterAvroSchema([ras, ]).is_compatible(ras),
'expected schema to be back-compatible with self')
# and confirm that it works with the convenience method
self.assertTrue(ras.back_compatible_with(ras),
'expected schema to be back-compatible with self')
def test_compatible_with_nullable_field_added(self):
'''Adding a nullable field (with default null) should be fine.'''
ras = RegisteredAvroSchema()
ras.schema_str = self.schema_str
# create schema with extra field added
new_ras = RegisteredAvroSchema()
new_ras.schema_str = self.get_schema_permutation(self.schema_str)
self.assertTrue(MasterAvroSchema([ras, ]).is_compatible(new_ras),
'expected new schema to be back-compatible')
# and confirm that it works with the convenience method
self.assertTrue(new_ras.back_compatible_with(ras),
'expected new schema to be back-compatible')
def test_compatible_with_nullable_field_removed(self):
'''Removing a nullable field (with default null) should be fine.'''
ras = RegisteredAvroSchema()
ras.schema_str = self.schema_str
# create schema with extra field added
new_ras = RegisteredAvroSchema()
new_ras.schema_str = self.get_schema_permutation(self.schema_str)
# we reverse the order, using "new" as the pre-existing schema
self.assertTrue(MasterAvroSchema([new_ras, ]).is_compatible(ras),
'expected schema to be back-compatible')
# and confirm that it works with the convenience method
self.assertTrue(ras.back_compatible_with(new_ras),
'expected schema to be back-compatible')
def test_compatible_with_required_field_made_null_then_removed(self):
'''Converting a required field to a nullable one is fine, and so is
removing a nullable field. We test the whole sequence here.'''
ras = RegisteredAvroSchema()
ras.schema_str = self.schema_str
# create first schema with extra, non-nullable field added
jd = json.loads(self.schema_str)
non_nullable_field_dict = {"name": "gold__extra",
"type": "string"}
jd['fields'].append(non_nullable_field_dict)
first_ras = RegisteredAvroSchema()
first_ras.schema_str = json.dumps(jd)
# now create second schema, with the extra field made nullable
second_ras = RegisteredAvroSchema()
second_ras.schema_str = self.get_schema_permutation(self.schema_str,
"gold__extra",
"string")
# the base ras is the third (newest) schema in the sequence
mas = MasterAvroSchema([first_ras, second_ras])
self.assertTrue(mas.is_compatible(ras),
'expected new schema to be back-compatible')
# and confirm that it works with the convenience method
self.assertTrue(ras.back_compatible_with([first_ras, second_ras]),
'expected new schema to be back-compatible')
# make sure that reversing the order of the first and second fails
try:
MasterAvroSchema([second_ras, first_ras])
self.fail('should have raised a ValueError as this order is bad')
except ValueError:
pass
# ensure that using the convenience method avoidf the raise
self.assertFalse(ras.back_compatible_with([second_ras, first_ras]),
'expected new schema to NOT be back-compatible')
def test_required_map_field_is_self_compatible(self):
'''Check that schemas with a required map field type are compatible.'''
jd = json.loads(self.schema_str)
req_map_field_dict = {"name": "extra",
"type": "map",
"values": "string"}
jd['fields'].append(req_map_field_dict)
ras = RegisteredAvroSchema()
ras.schema_str = json.dumps(jd)
self.assertTrue(MasterAvroSchema([ras, ]).is_compatible(ras),
'expected schema to be back-compatible with self')
# and confirm that it works with the convenience method
self.assertTrue(ras.back_compatible_with(ras),
'expected schema to be back-compatible with self')
def test_not_compatible_with_non_nullable_field_added(self):
'''Adding a non-nullable field is not allowed.'''
ras = RegisteredAvroSchema()
ras.schema_str = self.schema_str
# create schema with extra field added
jd = json.loads(self.schema_str)
non_nullable_field_dict = {"name": "gold__extra",
"type": "string"}
jd['fields'].append(non_nullable_field_dict)
new_ras = RegisteredAvroSchema()
new_ras.schema_str = json.dumps(jd)
self.assertFalse(MasterAvroSchema([ras, ]).is_compatible(new_ras),
'expected new schema to NOT be back-compatible')
# and confirm that it works with the convenience method
self.assertFalse(new_ras.back_compatible_with(ras),
'expected new schema to be back-compatible')
def reg_schema_from_url(url, method='GET', data=None, headers=None,
timeout=TIMEOUT, err_404='No such object.'):
'''A generic method to call a URL and transform the reply into a
RegisteredSchema object. Most of the API calls can use this skeleton.
'''
schema_str = None
resp = None
if headers == None:
headers = {'Accept': 'application/json', }
elif isinstance(headers, dict):
headers['Accept'] = 'application/json'
try:
if method.upper() == 'GET':
resp = requests.get(url, timeout=timeout)
schema_str = resp.content
elif method.upper() == 'POST':
resp = requests.post(url, data=data, headers=headers,
timeout=timeout)
schema_str = resp.content
elif method.upper() == 'PUT':
resp = requests.put(url, data=data, headers=headers,
timeout=timeout)
schema_str = resp.content
# check for error cases
if resp == None:
raise TASRError('Timeout for request to %s' % url)
if 404 == resp.status_code:
raise TASRError(err_404)
if 409 == resp.status_code:
raise TASRError(resp.content)
if not resp.status_code in [200, 201]:
raise TASRError('Failed request to %s (status code: %s)' %
(url, resp.status_code))
# OK - so construct the RS and return it
ras = RegisteredAvroSchema()
ras.schema_str = schema_str
ras.created = True if resp.status_code == 201 else False
schema_meta = SchemaHeaderBot.extract_metadata(resp)
if schema_str and not schema_meta.sha256_id == ras.sha256_id:
raise TASRError('Schema was modified in transit.')
ras.update_from_schema_metadata(schema_meta)
return ras
except Exception as exc:
raise TASRError(exc)
def test_not_compatible_with_field_type_change(self):
'''Changing the type of a field is not allowed.'''
# create schema with a string field added
jd = json.loads(self.schema_str)
string_field_dict = {"name": "gold__extra",
"type": "string"}
jd['fields'].append(string_field_dict)
str_ras = RegisteredAvroSchema()
str_ras.schema_str = json.dumps(jd)
# create a new schema where the field is an int type
jd2 = json.loads(self.schema_str)
int_field_dict = {"name": "gold__extra",
"type": "int"}
jd2['fields'].append(int_field_dict)
int_ras = RegisteredAvroSchema()
int_ras.schema_str = json.dumps(jd2)
self.assertFalse(MasterAvroSchema([str_ras, ]).is_compatible(int_ras),
'expected schema to NOT be back-compatible')
self.assertFalse(int_ras.back_compatible_with(str_ras),
'expected schema to NOT be back-compatible')
# and test the reverse case as well
self.assertFalse(MasterAvroSchema([int_ras, ]).is_compatible(str_ras),
'expected schema to NOT be back-compatible')
self.assertFalse(str_ras.back_compatible_with(int_ras),
'expected schema to NOT be back-compatible')
def schema_for_schema_str(schema_str, object_on_miss=False,
host=TASR_HOST, port=TASR_PORT, timeout=TIMEOUT):
''' POST /tasr/schema
In essence this is very similar to the schema_for_id_str, but with the
calculation of the ID string being moved to the server. That is, the
client POSTs the schema JSON itself, the server canonicalizes it, then
calculates the SHA256-based ID string for what was sent, then looks for
a matching schema based on that ID string. This allows clients that do not
know how to canonicalize or hash the schemas to find the metadata (is it
registered, what version does it have for a topic) with what they have.
A RegisteredSchema object is returned if the schema string POSTed has been
registered for one or more topics.
If the schema string POSTed has yet to be registered for a topic and the
object_on_miss flag is True, a RegisteredSchema calculated for the POSTed
schema string is returned (it will have no topic-versions as there are
none). This provides an easy way for a client to get the ID strings to
use for subsequent requests.
If the object_on_miss flag is False (the default), then a request for a
previously unregistered schema will raise a TASRError.
'''
url = 'http://%s:%s/tasr/schema' % (host, port)
headers = {'content-type': 'application/json; charset=utf8', }
resp = requests.post(url, data=schema_str, headers=headers,
timeout=timeout)
if resp == None:
raise TASRError('Timeout for request to %s' % url)
if 200 == resp.status_code:
# success -- return a normal reg schema
ras = RegisteredAvroSchema()
ras.schema_str = resp.context
schema_meta = SchemaHeaderBot.extract_metadata(resp)
ras.update_from_schema_metadata(schema_meta)
return ras
elif 404 == resp.status_code and object_on_miss:
ras = RegisteredAvroSchema()
ras.schema_str = schema_str
schema_meta = SchemaHeaderBot.extract_metadata(resp)
ras.update_from_schema_metadata(schema_meta)
return ras
raise TASRError('Schema not registered to any topics.')
def test_not_compatible_with_nullable_field_type_change(self):
'''Changing the type of a field is not allowed.'''
# create schema with a string field added
str_ras = RegisteredAvroSchema()
str_ras.schema_str = self.get_schema_permutation(self.schema_str,
"gold__extra",
"string")
# create a new schema where the field is a nullable int type
int_ras = RegisteredAvroSchema()
int_ras.schema_str = self.get_schema_permutation(self.schema_str,
"gold__extra",
"int")
self.assertFalse(MasterAvroSchema([str_ras, ]).is_compatible(int_ras),
'expected schema to NOT be back-compatible')
self.assertFalse(int_ras.back_compatible_with(str_ras),
'expected schema to NOT be back-compatible')
# and test the reverse case as well
self.assertFalse(MasterAvroSchema([int_ras, ]).is_compatible(str_ras),
'expected schema to NOT be back-compatible')
self.assertFalse(str_ras.back_compatible_with(int_ras),
'expected schema to NOT be back-compatible')
def test_compatible_with_non_nullable_field_removed(self):
'''Removing a non-nullable field is OK -- treated as converting it to a
nullable field with a default null, then removing that field.'''
ras = RegisteredAvroSchema()
ras.schema_str = self.schema_str
# create schema with extra field added
jd = json.loads(self.schema_str)
non_nullable_field_dict = {"name": "gold__extra",
"type": "string"}
jd['fields'].append(non_nullable_field_dict)
new_ras = RegisteredAvroSchema()
new_ras.schema_str = json.dumps(jd)
self.assertTrue(MasterAvroSchema([new_ras, ]).is_compatible(ras),
'expected schema to be back-compatible')
self.assertTrue(ras.back_compatible_with(new_ras),
'expected schema to be back-compatible')
# make sure the reverse order fails
self.assertFalse(MasterAvroSchema([ras, ]).is_compatible(new_ras),
'expected schema to NOT be back-compatible')
self.assertFalse(new_ras.back_compatible_with(ras),
'expected schema to NOT be back-compatible')
def test_set_schema_str(self):
ras = RegisteredAvroSchema()
ras.schema_str = self.schema_str
self.assertEqual(self.expect_sha256_id, ras.sha256_id, 'unexpected ID')
