def test_resumes_currently_syncing_stream(self):
selected_entry_a = CatalogEntry(tap_stream_id='a',
schema=Schema(),
metadata=[{
'metadata': {
'selected': True
},
'breadcrumb': []
}])
selected_entry_c = CatalogEntry(tap_stream_id='c',
schema=Schema(),
metadata=[{
'metadata': {
'selected': True
},
'breadcrumb': []
}])
catalog = Catalog([
selected_entry_a,
CatalogEntry(tap_stream_id='b', schema=Schema(), metadata=[]),
selected_entry_c
])
state = {'currently_syncing': 'c'}
selected_streams = catalog.get_selected_streams(state)
self.assertEquals([e for e in selected_streams][0], selected_entry_c)
def test_object_from_dict_with_defaults(self):
schema = Schema.from_dict(self.object_dict, inclusion='automatic')
self.assertEquals('whatever',
schema.inclusion,
msg='The schema value should override the default')
self.assertEquals('automatic', schema.properties['a_string'].inclusion)
self.assertEquals('automatic',
schema.properties['an_array'].items.inclusion)
def test_one_selected_stream(self):
selected_entry = CatalogEntry(tap_stream_id='a',
schema=Schema(),
metadata=[{
'metadata': {
'selected': True
},
'breadcrumb': []
}])
catalog = Catalog([
selected_entry,
CatalogEntry(tap_stream_id='b', schema=Schema(), metadata=[]),
CatalogEntry(tap_stream_id='c', schema=Schema(), metadata=[])
])
state = {}
selected_streams = catalog.get_selected_streams(state)
self.assertEquals([e for e in selected_streams], [selected_entry])
def test_object_from_dict(self):
self.assertEquals(self.object_obj, Schema.from_dict(self.object_dict))
def test_array_from_dict(self):
self.assertEquals(self.array_obj, Schema.from_dict(self.array_dict))
def test_integer_from_dict(self):
self.assertEquals(self.integer_obj,
Schema.from_dict(self.integer_dict))
def test_string_from_dict(self):
self.assertEquals(self.string_obj, Schema.from_dict(self.string_dict))
class TestSchema(unittest.TestCase):
# Raw data structures for several schema types
string_dict = {'type': 'string', 'maxLength': 32}
integer_dict = {'type': 'integer', 'maximum': 1000000}
array_dict = {'type': 'array', 'items': integer_dict}
object_dict = {
'type': 'object',
'properties': {
'a_string': string_dict,
'an_array': array_dict
},
'inclusion': 'whatever',
'additionalProperties': True,
}
# Schema object forms of the same schemas as above
string_obj = Schema(type='string', maxLength=32)
integer_obj = Schema(type='integer', maximum=1000000)
array_obj = Schema(type='array', items=integer_obj)
object_obj = Schema(type='object',
properties={
'a_string': string_obj,
'an_array': array_obj
},
inclusion='whatever',
additionalProperties=True)
def test_string_to_dict(self):
self.assertEquals(self.string_dict, self.string_obj.to_dict())
def test_integer_to_dict(self):
self.assertEquals(self.integer_dict, self.integer_obj.to_dict())
def test_array_to_dict(self):
self.assertEquals(self.array_dict, self.array_obj.to_dict())
def test_object_to_dict(self):
self.assertEquals(self.object_dict, self.object_obj.to_dict())
def test_string_from_dict(self):
self.assertEquals(self.string_obj, Schema.from_dict(self.string_dict))
def test_integer_from_dict(self):
self.assertEquals(self.integer_obj,
Schema.from_dict(self.integer_dict))
def test_array_from_dict(self):
self.assertEquals(self.array_obj, Schema.from_dict(self.array_dict))
def test_object_from_dict(self):
self.assertEquals(self.object_obj, Schema.from_dict(self.object_dict))
def test_repr_atomic(self):
self.assertEquals(self.string_obj, eval(repr(self.string_obj)))
def test_repr_recursive(self):
self.assertEquals(self.object_obj, eval(repr(self.object_obj)))
def test_object_from_dict_with_defaults(self):
schema = Schema.from_dict(self.object_dict, inclusion='automatic')
self.assertEquals('whatever',
schema.inclusion,
msg='The schema value should override the default')
self.assertEquals('automatic', schema.properties['a_string'].inclusion)
self.assertEquals('automatic',
schema.properties['an_array'].items.inclusion)
class TestToDictAndFromDict(unittest.TestCase):
dict_form = {
'streams': [{
'stream':
'users',
'tap_stream_id':
'prod_users',
'stream_alias':
'users_alias',
'database_name':
'prod',
'table_name':
'users',
'schema': {
'type': 'object',
'selected': True,
'properties': {
'id': {
'type': 'integer',
'selected': True
},
'name': {
'type': 'string',
'selected': True
}
}
},
'metadata': [
{
'metadata': {
'metadata-key': 'metadata-value'
},
'breadcrumb': [
'properties',
'name',
],
},
],
}, {
'stream': 'orders',
'tap_stream_id': 'prod_orders',
'database_name': 'prod',
'table_name': 'orders',
'schema': {
'type': 'object',
'selected': True,
'properties': {
'id': {
'type': 'integer',
'selected': True
},
'amount': {
'type': 'number',
'selected': True
}
}
}
}]
}
obj_form = Catalog(streams=[
CatalogEntry(stream='users',
tap_stream_id='prod_users',
stream_alias='users_alias',
database='prod',
table='users',
schema=Schema(
type='object',
selected=True,
properties={
'id': Schema(type='integer', selected=True),
'name': Schema(type='string', selected=True)
}),
metadata=[{
'metadata': {
'metadata-key': 'metadata-value'
},
'breadcrumb': [
'properties',
'name',
],
}]),
CatalogEntry(stream='orders',
tap_stream_id='prod_orders',
database='prod',
table='orders',
schema=Schema(
type='object',
selected=True,
properties={
'id': Schema(type='integer', selected=True),
'amount': Schema(type='number', selected=True)
}))
])
def test_from_dict(self):
self.assertEqual(self.obj_form, Catalog.from_dict(self.dict_form))
def test_to_dict(self):
self.assertEqual(self.dict_form, self.obj_form.to_dict())
def test_write_catalog_with_streams(self):
catalog = Catalog(
[CatalogEntry(tap_stream_id='a', schema=Schema(), metadata=[])])
write_catalog(catalog)