def test_store_in_not_dict(): blueprint = Blueprint() msg = "'store_in' must be a dict, not a 'int'" with pytest.raises(ValidationError) as e: blueprint.add_item({'name': 'foo', 'store_in': 42}) assert msg in str(e.value)
def test_unexisting_parent(): blueprint = Blueprint() msg = "Parent 'foo' does not exist." with pytest.raises(ValidationError) as e: blueprint.add_item({'name': 'test', 'parent': 'foo'}) assert msg in str(e.value)
def test_fields_not_dict(): blueprint = Blueprint() msg = "Fields must be a dict, not a 'int'" with pytest.raises(ValidationError) as e: blueprint.add_item({'name': 'foo', 'table': 'bar', 'fields': 42}) assert msg in str(e.value)
def test_item_generate(): blueprint = Blueprint() blueprint.add_item({ 'name': 'foo', 'table': 'test', 'fields': { 'a': { 'generator': 'Text' }, 'b': { 'generator': 'Integer' } } }) item = blueprint.items['foo'] buffer = Buffer(blueprint) item.generate(buffer, 10) assert len(buffer.buffers['foo']) == 10 for obj in buffer.buffers['foo']: assert isinstance(obj.a, str) assert isinstance(obj.b, int)
def test_count_valid(): blueprint = Blueprint() blueprint.add_item({'name': 'test1', 'table': 'bar', 'count': 42}) assert blueprint.items['test1'].count.number == 42 assert blueprint.items['test1'].count() == 42 blueprint.add_item({ 'name': 'test2', 'table': 'bar', 'count': { 'min': 10, 'max': 100 } }) assert blueprint.items['test2'].count.number is None assert blueprint.items['test2'].count.min == 10 assert blueprint.items['test2'].count.max == 100 assert 10 <= blueprint.items['test2'].count() <= 100 blueprint.add_item({'name': 'foo', 'table': 'bar'}) blueprint.add_item({ 'name': 'test3', 'table': 'bar', 'count': { 'number': 42, 'by': 'foo' } }) assert blueprint.items['test3'].count.number == 42 assert blueprint.items['test3'].count.by == 'foo'
def test_field_value(): blueprint = Blueprint() blueprint.add_item({ 'name': 'foo', 'table': 'bar', 'fields': { 'a': None, 'b': 42, 'c': '$foo', 'd': '' } }) item = blueprint.items['foo'] assert isinstance(item.fields['a'], generators.Value) assert item.fields['a'].value is None assert isinstance(item.fields['b'], generators.Value) assert item.fields['b'].value == 42 assert isinstance(item.fields['c'], generators.Value) assert isinstance(item.fields['c'].value, ValueExpression) assert item.fields['c'].value.var == 'foo' assert isinstance(item.fields['d'], generators.Value) assert item.fields['d'].value == ''
def test_with_extra_keys(): blueprint = Blueprint() msg = ("Unknown key(s) 'foo'. Possible keys are 'name, parent, " "table, count, fields, store_in'.") with pytest.raises(ValidationError) as e: blueprint.add_item({'name': 'foo', 'table': 'bar', 'foo': 'bar'}) assert msg in str(e.value)
def test_name_and_table(): blueprint = Blueprint() blueprint.add_item({'name': 'foo', 'table': 'bar'}) item = blueprint.items['foo'] assert item.name == 'foo' assert item.table == 'bar' assert list(item.fields.keys()) == ['id']
def test_write_empty_buffer(mocker): blueprint = Blueprint(backend=Backend()) blueprint.add_item({'name': 'foo', 'table': 'test', 'fields': {'a': 42}}) item = blueprint.items['foo'] mocker.patch.object(blueprint.backend, 'write') buffer = Buffer(blueprint) # the buffer for the item is empty buffer.write(item) assert not blueprint.backend.write.called
def test_field_non_existing_generator(): blueprint = Blueprint() msg = "Item 'foo', field 'a': Generator 'Foo' does not exist." with pytest.raises(ValidationError) as e: blueprint.add_item({ 'name': 'foo', 'table': 'bar', 'fields': { 'a': { 'generator': 'Foo' } } }) assert msg in str(e.value)
def test_item_preprocess(mocker): columns = ('id', 'firstname', 'lastname', 'birth', 'gender') existing = ((1, 'Homer', 'Simpson', date(1956, 6, 18), 'M'), (2, 'Marge', 'Simpson', date(1959, 6, 29), 'F'), (3, 'Bart', 'Simpson', date(1981, 4, 1), 'M'), (4, 'Lisa', 'Simpson', date(1984, 5, 9), 'F'), (5, 'Maggie', 'Simpson', date(1988, 11, 7), 'F')) class DummyBackend(Backend): def select(self, table, fields, **kwargs): for row in existing: yield tuple(row[columns.index(field)] for field in fields) blueprint = Blueprint(backend=mocker.Mock(wraps=DummyBackend())) item = Item(blueprint, 'person', 'test') item.add_field('id', 'Integer', unique=True) item.add_field('firstname', 'Text', unique=['lastname', 'birth']) item.add_field('lastname', 'Text') item.add_field('birth', 'Date') item.add_field('gender', 'Choices', choices=['M', 'F']) # add a unique field, but which isn't present in the db (shadow) item.add_field('parent', 'Choices', shadow=True, choices='$parents', unique=['birth']) item.preprocess() assert blueprint.backend.select.call_args == mocker.call( 'test', ['id', 'firstname', 'lastname', 'birth']) assert 1 in item.fields['id'].seen assert 5 in item.fields['id'].seen assert 6 not in item.fields['id'].seen seen = item.fields['firstname'].seen assert ('Homer', 'Simpson', date(1956, 6, 18)) in seen assert ('Lisa', 'Simpson', date(1984, 5, 9)) in seen assert ('Bart', 'Simpson', date(2016, 10, 9)) not in seen # add an item using the same table item2 = Item(blueprint, 'person2', 'test') item2.add_field('firstname', 'Text', unique=['lastname', 'birth']) item2.add_field('lastname', 'Text') # for this item, the date is unique item2.add_field('birth', 'Date', unique=True) blueprint.backend.select.reset_mock() item2.preprocess() # we should have queried only the birth assert blueprint.backend.select.call_args == mocker.call('test', ['birth']) seen = item2.fields['birth'].seen assert date(1956, 6, 18) in seen # we should use the same filter than for person seen = item2.fields['firstname'].seen assert ('Homer', 'Simpson', date(1956, 6, 18)) in seen
def test_field_without_generator(): blueprint = Blueprint() msg = ("Field 'a' in item 'foo' must either be a value, or " "a dict with a 'generator' key.") with pytest.raises(ValidationError) as e: blueprint.add_item({ 'name': 'foo', 'table': 'bar', 'fields': { 'a': { 'foo': 'bar' } } }) assert msg in str(e.value)
def test_field_extra_params(): blueprint = Blueprint() msg = ("Item 'foo', field 'a': Got extra param(s) for generator " "'Integer': foo") with pytest.raises(ValidationError) as e: blueprint.add_item({ 'name': 'foo', 'table': 'bar', 'fields': { 'a': { 'generator': 'Integer', 'foo': 42 } } }) assert msg in str(e.value)
def test_field_with_generator(): blueprint = Blueprint() blueprint.add_item({ 'name': 'foo', 'table': 'bar', 'fields': { 'a': { 'generator': 'Integer', 'max': 42 } } }) item = blueprint.items['foo'] assert isinstance(item.fields['a'], generators.Integer) assert item.fields['a'].max == 42
def test_item_generate_this_var(mocker): blueprint = Blueprint() blueprint.add_item({'name': 'foo', 'table': 'test'}) item = blueprint.items['foo'] call_count = {'count': 0} def _generate(self): call_count['count'] += 1 assert self.blueprint.vars['this'] == self return item.namedtuple(id=1) mocker.patch.object(ItemFactory, 'generate', _generate) buffer = Buffer(blueprint) item.generate(buffer, 10) assert call_count['count'] == 10
def test_store_in_non_existing_item(): blueprint = Blueprint() msg = ("Error in 'store_in' section in item 'foo': The " "item 'toto' does not exist.") with pytest.raises(ValidationError) as e: blueprint.add_item({ 'name': 'foo', 'table': 'bar', 'count': { 'by': 'toto' }, 'store_in': { 'this.toto.foos': '$this.id' } }) assert msg in str(e.value)
def test_generate_dependencies_ancestors(): class DummyBackend(Backend): def write(self, item, objs): return range(len(objs)) blueprint = Blueprint(backend=DummyBackend()) blueprint.add_item({'name': 'foo', 'table': 'test'}) blueprint.add_item({'name': 'foo2', 'parent': 'foo'}) blueprint.add_item({'name': 'foo3', 'parent': 'foo2'}) blueprint.add_item({ 'name': 'bar', 'table': 'test', 'count': { 'number': 2, 'by': 'foo' }, 'store_in': { 'bars': '$this' }, 'fields': { 'parent_id': '$this.foo.id' } }) buffer = Buffer(blueprint) foo2 = blueprint.items['foo2'] foo3 = blueprint.items['foo3'] foo3.generate(buffer, 2) foo2.generate(buffer, 2) assert len(buffer.buffers['foo2']) == 2 assert len(buffer.buffers['foo3']) == 2 assert len(buffer.buffers['bar']) == 0 assert len(blueprint.vars['bars']) == 0 buffer.write(foo3) assert len(buffer.buffers['foo2']) == 2 assert len(buffer.buffers['foo3']) == 0 assert len(buffer.buffers['bar']) == 0 assert len(blueprint.vars['bars']) == 4 buffer.write(foo2) assert len(buffer.buffers['foo2']) == 0 assert len(buffer.buffers['foo3']) == 0 assert len(buffer.buffers['bar']) == 0 assert len(blueprint.vars['bars']) == 8
def test_blueprint_generate(mocker): import random random.seed(42) blueprint = Blueprint() blueprint.add_item({'name': 'foo', 'table': 'test', 'count': 10}) blueprint.add_item({ 'name': 'bar', 'table': 'test', 'count': { 'number': 20, 'by': 'foo' } }) blueprint.add_item({ 'name': 'lol', 'table': 'test', 'count': { 'min': 10, 'max': 20 } }) foo = blueprint.items['foo'] bar = blueprint.items['bar'] lol = blueprint.items['lol'] mocker.patch.object(foo, 'generate') mocker.patch.object(bar, 'generate') mocker.patch.object(lol, 'generate') buffer = mocker.Mock(wraps=Buffer(blueprint)) mocker.patch('populous.blueprint.Buffer', return_value=buffer) blueprint.generate() assert foo.generate.call_args == mocker.call(buffer, 10) assert bar.generate.called is False if PY2: assert lol.generate.call_args == mocker.call(buffer, 17) else: assert lol.generate.call_args == mocker.call(buffer, 20) assert buffer.flush.called is True
def test_write_buffer(mocker): class DummyBackend(Backend): def write(self, item, objs): return range(len(objs)) blueprint = Blueprint(backend=DummyBackend()) blueprint.add_item({'name': 'foo', 'table': 'test', 'fields': {'a': 42}}) item = blueprint.items['foo'] mocker.patch.object(item, 'store_final_values') mocker.patch.object(item, 'generate_dependencies') buffer = Buffer(blueprint, maxlen=10) item.generate(buffer, 10) objs = tuple(item.namedtuple(id=x, a=42) for x in range(10)) assert len(buffer.buffers['foo']) == 0 assert item.store_final_values.call_args == mocker.call(objs) assert item.generate_dependencies.call_args == mocker.call(buffer, objs)
def test_flush_buffer(mocker): blueprint = Blueprint(backend=mocker.MagicMock()) blueprint.add_item({'name': 'foo', 'table': 'test'}) blueprint.add_item({'name': 'bar', 'table': 'test'}) buffer = Buffer(blueprint) blueprint.items['foo'].generate(buffer, 5) blueprint.items['bar'].generate(buffer, 4) assert len(buffer.buffers['foo']) == 5 assert len(buffer.buffers['bar']) == 4 buffer.flush() assert len(buffer.buffers['foo']) == 0 assert len(buffer.buffers['bar']) == 0 assert blueprint.backend.write.call_count == 2 assert (blueprint.backend.write.call_args_list[0] == mocker.call( blueprint.items['foo'], ((), (), (), (), ()))) assert (blueprint.backend.write.call_args_list[1] == mocker.call( blueprint.items['bar'], ((), (), (), ())))
def test_blueprint_preprocess(mocker): blueprint = Blueprint() blueprint.add_item({'name': 'foo', 'table': 'test'}) blueprint.add_item({'name': 'bar', 'table': 'test'}) foo = mocker.Mock(wraps=blueprint.items['foo']) bar = mocker.Mock(wraps=blueprint.items['bar']) blueprint.items['foo'] = foo blueprint.items['bar'] = bar blueprint.preprocess() assert foo.preprocess.called is True assert bar.preprocess.called is True
def test_store_values(): class DummyBackend(Backend): def write(self, item, objs): return range(len(objs)) blueprint = Blueprint(backend=DummyBackend()) blueprint.add_item({ 'name': 'foo', 'table': 'test', 'store_in': { 'foos': '$this' } }) blueprint.add_item({ 'name': 'bar', 'table': 'test2', 'count': { 'by': 'foo', 'number': 2 }, 'store_in': { 'this.foo.bar_ids': '$this.id' } }) buffer = Buffer(blueprint) blueprint.items['foo'].generate(buffer, 10) buffer.flush() assert list(foo.id for foo in blueprint.vars['foos']) == list(range(10)) ids = iter(range(20)) for foo in blueprint.vars['foos']: assert foo.bar_ids == [next(ids), next(ids)]
def test_count_var(): blueprint = Blueprint() blueprint.add_item({'name': 'test1', 'table': 'bar', 'count': '$foo'}) blueprint.vars['foo'] = 42 assert blueprint.items['test1'].count() == 42 blueprint.add_item({ 'name': 'test2', 'table': 'bar', 'count': { 'min': '$min', 'max': '$max' } }) blueprint.vars['min'] = 1 blueprint.vars['max'] = 10 assert 1 <= blueprint.items['test2'].count() <= 10
def test_parent_with_same_name(): blueprint = Blueprint() blueprint.add_item({'name': 'foo', 'table': 'bar'}) blueprint.add_item({'name': 'foo', 'table': 'test'}) assert len(blueprint.items) == 1 assert blueprint.items['foo'].table == 'test'
def test_add_var(): blueprint = Blueprint() blueprint.add_var('foo', 42) blueprint.add_var('bar', None) assert blueprint.vars['foo'] == 42 assert blueprint.vars['bar'] is None
def test_two_parents(): blueprint = Blueprint() blueprint.add_item({'name': 'foo', 'table': 'bar'}) msg = "Re-defining item 'foo' while setting 'bar' as parent is ambiguous" with pytest.raises(ValidationError) as e: blueprint.add_item({'name': 'foo', 'parent': 'bar'}) assert msg in str(e.value)
def test_description_wrong_type(): blueprint = Blueprint() msg = "A blueprint item must be a dict, not a 'NoneType'" with pytest.raises(ValidationError) as e: blueprint.add_item(None) assert msg in str(e.value) msg = "A blueprint item must be a dict, not a 'int'" with pytest.raises(ValidationError) as e: blueprint.add_item(42) assert msg in str(e.value) msg = "A blueprint item must be a dict, not a 'list'" with pytest.raises(ValidationError) as e: blueprint.add_item([]) assert msg in str(e.value)
def test_flush_buffer_with_dependencies(mocker): class DummyBackend(Backend): def write(self, item, objs): return range(len(objs)) blueprint = Blueprint(backend=mocker.Mock(wraps=DummyBackend())) blueprint.add_item({'name': 'foo', 'table': 'test'}) blueprint.add_item({ 'name': 'bar', 'table': 'test', 'count': { 'number': 1, 'by': 'foo' } }) blueprint.add_item({ 'name': 'lol', 'table': 'test', 'count': { 'number': 1, 'by': 'bar' } }) buffer = Buffer(blueprint) blueprint.items['foo'].generate(buffer, 5) assert len(buffer.buffers['foo']) == 5 assert len(buffer.buffers['bar']) == 0 assert len(buffer.buffers['lol']) == 0 buffer.flush() assert len(buffer.buffers['foo']) == 0 assert len(buffer.buffers['bar']) == 0 assert len(buffer.buffers['lol']) == 0 assert blueprint.backend.write.call_count == 3 assert (blueprint.backend.write.call_args_list[0] == mocker.call( blueprint.items['foo'], ((), (), (), (), ()))) assert (blueprint.backend.write.call_args_list[1] == mocker.call( blueprint.items['bar'], ((), (), (), (), ()))) assert (blueprint.backend.write.call_args_list[2] == mocker.call( blueprint.items['lol'], ((), (), (), (), ())))
def test_required(): blueprint = Blueprint() msg = "Items without a parent must have a name" with pytest.raises(ValidationError) as e: blueprint.add_item({}) assert msg in str(e.value) msg = "Item 'foo' does not have a table." with pytest.raises(ValidationError) as e: blueprint.add_item({'name': 'foo'}) assert msg in str(e.value)
def test_store_values(): class DummyBackend(Backend): counters = {} def write(self, item, objs): counter = self.counters.setdefault(item.name, count()) return [next(counter) for _ in objs] blueprint = Blueprint(backend=DummyBackend()) blueprint.add_item({ 'name': 'foo', 'table': 'test', 'store_in': { 'foos': '$this' } }) blueprint.add_item({ 'name': 'bar', 'table': 'test2', 'count': { 'by': 'foo', 'number': 2 }, 'store_in': { 'this.foo.bars': '$this' }, 'fields': { 'num': '$(this.foo.bars|length)' } }) buffer = Buffer(blueprint, maxlen=15) blueprint.items['foo'].generate(buffer, 10) # the values have been generated, but empty ids have been stored assert [foo.id for foo in blueprint.vars['foos']] == [None] * 10 buffer.write(blueprint.items['foo']) # the stored ids have now been replaced assert [foo.id for foo in blueprint.vars['foos']] == list(range(10)) # each foo object contains the corresponding bars, each bar has # an id & a number corresponding to the number of 'bars' in the # current 'foo' at the time of generation ids = iter(range(20)) for foo in blueprint.vars['foos']: assert [(bar.id, bar.num) for bar in foo.bars] == [ (next(ids), 0), (next(ids), 1), ]