def test_store_values(): class DummyBackend(Backend): def write(self, item, objs): return range(len(objs)) blueprint = Blueprint(backend=DummyBackend()) blueprint.add_item({ 'name': 'foo', 'table': 'test', 'store_in': { 'foos': '$this' } }) blueprint.add_item({ 'name': 'bar', 'table': 'test2', 'count': { 'by': 'foo', 'number': 2 }, 'store_in': { 'this.foo.bar_ids': '$this.id' } }) buffer = Buffer(blueprint) blueprint.items['foo'].generate(buffer, 10) buffer.flush() assert list(foo.id for foo in blueprint.vars['foos']) == list(range(10)) ids = iter(range(20)) for foo in blueprint.vars['foos']: assert foo.bar_ids == [next(ids), next(ids)]
def test_write_empty_buffer(mocker): blueprint = Blueprint(backend=Backend()) blueprint.add_item({'name': 'foo', 'table': 'test', 'fields': {'a': 42}}) item = blueprint.items['foo'] mocker.patch.object(blueprint.backend, 'write') buffer = Buffer(blueprint) # the buffer for the item is empty buffer.write(item) assert not blueprint.backend.write.called
def test_store_values(): class DummyBackend(Backend): counters = {} def write(self, item, objs): counter = self.counters.setdefault(item.name, count()) return [next(counter) for _ in objs] blueprint = Blueprint(backend=DummyBackend()) blueprint.add_item({ 'name': 'foo', 'table': 'test', 'store_in': { 'foos': '$this' } }) blueprint.add_item({ 'name': 'bar', 'table': 'test2', 'count': { 'by': 'foo', 'number': 2 }, 'store_in': { 'this.foo.bars': '$this' }, 'fields': { 'num': '$(this.foo.bars|length)' } }) buffer = Buffer(blueprint, maxlen=15) blueprint.items['foo'].generate(buffer, 10) # the values have been generated, but empty ids have been stored assert [foo.id for foo in blueprint.vars['foos']] == [None] * 10 buffer.write(blueprint.items['foo']) # the stored ids have now been replaced assert [foo.id for foo in blueprint.vars['foos']] == list(range(10)) # each foo object contains the corresponding bars, each bar has # an id & a number corresponding to the number of 'bars' in the # current 'foo' at the time of generation ids = iter(range(20)) for foo in blueprint.vars['foos']: assert [(bar.id, bar.num) for bar in foo.bars] == [ (next(ids), 0), (next(ids), 1), ]
def test_item_generate(): blueprint = Blueprint() blueprint.add_item({ 'name': 'foo', 'table': 'test', 'fields': { 'a': { 'generator': 'Text' }, 'b': { 'generator': 'Integer' } } }) item = blueprint.items['foo'] buffer = Buffer(blueprint) item.generate(buffer, 10) assert len(buffer.buffers['foo']) == 10 for obj in buffer.buffers['foo']: assert isinstance(obj.a, str) assert isinstance(obj.b, int)
def test_flush_buffer_with_dependencies(mocker): class DummyBackend(Backend): def write(self, item, objs): return range(len(objs)) blueprint = Blueprint(backend=mocker.Mock(wraps=DummyBackend())) blueprint.add_item({'name': 'foo', 'table': 'test'}) blueprint.add_item({ 'name': 'bar', 'table': 'test', 'count': { 'number': 1, 'by': 'foo' } }) blueprint.add_item({ 'name': 'lol', 'table': 'test', 'count': { 'number': 1, 'by': 'bar' } }) buffer = Buffer(blueprint) blueprint.items['foo'].generate(buffer, 5) assert len(buffer.buffers['foo']) == 5 assert len(buffer.buffers['bar']) == 0 assert len(buffer.buffers['lol']) == 0 buffer.flush() assert len(buffer.buffers['foo']) == 0 assert len(buffer.buffers['bar']) == 0 assert len(buffer.buffers['lol']) == 0 assert blueprint.backend.write.call_count == 3 assert (blueprint.backend.write.call_args_list[0] == mocker.call( blueprint.items['foo'], ((), (), (), (), ()))) assert (blueprint.backend.write.call_args_list[1] == mocker.call( blueprint.items['bar'], ((), (), (), (), ()))) assert (blueprint.backend.write.call_args_list[2] == mocker.call( blueprint.items['lol'], ((), (), (), (), ())))
def test_flush_buffer(mocker): blueprint = Blueprint(backend=mocker.MagicMock()) blueprint.add_item({'name': 'foo', 'table': 'test'}) blueprint.add_item({'name': 'bar', 'table': 'test'}) buffer = Buffer(blueprint) blueprint.items['foo'].generate(buffer, 5) blueprint.items['bar'].generate(buffer, 4) assert len(buffer.buffers['foo']) == 5 assert len(buffer.buffers['bar']) == 4 buffer.flush() assert len(buffer.buffers['foo']) == 0 assert len(buffer.buffers['bar']) == 0 assert blueprint.backend.write.call_count == 2 assert (blueprint.backend.write.call_args_list[0] == mocker.call( blueprint.items['foo'], ((), (), (), (), ()))) assert (blueprint.backend.write.call_args_list[1] == mocker.call( blueprint.items['bar'], ((), (), (), ())))
def generate(self): logger.info("Getting existing unique values...") self.preprocess() buffer = Buffer(self) logger.info("Starting generation...") for item in self.items.values(): if item.count.by: # we only create the items with no dependency, # the others will be created on the fly continue item.generate(buffer, item.count()) # write everything left in the buffer buffer.flush() logger.info("Generation done.")
def test_generate__count_with_var(): class DummyBackend(Backend): def write(self, item, objs): return range(len(objs)) blueprint = Blueprint(backend=DummyBackend()) blueprint.add_item({ 'name': 'foo', 'table': 'test', 'fields': { 'nb_bars': { 'generator': 'Integer', 'min': 1, 'max': 10 } }, 'store_in': { 'foos': '$this' } }) blueprint.add_item({ 'name': 'bar', 'table': 'test', 'count': { 'number': '$foo.nb_bars', 'by': 'foo' }, 'store_in': { 'this.foo.bars': '$this' } }) buffer = Buffer(blueprint) blueprint.items['foo'].generate(buffer, 10) buffer.flush() assert len(blueprint.vars['foos']) == 10 for foo in blueprint.vars['foos']: assert 1 <= foo.nb_bars <= 10 assert len(foo.bars) == foo.nb_bars
def test_item_generate_this_var(mocker): blueprint = Blueprint() blueprint.add_item({'name': 'foo', 'table': 'test'}) item = blueprint.items['foo'] call_count = {'count': 0} def _generate(self): call_count['count'] += 1 assert self.blueprint.vars['this'] == self return item.namedtuple(id=1) mocker.patch.object(ItemFactory, 'generate', _generate) buffer = Buffer(blueprint) item.generate(buffer, 10) assert call_count['count'] == 10
def test_blueprint_generate(mocker): import random random.seed(42) blueprint = Blueprint() blueprint.add_item({'name': 'foo', 'table': 'test', 'count': 10}) blueprint.add_item({ 'name': 'bar', 'table': 'test', 'count': { 'number': 20, 'by': 'foo' } }) blueprint.add_item({ 'name': 'lol', 'table': 'test', 'count': { 'min': 10, 'max': 20 } }) foo = blueprint.items['foo'] bar = blueprint.items['bar'] lol = blueprint.items['lol'] mocker.patch.object(foo, 'generate') mocker.patch.object(bar, 'generate') mocker.patch.object(lol, 'generate') buffer = mocker.Mock(wraps=Buffer(blueprint)) mocker.patch('populous.blueprint.Buffer', return_value=buffer) blueprint.generate() assert foo.generate.call_args == mocker.call(buffer, 10) assert bar.generate.called is False if PY2: assert lol.generate.call_args == mocker.call(buffer, 17) else: assert lol.generate.call_args == mocker.call(buffer, 20) assert buffer.flush.called is True
def test_write_buffer(mocker): class DummyBackend(Backend): def write(self, item, objs): return range(len(objs)) blueprint = Blueprint(backend=DummyBackend()) blueprint.add_item({'name': 'foo', 'table': 'test', 'fields': {'a': 42}}) item = blueprint.items['foo'] mocker.patch.object(item, 'store_final_values') mocker.patch.object(item, 'generate_dependencies') buffer = Buffer(blueprint, maxlen=10) item.generate(buffer, 10) objs = tuple(item.namedtuple(id=x, a=42) for x in range(10)) assert len(buffer.buffers['foo']) == 0 assert item.store_final_values.call_args == mocker.call(objs) assert item.generate_dependencies.call_args == mocker.call(buffer, objs)
def test_generate_dependencies_ancestors(): class DummyBackend(Backend): def write(self, item, objs): return range(len(objs)) blueprint = Blueprint(backend=DummyBackend()) blueprint.add_item({'name': 'foo', 'table': 'test'}) blueprint.add_item({'name': 'foo2', 'parent': 'foo'}) blueprint.add_item({'name': 'foo3', 'parent': 'foo2'}) blueprint.add_item({ 'name': 'bar', 'table': 'test', 'count': { 'number': 2, 'by': 'foo' }, 'store_in': { 'bars': '$this' }, 'fields': { 'parent_id': '$this.foo.id' } }) buffer = Buffer(blueprint) foo2 = blueprint.items['foo2'] foo3 = blueprint.items['foo3'] foo3.generate(buffer, 2) foo2.generate(buffer, 2) assert len(buffer.buffers['foo2']) == 2 assert len(buffer.buffers['foo3']) == 2 assert len(buffer.buffers['bar']) == 0 assert len(blueprint.vars['bars']) == 0 buffer.write(foo3) assert len(buffer.buffers['foo2']) == 2 assert len(buffer.buffers['foo3']) == 0 assert len(buffer.buffers['bar']) == 0 assert len(blueprint.vars['bars']) == 4 buffer.write(foo2) assert len(buffer.buffers['foo2']) == 0 assert len(buffer.buffers['foo3']) == 0 assert len(buffer.buffers['bar']) == 0 assert len(blueprint.vars['bars']) == 8
def test_generate_dependencies_tree(): class DummyBackend(Backend): def write(self, item, objs): return range(len(objs)) blueprint = Blueprint(backend=DummyBackend()) # definition without count blueprint.add_item({'name': 'foo', 'table': 'test'}) # root blueprint.add_item({ 'name': 'foo1', 'parent': 'foo', 'store_in': { 'foo1s': '$this' }, 'count': { 'number': 2 } }) # level 2 blueprint.add_item({ 'name': 'foo2', 'parent': 'foo1', 'store_in': { 'foo2s': '$this' }, 'count': { 'number': 2, 'by': 'foo1' } }) # level 3 blueprint.add_item({ 'name': 'foo3', 'parent': 'foo2', 'store_in': { 'foo3s': '$this' }, 'count': { 'number': 2, 'by': 'foo2' } }) # item created for each foo (root or level) blueprint.add_item({ 'name': 'bar', 'table': 'test', 'store_in': { 'bars': '$this' }, 'count': { 'number': 1, 'by': 'foo' } }) buffer = Buffer(blueprint) foo1 = blueprint.items['foo1'] foo1.generate(buffer, foo1.count()) # only the first level have been generated assert len(blueprint.vars['foo1s']) == 2 assert len(blueprint.vars['foo2s']) == 0 assert len(blueprint.vars['foo3s']) == 0 assert len(blueprint.vars['bars']) == 0 buffer.write(foo1) assert len(blueprint.vars['foo1s']) == 2 assert len(blueprint.vars['foo2s']) == 4 assert len(blueprint.vars['foo3s']) == 8 assert len(blueprint.vars['bars']) == 14
def test_generate_dependencies(): class DummyBackend(Backend): def write(self, item, objs): return range(len(objs)) blueprint = Blueprint(backend=DummyBackend()) blueprint.add_item({'name': 'foo', 'table': 'test'}) blueprint.add_item({ 'name': 'bar', 'table': 'test', 'count': { 'number': 2, 'by': 'foo' }, 'store_in': { 'bars': '$this' }, 'fields': { 'parent_id': '$this.foo.id' } }) blueprint.add_item({ 'name': 'lol', 'table': 'test', 'count': { 'min': 1, 'max': 2, 'by': 'foo' }, 'store_in': { 'lols': '$this' }, 'fields': { 'parent_id': '$this.foo.id' } }) blueprint.add_item({'name': 'abc', 'table': 'test'}) blueprint.add_item({ 'name': 'baz', 'table': 'test', 'count': { 'number': 20, 'by': 'abc' } }) buffer = Buffer(blueprint) foo = blueprint.items['foo'] foo.generate(buffer, 10) assert len(buffer.buffers['foo']) == 10 assert len(buffer.buffers['bar']) == 0 assert len(buffer.buffers['lol']) == 0 buffer.write(foo) assert len(buffer.buffers['foo']) == 0 assert len(buffer.buffers['bar']) == 0 assert len(buffer.buffers['lol']) == 0 assert len(blueprint.vars['bars']) == 20 assert 10 <= len(blueprint.vars['lols']) <= 20 def ids(): for x in range(10): yield x yield x for x, bar in zip(ids(), blueprint.vars['bars']): assert bar.parent_id == x assert bar.foo.id == x