def test_store_in_not_dict():
blueprint = Blueprint()
msg = "'store_in' must be a dict, not a 'int'"
with pytest.raises(ValidationError) as e:
blueprint.add_item({'name': 'foo', 'store_in': 42})
assert msg in str(e.value)
def test_unexisting_parent():
blueprint = Blueprint()
msg = "Parent 'foo' does not exist."
with pytest.raises(ValidationError) as e:
blueprint.add_item({'name': 'test', 'parent': 'foo'})
assert msg in str(e.value)
def test_fields_not_dict():
blueprint = Blueprint()
msg = "Fields must be a dict, not a 'int'"
with pytest.raises(ValidationError) as e:
blueprint.add_item({'name': 'foo', 'table': 'bar', 'fields': 42})
assert msg in str(e.value)
def test_item_generate():
blueprint = Blueprint()
blueprint.add_item({
'name': 'foo',
'table': 'test',
'fields': {
'a': {
'generator': 'Text'
},
'b': {
'generator': 'Integer'
}
}
})
item = blueprint.items['foo']
buffer = Buffer(blueprint)
item.generate(buffer, 10)
assert len(buffer.buffers['foo']) == 10
for obj in buffer.buffers['foo']:
assert isinstance(obj.a, str)
assert isinstance(obj.b, int)
def test_count_valid():
blueprint = Blueprint()
blueprint.add_item({'name': 'test1', 'table': 'bar', 'count': 42})
assert blueprint.items['test1'].count.number == 42
assert blueprint.items['test1'].count() == 42
blueprint.add_item({
'name': 'test2',
'table': 'bar',
'count': {
'min': 10,
'max': 100
}
})
assert blueprint.items['test2'].count.number is None
assert blueprint.items['test2'].count.min == 10
assert blueprint.items['test2'].count.max == 100
assert 10 <= blueprint.items['test2'].count() <= 100
blueprint.add_item({'name': 'foo', 'table': 'bar'})
blueprint.add_item({
'name': 'test3',
'table': 'bar',
'count': {
'number': 42,
'by': 'foo'
}
})
assert blueprint.items['test3'].count.number == 42
assert blueprint.items['test3'].count.by == 'foo'
def test_field_value():
blueprint = Blueprint()
blueprint.add_item({
'name': 'foo',
'table': 'bar',
'fields': {
'a': None,
'b': 42,
'c': '$foo',
'd': ''
}
})
item = blueprint.items['foo']
assert isinstance(item.fields['a'], generators.Value)
assert item.fields['a'].value is None
assert isinstance(item.fields['b'], generators.Value)
assert item.fields['b'].value == 42
assert isinstance(item.fields['c'], generators.Value)
assert isinstance(item.fields['c'].value, ValueExpression)
assert item.fields['c'].value.var == 'foo'
assert isinstance(item.fields['d'], generators.Value)
assert item.fields['d'].value == ''
def test_with_extra_keys():
blueprint = Blueprint()
msg = ("Unknown key(s) 'foo'. Possible keys are 'name, parent, "
"table, count, fields, store_in'.")
with pytest.raises(ValidationError) as e:
blueprint.add_item({'name': 'foo', 'table': 'bar', 'foo': 'bar'})
assert msg in str(e.value)
def test_name_and_table():
blueprint = Blueprint()
blueprint.add_item({'name': 'foo', 'table': 'bar'})
item = blueprint.items['foo']
assert item.name == 'foo'
assert item.table == 'bar'
assert list(item.fields.keys()) == ['id']
def test_write_empty_buffer(mocker):
blueprint = Blueprint(backend=Backend())
blueprint.add_item({'name': 'foo', 'table': 'test', 'fields': {'a': 42}})
item = blueprint.items['foo']
mocker.patch.object(blueprint.backend, 'write')
buffer = Buffer(blueprint)
# the buffer for the item is empty
buffer.write(item)
assert not blueprint.backend.write.called
def test_field_non_existing_generator():
blueprint = Blueprint()
msg = "Item 'foo', field 'a': Generator 'Foo' does not exist."
with pytest.raises(ValidationError) as e:
blueprint.add_item({
'name': 'foo',
'table': 'bar',
'fields': {
'a': {
'generator': 'Foo'
}
}
})
assert msg in str(e.value)
def test_item_preprocess(mocker):
columns = ('id', 'firstname', 'lastname', 'birth', 'gender')
existing = ((1, 'Homer', 'Simpson', date(1956, 6, 18),
'M'), (2, 'Marge', 'Simpson', date(1959, 6, 29),
'F'), (3, 'Bart', 'Simpson', date(1981, 4, 1), 'M'),
(4, 'Lisa', 'Simpson', date(1984, 5,
9), 'F'), (5, 'Maggie', 'Simpson',
date(1988, 11, 7), 'F'))
class DummyBackend(Backend):
def select(self, table, fields, **kwargs):
for row in existing:
yield tuple(row[columns.index(field)] for field in fields)
blueprint = Blueprint(backend=mocker.Mock(wraps=DummyBackend()))
item = Item(blueprint, 'person', 'test')
item.add_field('id', 'Integer', unique=True)
item.add_field('firstname', 'Text', unique=['lastname', 'birth'])
item.add_field('lastname', 'Text')
item.add_field('birth', 'Date')
item.add_field('gender', 'Choices', choices=['M', 'F'])
# add a unique field, but which isn't present in the db (shadow)
item.add_field('parent',
'Choices',
shadow=True,
choices='$parents',
unique=['birth'])
item.preprocess()
assert blueprint.backend.select.call_args == mocker.call(
'test', ['id', 'firstname', 'lastname', 'birth'])
assert 1 in item.fields['id'].seen
assert 5 in item.fields['id'].seen
assert 6 not in item.fields['id'].seen
seen = item.fields['firstname'].seen
assert ('Homer', 'Simpson', date(1956, 6, 18)) in seen
assert ('Lisa', 'Simpson', date(1984, 5, 9)) in seen
assert ('Bart', 'Simpson', date(2016, 10, 9)) not in seen
# add an item using the same table
item2 = Item(blueprint, 'person2', 'test')
item2.add_field('firstname', 'Text', unique=['lastname', 'birth'])
item2.add_field('lastname', 'Text')
# for this item, the date is unique
item2.add_field('birth', 'Date', unique=True)
blueprint.backend.select.reset_mock()
item2.preprocess()
# we should have queried only the birth
assert blueprint.backend.select.call_args == mocker.call('test', ['birth'])
seen = item2.fields['birth'].seen
assert date(1956, 6, 18) in seen
# we should use the same filter than for person
seen = item2.fields['firstname'].seen
assert ('Homer', 'Simpson', date(1956, 6, 18)) in seen
def test_field_without_generator():
blueprint = Blueprint()
msg = ("Field 'a' in item 'foo' must either be a value, or "
"a dict with a 'generator' key.")
with pytest.raises(ValidationError) as e:
blueprint.add_item({
'name': 'foo',
'table': 'bar',
'fields': {
'a': {
'foo': 'bar'
}
}
})
assert msg in str(e.value)
def test_field_extra_params():
blueprint = Blueprint()
msg = ("Item 'foo', field 'a': Got extra param(s) for generator "
"'Integer': foo")
with pytest.raises(ValidationError) as e:
blueprint.add_item({
'name': 'foo',
'table': 'bar',
'fields': {
'a': {
'generator': 'Integer',
'foo': 42
}
}
})
assert msg in str(e.value)
def test_field_with_generator():
blueprint = Blueprint()
blueprint.add_item({
'name': 'foo',
'table': 'bar',
'fields': {
'a': {
'generator': 'Integer',
'max': 42
}
}
})
item = blueprint.items['foo']
assert isinstance(item.fields['a'], generators.Integer)
assert item.fields['a'].max == 42
def test_item_generate_this_var(mocker):
blueprint = Blueprint()
blueprint.add_item({'name': 'foo', 'table': 'test'})
item = blueprint.items['foo']
call_count = {'count': 0}
def _generate(self):
call_count['count'] += 1
assert self.blueprint.vars['this'] == self
return item.namedtuple(id=1)
mocker.patch.object(ItemFactory, 'generate', _generate)
buffer = Buffer(blueprint)
item.generate(buffer, 10)
assert call_count['count'] == 10
def test_store_in_non_existing_item():
blueprint = Blueprint()
msg = ("Error in 'store_in' section in item 'foo': The "
"item 'toto' does not exist.")
with pytest.raises(ValidationError) as e:
blueprint.add_item({
'name': 'foo',
'table': 'bar',
'count': {
'by': 'toto'
},
'store_in': {
'this.toto.foos': '$this.id'
}
})
assert msg in str(e.value)
def test_generate_dependencies_ancestors():
class DummyBackend(Backend):
def write(self, item, objs):
return range(len(objs))
blueprint = Blueprint(backend=DummyBackend())
blueprint.add_item({'name': 'foo', 'table': 'test'})
blueprint.add_item({'name': 'foo2', 'parent': 'foo'})
blueprint.add_item({'name': 'foo3', 'parent': 'foo2'})
blueprint.add_item({
'name': 'bar',
'table': 'test',
'count': {
'number': 2,
'by': 'foo'
},
'store_in': {
'bars': '$this'
},
'fields': {
'parent_id': '$this.foo.id'
}
})
buffer = Buffer(blueprint)
foo2 = blueprint.items['foo2']
foo3 = blueprint.items['foo3']
foo3.generate(buffer, 2)
foo2.generate(buffer, 2)
assert len(buffer.buffers['foo2']) == 2
assert len(buffer.buffers['foo3']) == 2
assert len(buffer.buffers['bar']) == 0
assert len(blueprint.vars['bars']) == 0
buffer.write(foo3)
assert len(buffer.buffers['foo2']) == 2
assert len(buffer.buffers['foo3']) == 0
assert len(buffer.buffers['bar']) == 0
assert len(blueprint.vars['bars']) == 4
buffer.write(foo2)
assert len(buffer.buffers['foo2']) == 0
assert len(buffer.buffers['foo3']) == 0
assert len(buffer.buffers['bar']) == 0
assert len(blueprint.vars['bars']) == 8
def test_blueprint_generate(mocker):
import random
random.seed(42)
blueprint = Blueprint()
blueprint.add_item({'name': 'foo', 'table': 'test', 'count': 10})
blueprint.add_item({
'name': 'bar',
'table': 'test',
'count': {
'number': 20,
'by': 'foo'
}
})
blueprint.add_item({
'name': 'lol',
'table': 'test',
'count': {
'min': 10,
'max': 20
}
})
foo = blueprint.items['foo']
bar = blueprint.items['bar']
lol = blueprint.items['lol']
mocker.patch.object(foo, 'generate')
mocker.patch.object(bar, 'generate')
mocker.patch.object(lol, 'generate')
buffer = mocker.Mock(wraps=Buffer(blueprint))
mocker.patch('populous.blueprint.Buffer', return_value=buffer)
blueprint.generate()
assert foo.generate.call_args == mocker.call(buffer, 10)
assert bar.generate.called is False
if PY2:
assert lol.generate.call_args == mocker.call(buffer, 17)
else:
assert lol.generate.call_args == mocker.call(buffer, 20)
assert buffer.flush.called is True
def test_write_buffer(mocker):
class DummyBackend(Backend):
def write(self, item, objs):
return range(len(objs))
blueprint = Blueprint(backend=DummyBackend())
blueprint.add_item({'name': 'foo', 'table': 'test', 'fields': {'a': 42}})
item = blueprint.items['foo']
mocker.patch.object(item, 'store_final_values')
mocker.patch.object(item, 'generate_dependencies')
buffer = Buffer(blueprint, maxlen=10)
item.generate(buffer, 10)
objs = tuple(item.namedtuple(id=x, a=42) for x in range(10))
assert len(buffer.buffers['foo']) == 0
assert item.store_final_values.call_args == mocker.call(objs)
assert item.generate_dependencies.call_args == mocker.call(buffer, objs)
def test_flush_buffer(mocker):
blueprint = Blueprint(backend=mocker.MagicMock())
blueprint.add_item({'name': 'foo', 'table': 'test'})
blueprint.add_item({'name': 'bar', 'table': 'test'})
buffer = Buffer(blueprint)
blueprint.items['foo'].generate(buffer, 5)
blueprint.items['bar'].generate(buffer, 4)
assert len(buffer.buffers['foo']) == 5
assert len(buffer.buffers['bar']) == 4
buffer.flush()
assert len(buffer.buffers['foo']) == 0
assert len(buffer.buffers['bar']) == 0
assert blueprint.backend.write.call_count == 2
assert (blueprint.backend.write.call_args_list[0] == mocker.call(
blueprint.items['foo'], ((), (), (), (), ())))
assert (blueprint.backend.write.call_args_list[1] == mocker.call(
blueprint.items['bar'], ((), (), (), ())))
def test_blueprint_preprocess(mocker):
blueprint = Blueprint()
blueprint.add_item({'name': 'foo', 'table': 'test'})
blueprint.add_item({'name': 'bar', 'table': 'test'})
foo = mocker.Mock(wraps=blueprint.items['foo'])
bar = mocker.Mock(wraps=blueprint.items['bar'])
blueprint.items['foo'] = foo
blueprint.items['bar'] = bar
blueprint.preprocess()
assert foo.preprocess.called is True
assert bar.preprocess.called is True
def test_store_values():
class DummyBackend(Backend):
def write(self, item, objs):
return range(len(objs))
blueprint = Blueprint(backend=DummyBackend())
blueprint.add_item({
'name': 'foo',
'table': 'test',
'store_in': {
'foos': '$this'
}
})
blueprint.add_item({
'name': 'bar',
'table': 'test2',
'count': {
'by': 'foo',
'number': 2
},
'store_in': {
'this.foo.bar_ids': '$this.id'
}
})
buffer = Buffer(blueprint)
blueprint.items['foo'].generate(buffer, 10)
buffer.flush()
assert list(foo.id for foo in blueprint.vars['foos']) == list(range(10))
ids = iter(range(20))
for foo in blueprint.vars['foos']:
assert foo.bar_ids == [next(ids), next(ids)]
def test_count_var():
blueprint = Blueprint()
blueprint.add_item({'name': 'test1', 'table': 'bar', 'count': '$foo'})
blueprint.vars['foo'] = 42
assert blueprint.items['test1'].count() == 42
blueprint.add_item({
'name': 'test2',
'table': 'bar',
'count': {
'min': '$min',
'max': '$max'
}
})
blueprint.vars['min'] = 1
blueprint.vars['max'] = 10
assert 1 <= blueprint.items['test2'].count() <= 10
def test_parent_with_same_name():
blueprint = Blueprint()
blueprint.add_item({'name': 'foo', 'table': 'bar'})
blueprint.add_item({'name': 'foo', 'table': 'test'})
assert len(blueprint.items) == 1
assert blueprint.items['foo'].table == 'test'
def test_add_var():
blueprint = Blueprint()
blueprint.add_var('foo', 42)
blueprint.add_var('bar', None)
assert blueprint.vars['foo'] == 42
assert blueprint.vars['bar'] is None
def test_two_parents():
blueprint = Blueprint()
blueprint.add_item({'name': 'foo', 'table': 'bar'})
msg = "Re-defining item 'foo' while setting 'bar' as parent is ambiguous"
with pytest.raises(ValidationError) as e:
blueprint.add_item({'name': 'foo', 'parent': 'bar'})
assert msg in str(e.value)
def test_description_wrong_type():
blueprint = Blueprint()
msg = "A blueprint item must be a dict, not a 'NoneType'"
with pytest.raises(ValidationError) as e:
blueprint.add_item(None)
assert msg in str(e.value)
msg = "A blueprint item must be a dict, not a 'int'"
with pytest.raises(ValidationError) as e:
blueprint.add_item(42)
assert msg in str(e.value)
msg = "A blueprint item must be a dict, not a 'list'"
with pytest.raises(ValidationError) as e:
blueprint.add_item([])
assert msg in str(e.value)
def test_flush_buffer_with_dependencies(mocker):
class DummyBackend(Backend):
def write(self, item, objs):
return range(len(objs))
blueprint = Blueprint(backend=mocker.Mock(wraps=DummyBackend()))
blueprint.add_item({'name': 'foo', 'table': 'test'})
blueprint.add_item({
'name': 'bar',
'table': 'test',
'count': {
'number': 1,
'by': 'foo'
}
})
blueprint.add_item({
'name': 'lol',
'table': 'test',
'count': {
'number': 1,
'by': 'bar'
}
})
buffer = Buffer(blueprint)
blueprint.items['foo'].generate(buffer, 5)
assert len(buffer.buffers['foo']) == 5
assert len(buffer.buffers['bar']) == 0
assert len(buffer.buffers['lol']) == 0
buffer.flush()
assert len(buffer.buffers['foo']) == 0
assert len(buffer.buffers['bar']) == 0
assert len(buffer.buffers['lol']) == 0
assert blueprint.backend.write.call_count == 3
assert (blueprint.backend.write.call_args_list[0] == mocker.call(
blueprint.items['foo'], ((), (), (), (), ())))
assert (blueprint.backend.write.call_args_list[1] == mocker.call(
blueprint.items['bar'], ((), (), (), (), ())))
assert (blueprint.backend.write.call_args_list[2] == mocker.call(
blueprint.items['lol'], ((), (), (), (), ())))
def test_required():
blueprint = Blueprint()
msg = "Items without a parent must have a name"
with pytest.raises(ValidationError) as e:
blueprint.add_item({})
assert msg in str(e.value)
msg = "Item 'foo' does not have a table."
with pytest.raises(ValidationError) as e:
blueprint.add_item({'name': 'foo'})
assert msg in str(e.value)
def test_store_values():
class DummyBackend(Backend):
counters = {}
def write(self, item, objs):
counter = self.counters.setdefault(item.name, count())
return [next(counter) for _ in objs]
blueprint = Blueprint(backend=DummyBackend())
blueprint.add_item({
'name': 'foo',
'table': 'test',
'store_in': {
'foos': '$this'
}
})
blueprint.add_item({
'name': 'bar',
'table': 'test2',
'count': {
'by': 'foo',
'number': 2
},
'store_in': {
'this.foo.bars': '$this'
},
'fields': {
'num': '$(this.foo.bars|length)'
}
})
buffer = Buffer(blueprint, maxlen=15)
blueprint.items['foo'].generate(buffer, 10)
# the values have been generated, but empty ids have been stored
assert [foo.id for foo in blueprint.vars['foos']] == [None] * 10
buffer.write(blueprint.items['foo'])
# the stored ids have now been replaced
assert [foo.id for foo in blueprint.vars['foos']] == list(range(10))
# each foo object contains the corresponding bars, each bar has
# an id & a number corresponding to the number of 'bars' in the
# current 'foo' at the time of generation
ids = iter(range(20))
for foo in blueprint.vars['foos']:
assert [(bar.id, bar.num) for bar in foo.bars] == [
(next(ids), 0),
(next(ids), 1),
]
