def test_extend_bloom_filter():
bf = BloomFilter(capacity=100)
for key in range(99):
bf.add(key)
assert len(bf._filters) == 1
for key in range(99):
bf.add(key)
assert len(bf._filters) == 1
bf.add(99)
assert len(bf._filters) == 2
assert 42 in bf
assert 99 in bf
assert 100 not in bf
for key in range(1000):
bf.add(key)
# the capacity quadruples each time
assert len(bf._filters) == 3
assert bf._capacity == 1600
for key in range(1000):
assert key in bf
def test_store_values():
class DummyBackend(Backend):
def write(self, item, objs):
return range(len(objs))
blueprint = Blueprint(backend=DummyBackend())
blueprint.add_item({
'name': 'foo',
'table': 'test',
'store_in': {
'foos': '$this'
}
})
blueprint.add_item({
'name': 'bar',
'table': 'test2',
'count': {
'by': 'foo',
'number': 2
},
'store_in': {
'this.foo.bar_ids': '$this.id'
}
})
buffer = Buffer(blueprint)
blueprint.items['foo'].generate(buffer, 10)
buffer.flush()
assert list(foo.id for foo in blueprint.vars['foos']) == list(range(10))
ids = iter(range(20))
for foo in blueprint.vars['foos']:
assert foo.bar_ids == [next(ids), next(ids)]
def test_store_values():
class DummyBackend(Backend):
counters = {}
def write(self, item, objs):
counter = self.counters.setdefault(item.name, count())
return [next(counter) for _ in objs]
blueprint = Blueprint(backend=DummyBackend())
blueprint.add_item({
'name': 'foo',
'table': 'test',
'store_in': {
'foos': '$this'
}
})
blueprint.add_item({
'name': 'bar',
'table': 'test2',
'count': {
'by': 'foo',
'number': 2
},
'store_in': {
'this.foo.bars': '$this'
},
'fields': {
'num': '$(this.foo.bars|length)'
}
})
buffer = Buffer(blueprint, maxlen=15)
blueprint.items['foo'].generate(buffer, 10)
# the values have been generated, but empty ids have been stored
assert [foo.id for foo in blueprint.vars['foos']] == [None] * 10
buffer.write(blueprint.items['foo'])
# the stored ids have now been replaced
assert [foo.id for foo in blueprint.vars['foos']] == list(range(10))
# each foo object contains the corresponding bars, each bar has
# an id & a number corresponding to the number of 'bars' in the
# current 'foo' at the time of generation
ids = iter(range(20))
for foo in blueprint.vars['foos']:
assert [(bar.id, bar.num) for bar in foo.bars] == [
(next(ids), 0),
(next(ids), 1),
]
def test_error_rate():
bf = BloomFilter(capacity=1000, error_rate=0.01)
for x in range(1000):
bf.add(x)
for x in range(1000):
assert x in bf
errors = 0
for x in range(1000, 11000):
if x in bf:
errors += 1
assert errors / 10000. < 0.01
def write(self, item, objs):
with self.cursor as cursor:
stmt = "INSERT INTO {} ({}) VALUES {} RETURNING {}".format(
item.table, ", ".join(item.db_fields),
", ".join("({})".format(", ".join(
"%s" for _ in range(len(item.db_fields))))
for _ in range(len(objs))),
self.get_pk_column(item.table))
try:
cursor.execute(stmt, tuple(v for vs in objs for v in vs))
except psycopg2.DatabaseError as e:
raise BackendError("Error during the generation of "
"'{}': {}".format(item.name, e))
return tuple(e[0] for e in cursor.fetchall())
def test_jinja_random():
from populous.vars import JinjaValueExpression
from populous.vars import TemplateExpression
j = JinjaValueExpression('[21, 42]|random')
assert set(j.evaluate() for _ in range(100)) == set((21, 42))
j = JinjaValueExpression('[]|random')
with pytest.raises(GenerationError) as e:
j.evaluate()
msg = ("Error generating value '$([]|random)': No random item, sequence "
"was empty.")
assert msg in str(e.value)
t = TemplateExpression('{{ [21, 42]|random }}')
assert set(t.evaluate() for _ in range(100)) == set(("21", "42"))
def generate(self, buffer, count, parent=None):
factory = ItemFactory(self, parent=parent)
for i in range(count):
self.blueprint.vars['this'] = factory
obj = factory.generate()
del self.blueprint.vars['this']
buffer.add(obj)
def test_error_rate():
bf = BloomFilter(capacity=1000, error_rate=0.01)
for x in range(1000):
bf.add(x)
for x in range(1000):
assert x in bf
errors = 0
for x in range(1000, 11000):
if x in bf:
errors += 1
if not PY2:
# Something is fishy in Python 3
# So we increase the margin error
# Until we figure out where the error
# comes from
assert errors / 10000. < 0.015
else:
assert errors / 10000. < 0.01
def test_write_buffer(mocker):
class DummyBackend(Backend):
def write(self, item, objs):
return range(len(objs))
blueprint = Blueprint(backend=DummyBackend())
blueprint.add_item({'name': 'foo', 'table': 'test', 'fields': {'a': 42}})
item = blueprint.items['foo']
mocker.patch.object(item, 'store_final_values')
mocker.patch.object(item, 'generate_dependencies')
buffer = Buffer(blueprint, maxlen=10)
item.generate(buffer, 10)
objs = tuple(item.namedtuple(id=x, a=42) for x in range(10))
assert len(buffer.buffers['foo']) == 0
assert item.store_final_values.call_args == mocker.call(objs)
assert item.generate_dependencies.call_args == mocker.call(buffer, objs)
def write(self, item, objs):
return range(len(objs))
def ids():
for x in range(10):
yield x
yield x
