def test_extend_bloom_filter(): bf = BloomFilter(capacity=100) for key in range(99): bf.add(key) assert len(bf._filters) == 1 for key in range(99): bf.add(key) assert len(bf._filters) == 1 bf.add(99) assert len(bf._filters) == 2 assert 42 in bf assert 99 in bf assert 100 not in bf for key in range(1000): bf.add(key) # the capacity quadruples each time assert len(bf._filters) == 3 assert bf._capacity == 1600 for key in range(1000): assert key in bf
def test_store_values(): class DummyBackend(Backend): def write(self, item, objs): return range(len(objs)) blueprint = Blueprint(backend=DummyBackend()) blueprint.add_item({ 'name': 'foo', 'table': 'test', 'store_in': { 'foos': '$this' } }) blueprint.add_item({ 'name': 'bar', 'table': 'test2', 'count': { 'by': 'foo', 'number': 2 }, 'store_in': { 'this.foo.bar_ids': '$this.id' } }) buffer = Buffer(blueprint) blueprint.items['foo'].generate(buffer, 10) buffer.flush() assert list(foo.id for foo in blueprint.vars['foos']) == list(range(10)) ids = iter(range(20)) for foo in blueprint.vars['foos']: assert foo.bar_ids == [next(ids), next(ids)]
def test_store_values(): class DummyBackend(Backend): counters = {} def write(self, item, objs): counter = self.counters.setdefault(item.name, count()) return [next(counter) for _ in objs] blueprint = Blueprint(backend=DummyBackend()) blueprint.add_item({ 'name': 'foo', 'table': 'test', 'store_in': { 'foos': '$this' } }) blueprint.add_item({ 'name': 'bar', 'table': 'test2', 'count': { 'by': 'foo', 'number': 2 }, 'store_in': { 'this.foo.bars': '$this' }, 'fields': { 'num': '$(this.foo.bars|length)' } }) buffer = Buffer(blueprint, maxlen=15) blueprint.items['foo'].generate(buffer, 10) # the values have been generated, but empty ids have been stored assert [foo.id for foo in blueprint.vars['foos']] == [None] * 10 buffer.write(blueprint.items['foo']) # the stored ids have now been replaced assert [foo.id for foo in blueprint.vars['foos']] == list(range(10)) # each foo object contains the corresponding bars, each bar has # an id & a number corresponding to the number of 'bars' in the # current 'foo' at the time of generation ids = iter(range(20)) for foo in blueprint.vars['foos']: assert [(bar.id, bar.num) for bar in foo.bars] == [ (next(ids), 0), (next(ids), 1), ]
def test_error_rate(): bf = BloomFilter(capacity=1000, error_rate=0.01) for x in range(1000): bf.add(x) for x in range(1000): assert x in bf errors = 0 for x in range(1000, 11000): if x in bf: errors += 1 assert errors / 10000. < 0.01
def write(self, item, objs): with self.cursor as cursor: stmt = "INSERT INTO {} ({}) VALUES {} RETURNING {}".format( item.table, ", ".join(item.db_fields), ", ".join("({})".format(", ".join( "%s" for _ in range(len(item.db_fields)))) for _ in range(len(objs))), self.get_pk_column(item.table)) try: cursor.execute(stmt, tuple(v for vs in objs for v in vs)) except psycopg2.DatabaseError as e: raise BackendError("Error during the generation of " "'{}': {}".format(item.name, e)) return tuple(e[0] for e in cursor.fetchall())
def test_jinja_random(): from populous.vars import JinjaValueExpression from populous.vars import TemplateExpression j = JinjaValueExpression('[21, 42]|random') assert set(j.evaluate() for _ in range(100)) == set((21, 42)) j = JinjaValueExpression('[]|random') with pytest.raises(GenerationError) as e: j.evaluate() msg = ("Error generating value '$([]|random)': No random item, sequence " "was empty.") assert msg in str(e.value) t = TemplateExpression('{{ [21, 42]|random }}') assert set(t.evaluate() for _ in range(100)) == set(("21", "42"))
def generate(self, buffer, count, parent=None): factory = ItemFactory(self, parent=parent) for i in range(count): self.blueprint.vars['this'] = factory obj = factory.generate() del self.blueprint.vars['this'] buffer.add(obj)
def test_error_rate(): bf = BloomFilter(capacity=1000, error_rate=0.01) for x in range(1000): bf.add(x) for x in range(1000): assert x in bf errors = 0 for x in range(1000, 11000): if x in bf: errors += 1 if not PY2: # Something is fishy in Python 3 # So we increase the margin error # Until we figure out where the error # comes from assert errors / 10000. < 0.015 else: assert errors / 10000. < 0.01
def test_write_buffer(mocker): class DummyBackend(Backend): def write(self, item, objs): return range(len(objs)) blueprint = Blueprint(backend=DummyBackend()) blueprint.add_item({'name': 'foo', 'table': 'test', 'fields': {'a': 42}}) item = blueprint.items['foo'] mocker.patch.object(item, 'store_final_values') mocker.patch.object(item, 'generate_dependencies') buffer = Buffer(blueprint, maxlen=10) item.generate(buffer, 10) objs = tuple(item.namedtuple(id=x, a=42) for x in range(10)) assert len(buffer.buffers['foo']) == 0 assert item.store_final_values.call_args == mocker.call(objs) assert item.generate_dependencies.call_args == mocker.call(buffer, objs)
def write(self, item, objs): return range(len(objs))
def ids(): for x in range(10): yield x yield x