def test_truncation(counter_type): # noqa
num = 100
data = {}
for x in xrange(num):
data[str(x)] = x + 1
counter = counter_type(data)
assert len(counter) == num
truncation_length = 10
counter.truncate_most_common(truncation_length)
assert len(counter) == truncation_length
most_common = Counter(data).most_common(truncation_length)
from_counter = map(itemgetter(1), counter.most_common(truncation_length))
expected = map(itemgetter(1), most_common)
from_counter = map(abs, from_counter)
assert from_counter == expected
assert set(counter_type(dict(most_common)).items()) == set(counter.items())
expected = {}
for x in xrange(90, 100):
expected[str(x)] = x + 1
should_be_counter = counter_type(expected)
assert should_be_counter == counter
def test_readonly(counter_type):
fc = FeatureCollection({
'hello': counter_type(Counter('hello')),
'goodbye': counter_type(Counter('goodbye'))})
fc2 = FeatureCollection({
'hello': counter_type(Counter('hello')),
'goodbye': counter_type(Counter('goodbye'))})
fc.read_only = True
with pytest.raises(ReadOnlyException):
fc += fc2
with pytest.raises(ReadOnlyException):
fc -= fc2
with pytest.raises(ReadOnlyException):
fc *= 2
with pytest.raises(ReadOnlyException):
fc['woof'] = StringCounter()
if hasattr(counter_type, 'read_only'):
with pytest.raises(ReadOnlyException):
fc['hello']['l'] = 3
with pytest.raises(ReadOnlyException):
fc['hello']['l'] += 3
fc.read_only = False
fc += fc2
assert Counter(map(abs,fc['hello'].values())) == Counter({2: 3, 4: 1})
fc -= fc2
fc -= fc2
assert Counter(map(abs,fc['hello'].values())) == Counter()
def test_build_from_dict(counter_type):
mc = FeatureCollection({
'hello': counter_type(Counter('hello')),
'goodbye': counter_type(Counter('goodbye'))})
assert Counter(map(abs,mc['hello'].values())) == Counter({1: 3, 2: 1})
assert isinstance(mc['hello'], counter_type)
def test_meta_adding(counter_type):
mc = FeatureCollection({
'hello': counter_type(Counter('hello')),
'goodbye': counter_type(Counter('goodbye'))})
mc2 = mc + mc
assert Counter(map(abs,mc2['hello'].values())) == Counter({2: 3, 4: 1})
def test_serialize_deserialize(counter_type):
## build entity, serialize, deserialize, and verify its multisets
ent1 = FeatureCollection()
ent1["bow"] += counter_type(Counter(["big", "dog"]))
ent1["bow"] += counter_type(Counter("tall building"))
ent1["bon"] += counter_type(Counter(["Super Cat", "Small Cat", "Tiger Fish"]))
blob = ent1.dumps()
ent2 = FeatureCollection.loads(blob)
assert_same_fc(ent1, ent2)
def test_serialize_deserialize(counter_type):
## build entity, serialize, deserialize, and verify its multisets
ent1 = FeatureCollection()
ent1['bow'] += counter_type(Counter(['big', 'dog']))
ent1['bow'] += counter_type(Counter('tall building'))
ent1['bon'] += counter_type(Counter(['Super Cat', 'Small Cat',
'Tiger Fish']))
blob = ent1.dumps()
ent2 = FeatureCollection.loads(blob)
assert_same_fc(ent1, ent2)
def test_type(counter_type):
m1 = FeatureCollection()
m1['bow'] += counter_type(Counter(['big', 'dog']))
assert type(m1) == FeatureCollection
m2 = FeatureCollection()
m2['bow'] += counter_type(Counter(['cat']))
m1 += m2
assert type(m1) == FeatureCollection
def test_type(counter_type):
ent1 = FeatureCollection()
ent1["bow"] += counter_type(Counter(["big", "dog"]))
if counter_type.__name__ == "StringCounter":
ent1["bow"]["a"] += 1
assert isinstance(ent1, FeatureCollection)
ent3 = FeatureCollection()
ent3["bow"] += counter_type(Counter(["cat"]))
ent1 += ent3
assert isinstance(ent1, FeatureCollection)
def test_type(counter_type):
ent1 = FeatureCollection()
ent1['bow'] += counter_type(Counter(['big', 'dog']))
if counter_type.__name__ == 'StringCounter':
ent1['bow']['a'] += 1
assert isinstance(ent1, FeatureCollection)
ent3 = FeatureCollection()
ent3['bow'] += counter_type(Counter(['cat']))
ent1 += ent3
assert isinstance(ent1, FeatureCollection)
def test_fc_meta_adding_complex(counter_type):
fc = FeatureCollection({"hello": counter_type(Counter("hello")), "goodbye": counter_type(Counter("goodbye"))})
fc2 = FeatureCollection({"hello": counter_type(Counter("hello")), "goodbye": counter_type(Counter("goodbye"))})
fc3 = fc + fc2
assert Counter(map(abs, fc3["hello"].values())) == Counter({2: 3, 4: 1})
fc += fc2
assert Counter(map(abs, fc["hello"].values())) == Counter({2: 3, 4: 1})
fc3 -= fc2
assert Counter(map(abs, fc3["hello"].values())) == Counter({1: 3, 2: 1})
fc3 -= fc2
assert Counter(map(abs, fc3["hello"].values())) == Counter()
def test_multiset_change(counter_type):
ent1 = FeatureCollection()
ent1["bow"] += counter_type(Counter(["big", "dog"]))
ent1.pop("bow")
assert dict(ent1.items()) == dict()
## can pop empty -- fails
# ent1.pop('foo')
## set equal to
test_data = ["big2", "dog2"]
ent1["bow"] = counter_type(Counter(test_data))
assert list(map(abs, ent1["bow"].values())) == [1, 1]
ent1["bow"] += counter_type(Counter(test_data))
assert list(map(abs, ent1["bow"].values())) == [2, 2]
def test_multiset_change(counter_type):
ent1 = FeatureCollection()
ent1['bow'] += counter_type(Counter(['big', 'dog']))
ent1.pop('bow')
assert dict(ent1.items()) == dict()
## can pop empty -- fails
#ent1.pop('foo')
## set equal to
test_data = ['big2', 'dog2']
ent1['bow'] = counter_type(Counter(test_data))
assert list(ent1['bow'].values()) == [1,1]
ent1['bow'] += counter_type(Counter(test_data))
assert list(ent1['bow'].values()) == [2,2]
def test_default(counter_type):
'does a FC make a new counter that adds properly'
mc = FeatureCollection()
assert isinstance(mc['foo'], counter_type)
mc['foo'] += counter_type(Counter('dog'))
assert isinstance(mc['foo'], counter_type), \
'failed and made %s' % type(mc['foo'])
mc['foo'] -= counter_type(Counter('dog'))
assert isinstance(mc['foo'], counter_type), \
'failed and made %s' % type(mc['foo'])
if hasattr(mc['foo'], 'substract'):
mc['foo'].subtract(counter_type(Counter('dog')))
assert isinstance(mc['foo'], counter_type), \
'failed and made %s' % type(mc['foo'])
mc['foo'] += counter_type(Counter('dog'))
mc['foo'] += counter_type(Counter('dog'))
assert isinstance(mc['foo'], counter_type), \
'failed and made %s' % type(mc['foo'])
mc['foo'] += counter_type(Counter('dog'))
mc['foo'] += counter_type(Counter('dog cat'))
assert Counter(map(abs,mc['foo'].values())) == Counter({1: 4, 3: 3})
def test_bundle_default(counter_type):
'does it make a new counter that adds properly'
fc = FeatureCollection()
assert isinstance(fc['foo'], counter_type)
fc['foo'] += counter_type(Counter('dog'))
assert isinstance(fc['foo'], counter_type), \
'failed and made %s' % type(fc['foo'])
fc['foo'] -= counter_type(Counter('dog'))
assert isinstance(fc['foo'], counter_type), \
'failed and made %s' % type(fc['foo'])
if hasattr(fc['foo'], 'substract'):
fc['foo'].subtract(counter_type(Counter('dog')))
assert isinstance(fc['foo'], counter_type), \
'failed and made %s' % type(fc['foo'])
fc['foo'] += counter_type(Counter('dog'))
fc['foo'] += counter_type(Counter('dog'))
assert isinstance(fc['foo'], counter_type), \
'failed and made %s' % type(fc['foo'])
fc['foo'] += counter_type(Counter('dog'))
fc['foo'] += counter_type(Counter('dog cat'))
assert Counter(fc['foo'].values()) == Counter({1: 4, 3: 3})
def test_fc_meta_adding_complex(counter_type):
fc = FeatureCollection({
'hello': counter_type(Counter('hello')),
'goodbye': counter_type(Counter('goodbye'))})
fc2 = FeatureCollection({
'hello': counter_type(Counter('hello')),
'goodbye': counter_type(Counter('goodbye'))})
fc3 = fc + fc2
assert Counter(fc3['hello'].values()) == Counter({2: 3, 4: 1})
fc += fc2
assert Counter(fc['hello'].values()) == Counter({2: 3, 4: 1})
fc3 -= fc2
assert Counter(fc3['hello'].values()) == Counter({1: 3, 2: 1})
fc3 -= fc2
assert Counter(fc3['hello'].values()) == Counter()
def test_generation_pop(counter_type):
c1 = counter_type(dict(cat=5, dog=3))
if not has_gen(c1):
return
h11 = (id(c1), c1.generation)
c1.pop('cat')
h12 = (id(c1), c1.generation)
assert h11 != h12, c1
def test_meta_adding_complex(counter_type):
mc = FeatureCollection({
'hello': counter_type(Counter('hello')),
'goodbye': counter_type(Counter('goodbye'))})
mc2 = FeatureCollection({
'hello': counter_type(Counter('hello')),
'goodbye': counter_type(Counter('goodbye'))})
mc3 = mc + mc2
assert Counter(map(abs,mc3['hello'].values())) == Counter({2: 3, 4: 1})
mc += mc2
assert Counter(map(abs,mc['hello'].values())) == Counter({2: 3, 4: 1})
## isub tests
mc3 -= mc2
assert Counter(map(abs,mc3['hello'].values())) == Counter({1: 3, 2: 1})
mc3 -= mc2
assert Counter(map(abs,mc3['hello'].values())) == Counter()
def test_generation_delitem(counter_type):
c1 = counter_type(dict(cat=5, dog=3))
if not has_gen(c1):
return
assert not ('bird' in c1)
h11 = (id(c1), c1.generation)
del c1['cat']
h12 = (id(c1), c1.generation)
assert h11 != h12, c1
def test_generation_missing(counter_type):
c1 = counter_type(dict(cat=5, dog=3))
if not has_gen(c1):
return
assert not ('bird' in c1)
h11 = (id(c1), c1.generation)
assert c1['bird'] == 0
c1['bird'] += 1
h12 = (id(c1), c1.generation)
assert h11 != h12, c1
def test_fc_eq(counter_type):
fc1 = FeatureCollection({"hello": counter_type(Counter("hello")), "goodbye": counter_type(Counter("goodbye"))})
fc2 = FeatureCollection({"hello": counter_type(Counter("hello")), "goodbye": counter_type(Counter("goodbye"))})
fc3 = FeatureCollection({"hello": counter_type(Counter("hello")), "goodbye": counter_type(Counter("goodbye2"))})
assert fc1 == fc2
assert fc1 != fc3
def test_entity(counter_type):
## build entity, serialize, deserialize, and verify its multisets
fc1 = FeatureCollection()
fc1["bow"] += counter_type(Counter(["big", "dog"]))
fc1["bow"] += counter_type(Counter("tall building"))
fc1["bon"] += counter_type(Counter(["Super Cat", "Small Cat", "Tiger Fish"]))
## there should be nine items of size 1
assert Counter(map(abs, fc1["bow"].values()))[1] == 10, fc1["bow"].items()
## double the counts, should recurse down
fc1 += fc1
## check values doubled
assert Counter(map(abs, fc1["bow"].values()))[2] == 10, fc1["bow"].items()
## serialize/deserialize it
blob = fc1.dumps()
assert_same_fc(fc1, FeatureCollection.loads(blob))
## deserialize it via chunk
fc2 = FeatureCollection.loads(fc1.dumps())
assert_same_fc(fc1, fc2)
def test_entity(counter_type):
## build entity, serialize, deserialize, and verify its multisets
fc1 = FeatureCollection()
fc1['bow'] += counter_type(Counter(['big', 'dog']))
fc1['bow'] += counter_type(Counter('tall building'))
fc1['bon'] += counter_type(Counter(['Super Cat', 'Small Cat', 'Tiger Fish']))
## there should be nine items of size 1
assert Counter(fc1['bow'].values())[1] == 10, fc1['bow'].items()
## double the counts, should recurse down
fc1 += fc1
## check values doubled
assert Counter(fc1['bow'].values())[2] == 10, fc1['bow'].items()
## serialize/deserialize it
blob = fc1.dumps()
assert_same_fc(fc1, FeatureCollection.loads(blob))
## deserialize it via chunk
fc2 = FeatureCollection.loads(fc1.dumps())
assert_same_fc(fc1, fc2)
def test_fc_eq(counter_type):
fc1 = FeatureCollection({
'hello': counter_type(Counter('hello')),
'goodbye': counter_type(Counter('goodbye'))})
fc2 = FeatureCollection({
'hello': counter_type(Counter('hello')),
'goodbye': counter_type(Counter('goodbye'))})
fc3 = FeatureCollection({
'hello': counter_type(Counter('hello')),
'goodbye': counter_type(Counter('goodbye2'))})
assert fc1 == fc2
assert fc1 != fc3
def test_eq(counter_type):
mc1 = FeatureCollection({
'hello': counter_type(Counter('hello')),
'goodbye': counter_type(Counter('goodbye'))})
mc2 = FeatureCollection({
'hello': counter_type(Counter('hello')),
'goodbye': counter_type(Counter('goodbye'))})
mc3 = FeatureCollection({
'hello': counter_type(Counter('hello')),
'goodbye': counter_type(Counter('goodbye2'))})
assert mc1 == mc2
assert mc1 != mc3
def test_bundle_default(counter_type):
"does it make a new counter that adds properly"
fc = FeatureCollection()
assert isinstance(fc["foo"], counter_type)
fc["foo"] += counter_type(Counter("dog"))
assert isinstance(fc["foo"], counter_type), "failed and made %s" % type(fc["foo"])
fc["foo"] -= counter_type(Counter("dog"))
assert isinstance(fc["foo"], counter_type), "failed and made %s" % type(fc["foo"])
if hasattr(fc["foo"], "substract"):
fc["foo"].subtract(counter_type(Counter("dog")))
assert isinstance(fc["foo"], counter_type), "failed and made %s" % type(fc["foo"])
fc["foo"] += counter_type(Counter("dog"))
fc["foo"] += counter_type(Counter("dog"))
assert isinstance(fc["foo"], counter_type), "failed and made %s" % type(fc["foo"])
fc["foo"] += counter_type(Counter("dog"))
fc["foo"] += counter_type(Counter("dog cat"))
assert Counter(map(abs, fc["foo"].values())) == Counter({1: 4, 3: 3})
def test_fc_meta_adding(counter_type):
fc = FeatureCollection({"hello": counter_type(Counter("hello")), "goodbye": counter_type(Counter("goodbye"))})
fc2 = fc + fc
assert Counter(map(abs, fc2["hello"].values())) == Counter({2: 3, 4: 1})
def test_fc_meta_adding(counter_type):
fc = FeatureCollection({
'hello': counter_type(Counter('hello')),
'goodbye': counter_type(Counter('goodbye'))})
fc2 = fc + fc
assert Counter(fc2['hello'].values()) == Counter({2: 3, 4: 1})
def test_clone(counter_type):
ent1 = FeatureCollection()
ent1['bow'] += counter_type(Counter(['cat']))
ent2 = FeatureCollection(ent1)
assert ent2 is not ent1
assert ent2 == ent1
def test_multiset_equality(counter_type):
ent1 = FeatureCollection()
ent2 = FeatureCollection()
ent1['bow'] += counter_type(Counter(['big', 'dog']))
ent2['bow'] += counter_type(Counter(['big', 'dog']))
assert ent1 == ent2
def test_values(counter_type):
c = counter_type(Counter('dog'))
assert set(map(int, c.values())) == set([1])
assert len(c.values()) == 3