def test_ft_roundtrip():
fc = FeatureCollection()
fc['@NAME']['foo'].append([
('nltk', 5, 2),
])
fc2 = FeatureCollection.loads(fc.dumps())
assert fc['@NAME'] == fc2['@NAME']
def test_read_only_preserved_after_serialized():
fc = FeatureCollection({'NAME': {'foo': 1, 'baz': 2}})
fc.read_only = True
fcnew = FeatureCollection.loads(fc.dumps())
assert fcnew.read_only
with pytest.raises(ReadOnlyException):
fcnew['NAME']['foo'] += 1
def test_nsc_roundtrip():
fc = FeatureCollection()
fc['#testing'] = NestedStringCounter()
fc['#testing']['foo'] = StringCounter({'foo': 1})
fc['#testing']['bar'] = StringCounter({'foo': 2, 'bar': 1})
dumped = fc.dumps()
assert FeatureCollection.loads(dumped) == fc
def test_string_counter_serialize():
fc = FeatureCollection()
fc['thing1'] = StringCounter()
fc['thing1']['foo'] += 1
fc_str = fc.dumps()
fc2 = FeatureCollection.loads(fc_str)
assert fc2['thing1']['foo'] == 1
def test_serialize_deserialize(counter_type):
## build entity, serialize, deserialize, and verify its multisets
ent1 = FeatureCollection()
ent1["bow"] += counter_type(Counter(["big", "dog"]))
ent1["bow"] += counter_type(Counter("tall building"))
ent1["bon"] += counter_type(Counter(["Super Cat", "Small Cat", "Tiger Fish"]))
blob = ent1.dumps()
ent2 = FeatureCollection.loads(blob)
assert_same_fc(ent1, ent2)
def test_serialize_deserialize(counter_type):
## build entity, serialize, deserialize, and verify its multisets
ent1 = FeatureCollection()
ent1['bow'] += counter_type(Counter(['big', 'dog']))
ent1['bow'] += counter_type(Counter('tall building'))
ent1['bon'] += counter_type(Counter(['Super Cat', 'Small Cat',
'Tiger Fish']))
blob = ent1.dumps()
ent2 = FeatureCollection.loads(blob)
assert_same_fc(ent1, ent2)
def test_entity(counter_type):
## build entity, serialize, deserialize, and verify its multisets
fc1 = FeatureCollection()
fc1['bow'] += counter_type(Counter(['big', 'dog']))
fc1['bow'] += counter_type(Counter('tall building'))
fc1['bon'] += counter_type(Counter(['Super Cat', 'Small Cat', 'Tiger Fish']))
## there should be nine items of size 1
assert Counter(fc1['bow'].values())[1] == 10, fc1['bow'].items()
## double the counts, should recurse down
fc1 += fc1
## check values doubled
assert Counter(fc1['bow'].values())[2] == 10, fc1['bow'].items()
## serialize/deserialize it
blob = fc1.dumps()
assert_same_fc(fc1, FeatureCollection.loads(blob))
## deserialize it via chunk
fc2 = FeatureCollection.loads(fc1.dumps())
assert_same_fc(fc1, fc2)
def test_entity(counter_type):
## build entity, serialize, deserialize, and verify its multisets
fc1 = FeatureCollection()
fc1["bow"] += counter_type(Counter(["big", "dog"]))
fc1["bow"] += counter_type(Counter("tall building"))
fc1["bon"] += counter_type(Counter(["Super Cat", "Small Cat", "Tiger Fish"]))
## there should be nine items of size 1
assert Counter(map(abs, fc1["bow"].values()))[1] == 10, fc1["bow"].items()
## double the counts, should recurse down
fc1 += fc1
## check values doubled
assert Counter(map(abs, fc1["bow"].values()))[2] == 10, fc1["bow"].items()
## serialize/deserialize it
blob = fc1.dumps()
assert_same_fc(fc1, FeatureCollection.loads(blob))
## deserialize it via chunk
fc2 = FeatureCollection.loads(fc1.dumps())
assert_same_fc(fc1, fc2)
def test_json_serializer():
with registry:
registry.add('StringCounter', JsonSerializer)
fc = FeatureCollection()
fc['thing2'] = StringCounter(dict(hello='people'))
fc['thing2']['another'] = 5
fc['thing3'] = StringCounter(dict(hello='people2'))
fc_str = fc.dumps()
fc2 = FeatureCollection.loads(fc_str)
assert fc2['thing2']['another'] == 5
assert fc2['thing2']['hello'] == 'people'
assert fc2['thing3']['hello'] == 'people2'
def test_thing_serializer():
with registry:
registry.add('StringCounter', ThingSerializer)
fc = FeatureCollection()
fc['thing1'] = Thing(json.dumps(dict(hello='people')))
fc['thing1']['another'] = 'more'
fc['thing1'].do_more_things()
fc_str = fc.dumps()
fc2 = FeatureCollection.loads(fc_str)
assert fc2['thing1']['another'] == 'more'
assert fc2['thing1']['hello'] == 'people'
assert fc2['thing1']['doing'] == 'something'
def perftest_throughput_feature_collection():
with registry:
registry.add('StringCounter', ThingSerializer)
fc = FeatureCollection()
fc['thing1'] = Thing(json.dumps(dict(one_mb=' ' * 2**20)))
fc_str = fc.dumps()
start_time = time.time()
num = 1000
for i in range(num):
fc2 = FeatureCollection.loads(fc_str)
fc2.dumps()
elapsed = time.time() - start_time
rate = float(num) / elapsed
print('%d MB in %.1f sec --> %.1f MB per sec' % (num, elapsed, rate))
def fc_loads(raw):
return FeatureCollection.loads(snappy.decompress(raw))
def test_non_counter_features_serialize():
fc = FeatureCollection({"NAME": u"foobaz"})
fc = FeatureCollection.loads(fc.dumps())
def test_unicode():
fc = FeatureCollection({'NAME': {'foo': 1}})
fc = FeatureCollection.loads(fc.dumps())
assert type(fc.keys()[0]) is unicode
assert type(fc['NAME'].keys()[0]) is unicode
def test_geo_roundtrip():
fc = FeatureCollection()
fc['!co_LOC']['foo'].append((-55, 22, 0, None))
fc2 = FeatureCollection.loads(fc.dumps())
assert fc['!co_LOC'] == fc2['!co_LOC']
def test_non_counter_features_serialize():
fc = FeatureCollection({'NAME': u'foobaz'})
fc = FeatureCollection.loads(fc.dumps())
def fc_loads(raw):
return FeatureCollection.loads(snappy.decompress(raw))
