def test_shard_model(self):
self.assertIs(BlueShard.shard_model('111101123456'), BlueShard0)
self.assertIs(BlueShard.shard_model('0000f0123456'), BlueShardF)
self.assertIs(BlueShard.shard_model(''), None)
self.assertIs(BlueShard.shard_model(None), None)
mac = encode_mac('0000f0123456')
self.assertEqual(BlueShard.shard_model(mac), BlueShardF)
def test_shard_id(self):
assert WifiShard.shard_id('111101123456') == '0'
assert WifiShard.shard_id('0000f0123456') == 'f'
assert WifiShard.shard_id('') is None
assert WifiShard.shard_id(None) is None
mac = encode_mac('0000f0123456')
assert WifiShard.shard_id(mac) == 'f'
def test_shard_id(self):
self.assertEqual(BlueShard.shard_id('111101123456'), '0')
self.assertEqual(BlueShard.shard_id('0000f0123456'), 'f')
self.assertEqual(BlueShard.shard_id(''), None)
self.assertEqual(BlueShard.shard_id(None), None)
mac = encode_mac('0000f0123456')
self.assertEqual(BlueShard.shard_id(mac), 'f')
def test_shard_model(self):
assert WifiShard.shard_model('111101123456') is WifiShard0
assert WifiShard.shard_model('0000f0123456') is WifiShardF
assert WifiShard.shard_model('') is None
assert WifiShard.shard_model(None) is None
mac = encode_mac('0000f0123456')
assert WifiShard.shard_model(mac) is WifiShardF
def test_shard_id(self):
assert WifiShard.shard_id('111101123456') == '0'
assert WifiShard.shard_id('0000f0123456') == 'f'
assert WifiShard.shard_id('') is None
assert WifiShard.shard_id(None) is None
mac = encode_mac('0000f0123456')
assert WifiShard.shard_id(mac) == 'f'
def test_shard_model(self):
assert WifiShard.shard_model('111101123456') is WifiShard0
assert WifiShard.shard_model('0000f0123456') is WifiShardF
assert WifiShard.shard_model('') is None
assert WifiShard.shard_model(None) is None
mac = encode_mac('0000f0123456')
assert WifiShard.shard_model(mac) is WifiShardF
def test_shard_id(self):
assert WifiShard.shard_id("111101123456") == "0"
assert WifiShard.shard_id("0000f0123456") == "f"
assert WifiShard.shard_id("") is None
assert WifiShard.shard_id(None) is None
mac = encode_mac("0000f0123456")
assert WifiShard.shard_id(mac) == "f"
def test_shard_model(self):
assert WifiShard.shard_model("111101123456") is WifiShard0
assert WifiShard.shard_model("0000f0123456") is WifiShardF
assert WifiShard.shard_model("") is None
assert WifiShard.shard_model(None) is None
mac = encode_mac("0000f0123456")
assert WifiShard.shard_model(mac) is WifiShardF
def _cache_keys_wifi(self, wifi_query):
keys = []
for wifi in wifi_query:
keys.append(self.cache_key_wifi + encode_mac(wifi.mac))
return keys
def _cache_keys_blue(self, blue_query):
keys = []
for blue in blue_query:
keys.append(self.cache_key_blue + encode_mac(blue.mac))
return keys
def test_used_networks(self):
wifi = WifiShardFactory.build()
network = ('wifi', encode_mac(wifi.mac), True)
assert Result().used_networks == []
assert Result(used_networks=[network]).used_networks == [network]
def test_used_networks(self):
wifi = WifiShardFactory.build()
network = ('wifi', encode_mac(wifi.mac), True)
assert Result().used_networks == []
assert Result(used_networks=[network]).used_networks == [network]
def cluster_networks(models,
lookups,
min_age=0,
min_radius=None,
min_signal=None,
max_distance=None):
"""
Given a list of database models and lookups, return
a list of clusters of nearby networks.
"""
now = util.utcnow()
today = now.date()
# Create a dict of macs mapped to their age and signal strength.
obs_data = {}
for lookup in lookups:
obs_data[decode_mac(lookup.mac)] = (
max(abs(lookup.age or min_age), 1000),
lookup.signalStrength or min_signal,
)
networks = numpy.array(
[(
model.lat,
model.lon,
model.radius or min_radius,
obs_data[model.mac][0],
obs_data[model.mac][1],
station_score(model, now),
encode_mac(model.mac, codec="base64"),
bool(model.last_seen is not None and model.last_seen >= today),
) for model in models],
dtype=NETWORK_DTYPE,
)
# Only consider clusters that have at least 2 found networks
# inside them. Otherwise someone could use a combination of
# one real network and one fake and therefor not found network to
# get the position of the real network.
length = len(networks)
if length < 2:
# Not enough networks to form a valid cluster.
return []
positions = networks[["lat", "lon"]]
if length == 2:
one = positions[0]
two = positions[1]
if distance(one[0], one[1], two[0], two[1]) <= max_distance:
# Only two networks and they agree, so cluster them.
return [networks]
else:
# Or they disagree forming two clusters of size one,
# neither of which is large enough to be returned.
return []
# Calculate the condensed distance matrix based on distance in meters.
# This avoids calculating the square form, which would calculate
# each value twice and avoids calculating the diagonal of zeros.
# We avoid the special cases for length < 2 with the above checks.
# See scipy.spatial.distance.squareform and
# https://stackoverflow.com/questions/13079563
dist_matrix = numpy.zeros(length * (length - 1) // 2, dtype=numpy.double)
for i, (a, b) in enumerate(itertools.combinations(positions, 2)):
dist_matrix[i] = distance(a[0], a[1], b[0], b[1])
link_matrix = hierarchy.linkage(dist_matrix, method="complete")
assignments = hierarchy.fcluster(link_matrix,
max_distance,
criterion="distance",
depth=2)
indexed_clusters = defaultdict(list)
for i, net in zip(assignments, networks):
indexed_clusters[i].append(net)
clusters = []
for values in indexed_clusters.values():
if len(values) >= 2:
clusters.append(numpy.array(values, dtype=NETWORK_DTYPE))
return clusters
def _cache_keys_wifi(self, wifi_query):
keys = []
for wifi in wifi_query:
keys.append(self.cache_key_wifi + encode_mac(wifi.mac))
return keys
def _cache_keys_blue(self, blue_query):
keys = []
for blue in blue_query:
keys.append(self.cache_key_blue + encode_mac(blue.mac))
return keys
def cluster_networks(models, lookups,
min_age=0, min_radius=None, min_signal=None,
max_distance=None):
"""
Given a list of database models and lookups, return
a list of clusters of nearby networks.
"""
now = util.utcnow()
today = now.date()
# Create a dict of macs mapped to their age and signal strength.
obs_data = {}
for lookup in lookups:
obs_data[decode_mac(lookup.mac)] = (
max(abs(lookup.age or min_age), 1000),
lookup.signalStrength or min_signal)
networks = numpy.array([(
model.lat, model.lon,
model.radius or min_radius,
obs_data[model.mac][0],
obs_data[model.mac][1],
station_score(model, now),
encode_mac(model.mac),
bool(model.last_seen >= today))
for model in models],
dtype=NETWORK_DTYPE)
# Only consider clusters that have at least 2 found networks
# inside them. Otherwise someone could use a combination of
# one real network and one fake and therefor not found network to
# get the position of the real network.
length = len(networks)
if length < 2:
# Not enough networks to form a valid cluster.
return []
positions = networks[['lat', 'lon']]
if length == 2:
one = positions[0]
two = positions[1]
if distance(one[0], one[1],
two[0], two[1]) <= max_distance:
# Only two networks and they agree, so cluster them.
return [networks]
else:
# Or they disagree forming two clusters of size one,
# neither of which is large enough to be returned.
return []
# Calculate the condensed distance matrix based on distance in meters.
# This avoids calculating the square form, which would calculate
# each value twice and avoids calculating the diagonal of zeros.
# We avoid the special cases for length < 2 with the above checks.
# See scipy.spatial.distance.squareform and
# https://stackoverflow.com/questions/13079563
dist_matrix = numpy.zeros(length * (length - 1) // 2, dtype=numpy.double)
for i, (a, b) in enumerate(itertools.combinations(positions, 2)):
dist_matrix[i] = distance(a[0], a[1], b[0], b[1])
link_matrix = hierarchy.linkage(dist_matrix, method='complete')
assignments = hierarchy.fcluster(
link_matrix, max_distance, criterion='distance', depth=2)
indexed_clusters = defaultdict(list)
for i, net in zip(assignments, networks):
indexed_clusters[i].append(net)
clusters = []
for values in indexed_clusters.values():
if len(values) >= 2:
clusters.append(numpy.array(values, dtype=NETWORK_DTYPE))
return clusters
