def test_distance_vector(self):
# [mtl, qc, qc]
lat = array([45., 46., 46.])
lon = array([73., 71., 71.])
# [[mtl-mtl, mtl-qc, mtl-qc],
# [qc-mtl, qc-qc, qc-qc],
# [qc-mtl, qc-qc, qc-qc]]
ds = lat_lon_fast_distance(lat.reshape(-1, 1), lon.reshape(-1, 1),
lat.reshape(1, -1), lon.reshape(1, -1))
mtl_qc = 191.5
expected = array([[0, mtl_qc, mtl_qc], [mtl_qc, 0, 0], [mtl_qc, 0, 0]])
assert_allclose(ds, expected, 0, 0.05)
def test_distance_vector(self):
# [mtl, qc, qc]
lat = array([45., 46., 46.])
lon = array([73., 71., 71.])
# [[mtl-mtl, mtl-qc, mtl-qc],
# [qc-mtl, qc-qc, qc-qc],
# [qc-mtl, qc-qc, qc-qc]]
ds = lat_lon_fast_distance(lat.reshape(-1,1), lon.reshape(-1,1),
lat.reshape(1,-1), lon.reshape(1,-1))
mtl_qc = 191.5
expected = array([[0, mtl_qc, mtl_qc], [mtl_qc, 0, 0], [mtl_qc, 0, 0]])
assert_allclose(ds, expected, 0, 0.05)
def distance_fn(tuple1, tuple2):
_,lat1,lon1 = tuple1
_,lat2,lon2 = tuple2
return lat_lon_fast_distance(lat1, lon1, lat2, lon2)
def add_priority_index(session, fast_mode=False):
""" decides the order in which the cities should be selected """
cities = session.query(City,
func.ST_Y(cast(City.location, Geometry())),
func.ST_X(cast(City.location, Geometry()))) \
.join(MonthlyStat) \
.order_by(City.region_rank, City.country_rank,
desc(func.count(MonthlyStat.id))) \
.group_by(City.id) \
.yield_per(1000).all()
if fast_mode:
logger.info('doing the fast version of priority index')
for i,city in enumerate(cities):
city[0].priority_index = i
session.commit()
return
def distance_fn(tuple1, tuple2):
_,lat1,lon1 = tuple1
_,lat2,lon2 = tuple2
return lat_lon_fast_distance(lat1, lon1, lat2, lon2)
indices = [0]
indices_left = list(range(1,len(cities)))
# pre-calculate the distances between all the cities
logger.info('pre-calculating the distances between all cities')
lats = numpy.array([c[1] for c in cities])
lons = numpy.array([c[2] for c in cities])
distances = lat_lon_fast_distance(lats.reshape(-1,1),
lons.reshape(-1,1),
lats.reshape(1,-1),
lons.reshape(1,-1))
class CityComp(object):
idx = None
max_dist = None
max_dist_idx = None
def __init__(self, max_dist, max_dist_idx):
self.max_dist = max_dist
self.max_dist_idx = max_dist_idx
def __lt__(self, other):
return self.max_dist < other.max_dist
# each city is compared to all the previous ones (maximum)
timer = Timer(len(indices_left))
# percent of closest cities to choose from
perc_closest_cities = 0.1
# same but max
max_closest_cities = 200
while len(indices_left) > 0:
# let's find the next city amongst the next candidates
# this will be our (heap) list of good candidates, i.e. the ones
# farthest from all the others
good_candidates = []
nb_keep = min(perc_closest_cities * len(indices_left),
max_closest_cities)
nb_keep = max(1, nb_keep) # at least 1!
logger.debug('will keep the farthest %i', nb_keep)
# max_dist = 0.
# max_dist_idx = 0
logger.debug('---------looking for the next one----------')
for no_candidate, i_left in enumerate(indices_left):
# logger.debug('candidate %i, idx %i', no_candidate, i_left)
# find how close is the nearest neighbor for this city
# we are looking for the city with the fartest nearest neighbor
dist_nearest_neighbor = 1e9
# get the distance of our candidate to the closest (already chosen)
# city
too_close = False
for i_chosen in indices:
cur_dist = distances[i_chosen, i_left]
# if we already have enough candidates, and if the current is
# worse than all others, let's skip it
if len(good_candidates) >= nb_keep \
and cur_dist <= good_candidates[0].max_dist:
too_close = True
# logger.debug('too close @%f', cur_dist)
break
dist_nearest_neighbor = min(dist_nearest_neighbor, cur_dist)
# we don't compare the distance of this candidate with all cities
# if it's closer to (already chosen) city than our best candidate
# so far
if too_close:
continue
# dist_nearest_neighbor = numpy.min(distances[indices][:,i_left])
# logger.debug('candidate %i has a city at %f', no_candidate,
# dist_nearest_neighbor)
# if dist_nearest_neighbor > best_candidate.max_dist:
# logger.debug('(new max)')
new_candidate = CityComp(dist_nearest_neighbor, no_candidate)
# logger.debug('trying to add new candidate with dist %f',
# new_candidate.max_dist)
# if we don't have enough anyway
if len(good_candidates) < nb_keep:
heapq.heappush(good_candidates, new_candidate)
else:
# if we have enough, just keep the n best
rejected_cand = heapq.heappushpop(good_candidates,
new_candidate)
# logger.debug('removed candidate %i with dist %f',
# rejected_cand.max_dist_idx,
# rejected_cand.max_dist)
# take the smallest index in our good candidates. this corresponds to
# the best (according to our first ORDER BY) amongst the "far enough"
# candidates
best_candidate = min(good_candidates, key=lambda x: x.max_dist_idx)
logger.debug('keeping %s with pop %i',
cities[indices_left[best_candidate.max_dist_idx]][0].name,
cities[indices_left[best_candidate.max_dist_idx]][0].population,)
# input('press to continue')
indices.append(indices_left.pop(best_candidate.max_dist_idx))
logger.debug('done, best candidate was %i with distance %f',
best_candidate.max_dist_idx, best_candidate.max_dist)
logger.debug('done, chosen: %i, remaining: %i', len(indices),
len(indices_left))
timer.update()
assert len(indices) == len(cities)
for priority_index, i in enumerate(indices):
cities[i][0].priority_index = priority_index
session.commit()
def distance_fn(tuple1, tuple2):
_, lat1, lon1 = tuple1
_, lat2, lon2 = tuple2
return lat_lon_fast_distance(lat1, lon1, lat2, lon2)
def add_priority_index(session, fast_mode=False):
""" decides the order in which the cities should be selected """
cities = session.query(City,
func.ST_Y(cast(City.location, Geometry())),
func.ST_X(cast(City.location, Geometry()))) \
.join(MonthlyStat) \
.order_by(City.region_rank, City.country_rank,
desc(func.count(MonthlyStat.id))) \
.group_by(City.id) \
.yield_per(1000).all()
if fast_mode:
logger.info('doing the fast version of priority index')
for i, city in enumerate(cities):
city[0].priority_index = i
session.commit()
return
def distance_fn(tuple1, tuple2):
_, lat1, lon1 = tuple1
_, lat2, lon2 = tuple2
return lat_lon_fast_distance(lat1, lon1, lat2, lon2)
indices = [0]
indices_left = list(range(1, len(cities)))
# pre-calculate the distances between all the cities
logger.info('pre-calculating the distances between all cities')
lats = numpy.array([c[1] for c in cities])
lons = numpy.array([c[2] for c in cities])
distances = lat_lon_fast_distance(lats.reshape(-1, 1), lons.reshape(-1, 1),
lats.reshape(1, -1), lons.reshape(1, -1))
class CityComp(object):
idx = None
max_dist = None
max_dist_idx = None
def __init__(self, max_dist, max_dist_idx):
self.max_dist = max_dist
self.max_dist_idx = max_dist_idx
def __lt__(self, other):
return self.max_dist < other.max_dist
# each city is compared to all the previous ones (maximum)
timer = Timer(len(indices_left))
# percent of closest cities to choose from
perc_closest_cities = 0.1
# same but max
max_closest_cities = 200
while len(indices_left) > 0:
# let's find the next city amongst the next candidates
# this will be our (heap) list of good candidates, i.e. the ones
# farthest from all the others
good_candidates = []
nb_keep = min(perc_closest_cities * len(indices_left),
max_closest_cities)
nb_keep = max(1, nb_keep) # at least 1!
logger.debug('will keep the farthest %i', nb_keep)
# max_dist = 0.
# max_dist_idx = 0
logger.debug('---------looking for the next one----------')
for no_candidate, i_left in enumerate(indices_left):
# logger.debug('candidate %i, idx %i', no_candidate, i_left)
# find how close is the nearest neighbor for this city
# we are looking for the city with the fartest nearest neighbor
dist_nearest_neighbor = 1e9
# get the distance of our candidate to the closest (already chosen)
# city
too_close = False
for i_chosen in indices:
cur_dist = distances[i_chosen, i_left]
# if we already have enough candidates, and if the current is
# worse than all others, let's skip it
if len(good_candidates) >= nb_keep \
and cur_dist <= good_candidates[0].max_dist:
too_close = True
# logger.debug('too close @%f', cur_dist)
break
dist_nearest_neighbor = min(dist_nearest_neighbor, cur_dist)
# we don't compare the distance of this candidate with all cities
# if it's closer to (already chosen) city than our best candidate
# so far
if too_close:
continue
# dist_nearest_neighbor = numpy.min(distances[indices][:,i_left])
# logger.debug('candidate %i has a city at %f', no_candidate,
# dist_nearest_neighbor)
# if dist_nearest_neighbor > best_candidate.max_dist:
# logger.debug('(new max)')
new_candidate = CityComp(dist_nearest_neighbor, no_candidate)
# logger.debug('trying to add new candidate with dist %f',
# new_candidate.max_dist)
# if we don't have enough anyway
if len(good_candidates) < nb_keep:
heapq.heappush(good_candidates, new_candidate)
else:
# if we have enough, just keep the n best
rejected_cand = heapq.heappushpop(good_candidates,
new_candidate)
# logger.debug('removed candidate %i with dist %f',
# rejected_cand.max_dist_idx,
# rejected_cand.max_dist)
# take the smallest index in our good candidates. this corresponds to
# the best (according to our first ORDER BY) amongst the "far enough"
# candidates
best_candidate = min(good_candidates, key=lambda x: x.max_dist_idx)
logger.debug(
'keeping %s with pop %i',
cities[indices_left[best_candidate.max_dist_idx]][0].name,
cities[indices_left[best_candidate.max_dist_idx]][0].population,
)
# input('press to continue')
indices.append(indices_left.pop(best_candidate.max_dist_idx))
logger.debug('done, best candidate was %i with distance %f',
best_candidate.max_dist_idx, best_candidate.max_dist)
logger.debug('done, chosen: %i, remaining: %i', len(indices),
len(indices_left))
timer.update()
assert len(indices) == len(cities)
for priority_index, i in enumerate(indices):
cities[i][0].priority_index = priority_index
session.commit()
