def test_distance_vector(self): # [mtl, qc, qc] lat = array([45., 46., 46.]) lon = array([73., 71., 71.]) # [[mtl-mtl, mtl-qc, mtl-qc], # [qc-mtl, qc-qc, qc-qc], # [qc-mtl, qc-qc, qc-qc]] ds = lat_lon_fast_distance(lat.reshape(-1, 1), lon.reshape(-1, 1), lat.reshape(1, -1), lon.reshape(1, -1)) mtl_qc = 191.5 expected = array([[0, mtl_qc, mtl_qc], [mtl_qc, 0, 0], [mtl_qc, 0, 0]]) assert_allclose(ds, expected, 0, 0.05)
def test_distance_vector(self): # [mtl, qc, qc] lat = array([45., 46., 46.]) lon = array([73., 71., 71.]) # [[mtl-mtl, mtl-qc, mtl-qc], # [qc-mtl, qc-qc, qc-qc], # [qc-mtl, qc-qc, qc-qc]] ds = lat_lon_fast_distance(lat.reshape(-1,1), lon.reshape(-1,1), lat.reshape(1,-1), lon.reshape(1,-1)) mtl_qc = 191.5 expected = array([[0, mtl_qc, mtl_qc], [mtl_qc, 0, 0], [mtl_qc, 0, 0]]) assert_allclose(ds, expected, 0, 0.05)
def distance_fn(tuple1, tuple2): _,lat1,lon1 = tuple1 _,lat2,lon2 = tuple2 return lat_lon_fast_distance(lat1, lon1, lat2, lon2)
def add_priority_index(session, fast_mode=False): """ decides the order in which the cities should be selected """ cities = session.query(City, func.ST_Y(cast(City.location, Geometry())), func.ST_X(cast(City.location, Geometry()))) \ .join(MonthlyStat) \ .order_by(City.region_rank, City.country_rank, desc(func.count(MonthlyStat.id))) \ .group_by(City.id) \ .yield_per(1000).all() if fast_mode: logger.info('doing the fast version of priority index') for i,city in enumerate(cities): city[0].priority_index = i session.commit() return def distance_fn(tuple1, tuple2): _,lat1,lon1 = tuple1 _,lat2,lon2 = tuple2 return lat_lon_fast_distance(lat1, lon1, lat2, lon2) indices = [0] indices_left = list(range(1,len(cities))) # pre-calculate the distances between all the cities logger.info('pre-calculating the distances between all cities') lats = numpy.array([c[1] for c in cities]) lons = numpy.array([c[2] for c in cities]) distances = lat_lon_fast_distance(lats.reshape(-1,1), lons.reshape(-1,1), lats.reshape(1,-1), lons.reshape(1,-1)) class CityComp(object): idx = None max_dist = None max_dist_idx = None def __init__(self, max_dist, max_dist_idx): self.max_dist = max_dist self.max_dist_idx = max_dist_idx def __lt__(self, other): return self.max_dist < other.max_dist # each city is compared to all the previous ones (maximum) timer = Timer(len(indices_left)) # percent of closest cities to choose from perc_closest_cities = 0.1 # same but max max_closest_cities = 200 while len(indices_left) > 0: # let's find the next city amongst the next candidates # this will be our (heap) list of good candidates, i.e. the ones # farthest from all the others good_candidates = [] nb_keep = min(perc_closest_cities * len(indices_left), max_closest_cities) nb_keep = max(1, nb_keep) # at least 1! logger.debug('will keep the farthest %i', nb_keep) # max_dist = 0. # max_dist_idx = 0 logger.debug('---------looking for the next one----------') for no_candidate, i_left in enumerate(indices_left): # logger.debug('candidate %i, idx %i', no_candidate, i_left) # find how close is the nearest neighbor for this city # we are looking for the city with the fartest nearest neighbor dist_nearest_neighbor = 1e9 # get the distance of our candidate to the closest (already chosen) # city too_close = False for i_chosen in indices: cur_dist = distances[i_chosen, i_left] # if we already have enough candidates, and if the current is # worse than all others, let's skip it if len(good_candidates) >= nb_keep \ and cur_dist <= good_candidates[0].max_dist: too_close = True # logger.debug('too close @%f', cur_dist) break dist_nearest_neighbor = min(dist_nearest_neighbor, cur_dist) # we don't compare the distance of this candidate with all cities # if it's closer to (already chosen) city than our best candidate # so far if too_close: continue # dist_nearest_neighbor = numpy.min(distances[indices][:,i_left]) # logger.debug('candidate %i has a city at %f', no_candidate, # dist_nearest_neighbor) # if dist_nearest_neighbor > best_candidate.max_dist: # logger.debug('(new max)') new_candidate = CityComp(dist_nearest_neighbor, no_candidate) # logger.debug('trying to add new candidate with dist %f', # new_candidate.max_dist) # if we don't have enough anyway if len(good_candidates) < nb_keep: heapq.heappush(good_candidates, new_candidate) else: # if we have enough, just keep the n best rejected_cand = heapq.heappushpop(good_candidates, new_candidate) # logger.debug('removed candidate %i with dist %f', # rejected_cand.max_dist_idx, # rejected_cand.max_dist) # take the smallest index in our good candidates. this corresponds to # the best (according to our first ORDER BY) amongst the "far enough" # candidates best_candidate = min(good_candidates, key=lambda x: x.max_dist_idx) logger.debug('keeping %s with pop %i', cities[indices_left[best_candidate.max_dist_idx]][0].name, cities[indices_left[best_candidate.max_dist_idx]][0].population,) # input('press to continue') indices.append(indices_left.pop(best_candidate.max_dist_idx)) logger.debug('done, best candidate was %i with distance %f', best_candidate.max_dist_idx, best_candidate.max_dist) logger.debug('done, chosen: %i, remaining: %i', len(indices), len(indices_left)) timer.update() assert len(indices) == len(cities) for priority_index, i in enumerate(indices): cities[i][0].priority_index = priority_index session.commit()
def distance_fn(tuple1, tuple2): _, lat1, lon1 = tuple1 _, lat2, lon2 = tuple2 return lat_lon_fast_distance(lat1, lon1, lat2, lon2)
def add_priority_index(session, fast_mode=False): """ decides the order in which the cities should be selected """ cities = session.query(City, func.ST_Y(cast(City.location, Geometry())), func.ST_X(cast(City.location, Geometry()))) \ .join(MonthlyStat) \ .order_by(City.region_rank, City.country_rank, desc(func.count(MonthlyStat.id))) \ .group_by(City.id) \ .yield_per(1000).all() if fast_mode: logger.info('doing the fast version of priority index') for i, city in enumerate(cities): city[0].priority_index = i session.commit() return def distance_fn(tuple1, tuple2): _, lat1, lon1 = tuple1 _, lat2, lon2 = tuple2 return lat_lon_fast_distance(lat1, lon1, lat2, lon2) indices = [0] indices_left = list(range(1, len(cities))) # pre-calculate the distances between all the cities logger.info('pre-calculating the distances between all cities') lats = numpy.array([c[1] for c in cities]) lons = numpy.array([c[2] for c in cities]) distances = lat_lon_fast_distance(lats.reshape(-1, 1), lons.reshape(-1, 1), lats.reshape(1, -1), lons.reshape(1, -1)) class CityComp(object): idx = None max_dist = None max_dist_idx = None def __init__(self, max_dist, max_dist_idx): self.max_dist = max_dist self.max_dist_idx = max_dist_idx def __lt__(self, other): return self.max_dist < other.max_dist # each city is compared to all the previous ones (maximum) timer = Timer(len(indices_left)) # percent of closest cities to choose from perc_closest_cities = 0.1 # same but max max_closest_cities = 200 while len(indices_left) > 0: # let's find the next city amongst the next candidates # this will be our (heap) list of good candidates, i.e. the ones # farthest from all the others good_candidates = [] nb_keep = min(perc_closest_cities * len(indices_left), max_closest_cities) nb_keep = max(1, nb_keep) # at least 1! logger.debug('will keep the farthest %i', nb_keep) # max_dist = 0. # max_dist_idx = 0 logger.debug('---------looking for the next one----------') for no_candidate, i_left in enumerate(indices_left): # logger.debug('candidate %i, idx %i', no_candidate, i_left) # find how close is the nearest neighbor for this city # we are looking for the city with the fartest nearest neighbor dist_nearest_neighbor = 1e9 # get the distance of our candidate to the closest (already chosen) # city too_close = False for i_chosen in indices: cur_dist = distances[i_chosen, i_left] # if we already have enough candidates, and if the current is # worse than all others, let's skip it if len(good_candidates) >= nb_keep \ and cur_dist <= good_candidates[0].max_dist: too_close = True # logger.debug('too close @%f', cur_dist) break dist_nearest_neighbor = min(dist_nearest_neighbor, cur_dist) # we don't compare the distance of this candidate with all cities # if it's closer to (already chosen) city than our best candidate # so far if too_close: continue # dist_nearest_neighbor = numpy.min(distances[indices][:,i_left]) # logger.debug('candidate %i has a city at %f', no_candidate, # dist_nearest_neighbor) # if dist_nearest_neighbor > best_candidate.max_dist: # logger.debug('(new max)') new_candidate = CityComp(dist_nearest_neighbor, no_candidate) # logger.debug('trying to add new candidate with dist %f', # new_candidate.max_dist) # if we don't have enough anyway if len(good_candidates) < nb_keep: heapq.heappush(good_candidates, new_candidate) else: # if we have enough, just keep the n best rejected_cand = heapq.heappushpop(good_candidates, new_candidate) # logger.debug('removed candidate %i with dist %f', # rejected_cand.max_dist_idx, # rejected_cand.max_dist) # take the smallest index in our good candidates. this corresponds to # the best (according to our first ORDER BY) amongst the "far enough" # candidates best_candidate = min(good_candidates, key=lambda x: x.max_dist_idx) logger.debug( 'keeping %s with pop %i', cities[indices_left[best_candidate.max_dist_idx]][0].name, cities[indices_left[best_candidate.max_dist_idx]][0].population, ) # input('press to continue') indices.append(indices_left.pop(best_candidate.max_dist_idx)) logger.debug('done, best candidate was %i with distance %f', best_candidate.max_dist_idx, best_candidate.max_dist) logger.debug('done, chosen: %i, remaining: %i', len(indices), len(indices_left)) timer.update() assert len(indices) == len(cities) for priority_index, i in enumerate(indices): cities[i][0].priority_index = priority_index session.commit()