def generate_rtree_from_entities(self): """Create an rtree with all entities with bounding rectangles.""" self.bounding_rects = { id_: e.bounding_rect for id_, e in self.entities.items() if hasattr(e, 'bounding_rect') } self.rtree = Rtree( (id_, rect, None) for id_, rect in self.bounding_rects.items())
def createRtree(data): """ Creates an R-Tree from the given data """ tree = Rtree() for index, pair in enumerate(data): tree.insert(index, (pair[3], pair[4]), obj=pair) return tree
def reset(self, name: Optional[str] = None): """Reset index and set name.""" self.name = name self.id_count = count() self.entities = {} self.bounding_rects = {} self.rtree = Rtree() self.path_map = PathMap() self.register_updates = False self.simulation = Simulation() self.stats = defaultdict(dict) self._updates = set()
def _build_conflict_points(node: Node, curves: Dict[LaneConnection, Curve]): """Create conflict points from lane connection curves. Fill conflict points in given Curve objects. Merges points that are less than MERGE_RADIUS apart and add points that are within NEIGHBOR_RADIUS as neighbors. """ id_generator = count() points: Dict[int, ConflictPoint] = {} diverge = defaultdict(set) merge = defaultdict(set) for (lanes1, curve1), (lanes2, curve2) in combinations(curves.items(), 2): if lanes1[0] == lanes2[0]: diverge[lanes1[0]].update((curve1, curve2)) if lanes1[1] == lanes2[1]: merge[lanes1[1]].update((curve1, curve2)) _add_crossing_conflict_points(id_generator, points, curve1, curve2) _add_diverge_merge_conflict_points(id_generator, points, diverge, merge) if len(points) > 1: rtree = Rtree((id_, p.point.bounding_rect, None) for id_, p in points.items()) _merge_points(points, rtree) _fill_neighbors(points, rtree) for point in points.values(): point.create_lock_order() # This makes the conflict point positions relative to the node. point.point = point.point - node.position for curve in curves.values(): curve.remove_redundant_conflict_point() # Remove from neighbors points that aren't in any curve points = {p for _, p in chain.from_iterable(c.conflict_points for c in curves.values())} for point in points: for neighbor in list(point.neighbors): if neighbor not in points: point.neighbors.remove(neighbor)
def main(query, train, query_num, qgram_size): logger = logging.getLogger('search_rtree') logger.setLevel(logging.DEBUG) fh = logging.FileHandler('./log/%s' % query) fh.setLevel(logging.DEBUG) # create console handler with a higher log level ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) # create formatter and add it to the handlers formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) ch.setFormatter(formatter) # add the handlers to the logger logger.addHandler(fh) logger.addHandler(ch) logger.info( '------------------------- Calculate common Q-grams for query trajectories -------------------------' ) qgram_tag = 'q_%d' % qgram_size query_path = './data/processed/%s.txt' % query rtree_path = './data/interim/%s/my_rtree_%s' % (train, qgram_tag) logger.info('Query trajectory path: %s' % query_path) logger.info('Rtree path: %s' % rtree_path) query_data = load_trajectory(query_path, n=query_num) logger.info('Load %d query trajectories' % query_num) qry_qgram, qry_id_list = build_qgram(query_data, qgram_size) qry_id_dict = build_id_to_key( qry_id_list) # key: query_id, value: query_key data_index = Rtree(rtree_path) conf = SparkConf().setAppName("PythonWordCount").setMaster("local") sc = SparkContext(conf=conf) all_data = [] for qry_id, qry_qgrams in qry_qgram.items(): qry_key = qry_id_dict[qry_id] data = [] for qry_qgram in qry_qgrams: matches = [ hit.object for hit in data_index.intersection(qry_qgram, objects=True) ] matches = set(matches) data.append(list(matches)) flat_data = [item for sublist in data for item in sublist] # print(flat_data) dist_data = sc.parallelize(flat_data) map_data = dist_data.map(lambda x: (x, 1)) reduce_data = map_data.reduceByKey(lambda a, b: a + b).sortBy( lambda x: x[1], ascending=False).collect() # print(reduce_data) all_data.append([qry_key, reduce_data]) if not os.path.exists('./data/interim/%s' % query): os.mkdir('./data/interim/%s' % query) if not os.path.exists('./data/interim/%s/%s' % (query, train)): os.mkdir('./data/interim/%s/%s' % (query, train)) candidate_traj_path = './data/interim/%s/%s/candidate_trajectory_%s.txt' % ( query, train, qgram_tag) save_pickle(all_data, candidate_traj_path) logger.info('Output candidate_trajectory: %s' % candidate_traj_path) query_id_dict_path = './data/interim/%s/%s/query_id_dict_%s.txt' % ( query, train, qgram_tag) logger.info('Output query_id_dict: %s' % query_id_dict_path) save_pickle(qry_id_dict, query_id_dict_path) gc.collect()
RTreePage.objects(name=self.name, page=page).delete(safe=True) except: returnError.contents.value = self.InvalidPageError hasData = property( lambda self: RTreePage.objects(name=self.name).first() is not None) if __name__ == '__main__': settings = Property() settings.writethrough = True settings.buffering_capacity = 1 storage = MongoStorage('test') storage.clear() r = Rtree(storage, properties=settings) r.add(123, (0, 0, 1, 1)) print "test 1 should be true" item = list(r.nearest((0, 0), 1, objects=True))[0] print item.id print r.valid() print "test 2 should be true" r.delete(123, (0, 0, 1, 1)) print r.valid() print "test 3 should be true" r.clearBuffer() print r.valid()
def main(train, qgram_size): logger = logging.getLogger('build_rtree') logger.setLevel(logging.DEBUG) fh = logging.FileHandler('./log/%s' % train) fh.setLevel(logging.DEBUG) # create console handler with a higher log level ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) # create formatter and add it to the handlers formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) ch.setFormatter(formatter) # add the handlers to the logger logger.addHandler(fh) logger.addHandler(ch) logger.info('---------------------------- Build R-tree ----------------------------') qgram_tag = 'q_%d' % qgram_size train_path = './data/processed/%s.txt' % train data = load_trajectory(train_path) logger.info('Load train trajectory: %s' % train_path) trajectory, id_list = build_qgram(data, qgram_size) id_to_key_dict = build_id_to_key(id_list) # order_key_dict = build_order_dict(id_list) #save orderId-key mapping #key: trajectory id in string, value: encoded key rtree_id_dict_path = './data/interim/%s/rtree_id_dict_%s.txt' % (train, qgram_tag) save_pickle(id_to_key_dict, rtree_id_dict_path) logger.info('Output rtree_id_dict: %s' % rtree_id_dict_path) #key: key, value: trajectory id in string # filename = '../data/processed/order_key_dict.txt' # outfile = open(filename,'wb') # pickle.dump(order_key_dict,outfile) # outfile.close() # R-tree constructor # parameter: 'data_full' is the filename of R-tree storage # 2 files are created: data_full.dat, data_full.idx # return: r-tree index rtree_path = './data/interim/%s/my_rtree_%s' % (train, qgram_tag) data_idx = Rtree(rtree_path) logger.info('Output R-tree: %s' % rtree_path) # put all trajectories into r-tree in the form of bounding box node_id = 0 start_time = time.time() for key, qgrams in trajectory.items(): for qgram in qgrams: # parameters: # 1. node id # 2. bounding box(point): (x,y,x,y) # 3. data inside each node: trajectory's key from order_dict x = np.around(qgram[0], decimals=5) y = np.around(qgram[1], decimals=5) data_idx.insert(node_id, (x, y, x, y), obj=(id_to_key_dict[key])) node_id += 1 del data_idx end_time = time.time() logger.info("exec time: "+str(end_time-start_time)) logger.info('Finished building R-tree')
def searching(feature_vectors_database, feature_vectors_retrieval, similarity_metric, image_paths, retrieval_number, file, list_of_parameters, feature_extraction_method, path_database): ''' feature_vectors: atriutos calculados labels: label de cada classe similarity_metric: qual medida utilizar recuperados as k imagens com menor distancia. Se k = 0, entao o valor eh setado como sendo o tamanho da classe da imagem ''' #name to save the pickle file parameters_name = "" for parameter in list_of_parameters: parameters_name = parameters_name + "_" + parameter file = path_database + "features/sortingRTree" + "_" + feature_extraction_method + parameters_name + '_' + similarity_metric feature_vectors_retrieval = preprocessing.scale(feature_vectors_retrieval) if not (os.path.isfile(file + '.dat')): #normalize signatures feature_vectors_database = preprocessing.scale( feature_vectors_database) # Create a N-Dimensional index p = index.Property() p.dimension = feature_vectors_database.shape[1] idx = index.Index(file, properties=p) # Create the tree for i, vector in enumerate(feature_vectors_database): idx.add(i, vector.tolist()) #save_format = idx.dumps(idx) #with open(file, 'wb') as handle: #pickle.dump(save_format, handle) else: # Create a N-Dimensional index p = index.Property() p.dimension = feature_vectors_database.shape[1] idx = Rtree(file, properties=p) #with open(file, 'rb') as handle: # idx = pickle.load(handle) # Find closests pair for the first N points ########### debug this part ########### small_distances = [] for id1, query in enumerate(feature_vectors_retrieval): nearest = list(idx.nearest(query.tolist(), retrieval_number)) small_distances.append(nearest) result = [] for cont1, i in enumerate(small_distances): aux = [] for j in i: aux.append(image_paths[j]) result.append(aux) return result